Multiple scene URLs (#3852)

* Add URLs scene relationship
* Update unit tests
* Update scene edit and details pages
* Update scrapers to use urls
* Post-process scenes during query scrape
* Update UI for URLs
* Change urls label
This commit is contained in:
WithoutPants 2023-07-12 11:51:52 +10:00 committed by GitHub
parent 76a4bfa49a
commit 67d4f9729a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 978 additions and 205 deletions

View file

@ -4,7 +4,7 @@ fragment SlimSceneData on Scene {
code code
details details
director director
url urls
date date
rating100 rating100
o_counter o_counter

View file

@ -4,7 +4,7 @@ fragment SceneData on Scene {
code code
details details
director director
url urls
date date
rating100 rating100
o_counter o_counter

View file

@ -114,7 +114,7 @@ fragment ScrapedSceneData on ScrapedScene {
code code
details details
director director
url urls
date date
image image
remote_site_id remote_site_id

View file

@ -40,7 +40,8 @@ type Scene {
code: String code: String
details: String details: String
director: String director: String
url: String url: String @deprecated(reason: "Use urls")
urls: [String!]
date: String date: String
# rating expressed as 1-5 # rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100") rating: Int @deprecated(reason: "Use 1-100 range with rating100")
@ -91,7 +92,8 @@ input SceneCreateInput {
code: String code: String
details: String details: String
director: String director: String
url: String url: String @deprecated(reason: "Use urls")
urls: [String!]
date: String date: String
# rating expressed as 1-5 # rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100") rating: Int @deprecated(reason: "Use 1-100 range with rating100")
@ -119,7 +121,8 @@ input SceneUpdateInput {
code: String code: String
details: String details: String
director: String director: String
url: String url: String @deprecated(reason: "Use urls")
urls: [String!]
date: String date: String
# rating expressed as 1-5 # rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100") rating: Int @deprecated(reason: "Use 1-100 range with rating100")
@ -164,7 +167,8 @@ input BulkSceneUpdateInput {
code: String code: String
details: String details: String
director: String director: String
url: String url: String @deprecated(reason: "Use urls")
urls: BulkUpdateStrings
date: String date: String
# rating expressed as 1-5 # rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100") rating: Int @deprecated(reason: "Use 1-100 range with rating100")

View file

@ -64,7 +64,8 @@ type ScrapedScene {
code: String code: String
details: String details: String
director: String director: String
url: String url: String @deprecated(reason: "use urls")
urls: [String!]
date: String date: String
"""This should be a base64 encoded data URL""" """This should be a base64 encoded data URL"""
@ -87,7 +88,8 @@ input ScrapedSceneInput {
code: String code: String
details: String details: String
director: String director: String
url: String url: String @deprecated(reason: "use urls")
urls: [String!]
date: String date: String
# no image, file, duration or relationships # no image, file, duration or relationships

View file

@ -405,3 +405,32 @@ func (r *sceneResolver) InteractiveSpeed(ctx context.Context, obj *models.Scene)
return primaryFile.InteractiveSpeed, nil return primaryFile.InteractiveSpeed, nil
} }
func (r *sceneResolver) URL(ctx context.Context, obj *models.Scene) (*string, error) {
if !obj.URLs.Loaded() {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
return obj.LoadURLs(ctx, r.repository.Scene)
}); err != nil {
return nil, err
}
}
urls := obj.URLs.List()
if len(urls) == 0 {
return nil, nil
}
return &urls[0], nil
}
func (r *sceneResolver) Urls(ctx context.Context, obj *models.Scene) ([]string, error) {
if !obj.URLs.Loaded() {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
return obj.LoadURLs(ctx, r.repository.Scene)
}); err != nil {
return nil, err
}
}
return obj.URLs.List(), nil
}

View file

@ -67,7 +67,6 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp
Code: translator.string(input.Code, "code"), Code: translator.string(input.Code, "code"),
Details: translator.string(input.Details, "details"), Details: translator.string(input.Details, "details"),
Director: translator.string(input.Director, "director"), Director: translator.string(input.Director, "director"),
URL: translator.string(input.URL, "url"),
Date: translator.datePtr(input.Date, "date"), Date: translator.datePtr(input.Date, "date"),
Rating: translator.ratingConversionInt(input.Rating, input.Rating100), Rating: translator.ratingConversionInt(input.Rating, input.Rating100),
Organized: translator.bool(input.Organized, "organized"), Organized: translator.bool(input.Organized, "organized"),
@ -83,6 +82,12 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp
return nil, fmt.Errorf("converting studio id: %w", err) return nil, fmt.Errorf("converting studio id: %w", err)
} }
if input.Urls != nil {
newScene.URLs = models.NewRelatedStrings(input.Urls)
} else if input.URL != nil {
newScene.URLs = models.NewRelatedStrings([]string{*input.URL})
}
var coverImageData []byte var coverImageData []byte
if input.CoverImage != nil && *input.CoverImage != "" { if input.CoverImage != nil && *input.CoverImage != "" {
var err error var err error
@ -168,7 +173,6 @@ func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTr
updatedScene.Code = translator.optionalString(input.Code, "code") updatedScene.Code = translator.optionalString(input.Code, "code")
updatedScene.Details = translator.optionalString(input.Details, "details") updatedScene.Details = translator.optionalString(input.Details, "details")
updatedScene.Director = translator.optionalString(input.Director, "director") updatedScene.Director = translator.optionalString(input.Director, "director")
updatedScene.URL = translator.optionalString(input.URL, "url")
updatedScene.Date = translator.optionalDate(input.Date, "date") updatedScene.Date = translator.optionalDate(input.Date, "date")
updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100)
updatedScene.OCounter = translator.optionalInt(input.OCounter, "o_counter") updatedScene.OCounter = translator.optionalInt(input.OCounter, "o_counter")
@ -182,6 +186,18 @@ func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTr
updatedScene.Organized = translator.optionalBool(input.Organized, "organized") updatedScene.Organized = translator.optionalBool(input.Organized, "organized")
if translator.hasField("urls") {
updatedScene.URLs = &models.UpdateStrings{
Values: input.Urls,
Mode: models.RelationshipUpdateModeSet,
}
} else if translator.hasField("url") {
updatedScene.URLs = &models.UpdateStrings{
Values: []string{*input.URL},
Mode: models.RelationshipUpdateModeSet,
}
}
if input.PrimaryFileID != nil { if input.PrimaryFileID != nil {
primaryFileID, err := strconv.Atoi(*input.PrimaryFileID) primaryFileID, err := strconv.Atoi(*input.PrimaryFileID)
if err != nil { if err != nil {
@ -339,7 +355,6 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU
updatedScene.Code = translator.optionalString(input.Code, "code") updatedScene.Code = translator.optionalString(input.Code, "code")
updatedScene.Details = translator.optionalString(input.Details, "details") updatedScene.Details = translator.optionalString(input.Details, "details")
updatedScene.Director = translator.optionalString(input.Director, "director") updatedScene.Director = translator.optionalString(input.Director, "director")
updatedScene.URL = translator.optionalString(input.URL, "url")
updatedScene.Date = translator.optionalDate(input.Date, "date") updatedScene.Date = translator.optionalDate(input.Date, "date")
updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100) updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100)
updatedScene.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id") updatedScene.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id")
@ -349,6 +364,18 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU
updatedScene.Organized = translator.optionalBool(input.Organized, "organized") updatedScene.Organized = translator.optionalBool(input.Organized, "organized")
if translator.hasField("urls") {
updatedScene.URLs = &models.UpdateStrings{
Values: input.Urls.Values,
Mode: input.Urls.Mode,
}
} else if translator.hasField("url") {
updatedScene.URLs = &models.UpdateStrings{
Values: []string{*input.URL},
Mode: models.RelationshipUpdateModeSet,
}
}
if translator.hasField("performer_ids") { if translator.hasField("performer_ids") {
updatedScene.PerformerIDs, err = translateUpdateIDs(input.PerformerIds.Ids, input.PerformerIds.Mode) updatedScene.PerformerIDs, err = translateUpdateIDs(input.PerformerIds.Ids, input.PerformerIds.Mode)
if err != nil { if err != nil {

View file

@ -68,6 +68,10 @@ func (r *mutationResolver) SubmitStashBoxSceneDraft(ctx context.Context, input S
logger.Errorf("Error getting scene cover: %v", err) logger.Errorf("Error getting scene cover: %v", err)
} }
if err := scene.LoadURLs(ctx, r.repository.Scene); err != nil {
return fmt.Errorf("loading scene URLs: %w", err)
}
res, err = client.SubmitSceneDraft(ctx, scene, boxes[input.StashBoxIndex].Endpoint, cover) res, err = client.SubmitSceneDraft(ctx, scene, boxes[input.StashBoxIndex].Endpoint, cover)
return err return err
}) })

View file

@ -176,7 +176,7 @@ func createScenes(ctx context.Context, sqb models.SceneReaderWriter, folderStore
s := &models.Scene{ s := &models.Scene{
Title: expectedMatchTitle, Title: expectedMatchTitle,
URL: existingStudioSceneName, Code: existingStudioSceneName,
StudioID: &existingStudioID, StudioID: &existingStudioID,
} }
if err := createScene(ctx, sqb, s, f); err != nil { if err := createScene(ctx, sqb, s, f); err != nil {
@ -625,7 +625,7 @@ func TestParseStudioScenes(t *testing.T) {
for _, scene := range scenes { for _, scene := range scenes {
// check for existing studio id scene first // check for existing studio id scene first
if scene.URL == existingStudioSceneName { if scene.Code == existingStudioSceneName {
if scene.StudioID == nil || *scene.StudioID != existingStudioID { if scene.StudioID == nil || *scene.StudioID != existingStudioID {
t.Error("Incorrectly overwrote studio ID for scene with existing studio ID") t.Error("Incorrectly overwrote studio ID for scene with existing studio ID")
} }

View file

@ -10,7 +10,7 @@ import (
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/scraper" "github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/txn" "github.com/stashapp/stash/pkg/txn"
"github.com/stashapp/stash/pkg/utils" "github.com/stashapp/stash/pkg/utils"
) )
@ -226,7 +226,7 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene,
return nil, fmt.Errorf("error converting tag ID %s: %w", *options.SkipSingleNamePerformerTag, err) return nil, fmt.Errorf("error converting tag ID %s: %w", *options.SkipSingleNamePerformerTag, err)
} }
tagIDs = intslice.IntAppendUnique(tagIDs, int(tagID)) tagIDs = sliceutil.AppendUnique(tagIDs, int(tagID))
} }
if tagIDs != nil { if tagIDs != nil {
ret.Partial.TagIDs = &models.UpdateIDs{ ret.Partial.TagIDs = &models.UpdateIDs{
@ -260,6 +260,9 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage
var updater *scene.UpdateSet var updater *scene.UpdateSet
if err := txn.WithTxn(ctx, txnManager, func(ctx context.Context) error { if err := txn.WithTxn(ctx, txnManager, func(ctx context.Context) error {
// load scene relationships // load scene relationships
if err := s.LoadURLs(ctx, t.SceneReaderUpdater); err != nil {
return err
}
if err := s.LoadPerformerIDs(ctx, t.SceneReaderUpdater); err != nil { if err := s.LoadPerformerIDs(ctx, t.SceneReaderUpdater); err != nil {
return err return err
} }
@ -320,7 +323,7 @@ func (t *SceneIdentifier) addTagToScene(ctx context.Context, txnManager txn.Mana
} }
existing := s.TagIDs.List() existing := s.TagIDs.List()
if intslice.IntInclude(existing, tagID) { if sliceutil.Include(existing, tagID) {
// skip if the scene was already tagged // skip if the scene was already tagged
return nil return nil
} }
@ -376,9 +379,27 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp
partial.Details = models.NewOptionalString(*scraped.Details) partial.Details = models.NewOptionalString(*scraped.Details)
} }
} }
if scraped.URL != nil && (scene.URL != *scraped.URL) { if len(scraped.URLs) > 0 && shouldSetSingleValueField(fieldOptions["url"], false) {
if shouldSetSingleValueField(fieldOptions["url"], scene.URL != "") { // if overwrite, then set over the top
partial.URL = models.NewOptionalString(*scraped.URL) switch getFieldStrategy(fieldOptions["url"]) {
case FieldStrategyOverwrite:
// only overwrite if not equal
if len(sliceutil.Exclude(scene.URLs.List(), scraped.URLs)) != 0 {
partial.URLs = &models.UpdateStrings{
Values: scraped.URLs,
Mode: models.RelationshipUpdateModeSet,
}
}
case FieldStrategyMerge:
// if merge, add if not already present
urls := sliceutil.AppendUniques(scene.URLs.List(), scraped.URLs)
if len(urls) != len(scene.URLs.List()) {
partial.URLs = &models.UpdateStrings{
Values: urls,
Mode: models.RelationshipUpdateModeSet,
}
}
} }
} }
if scraped.Director != nil && (scene.Director != *scraped.Director) { if scraped.Director != nil && (scene.Director != *scraped.Director) {
@ -399,7 +420,7 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp
return partial return partial
} }
func shouldSetSingleValueField(strategy *FieldOptions, hasExistingValue bool) bool { func getFieldStrategy(strategy *FieldOptions) FieldStrategy {
// if unset then default to MERGE // if unset then default to MERGE
fs := FieldStrategyMerge fs := FieldStrategyMerge
@ -407,6 +428,13 @@ func shouldSetSingleValueField(strategy *FieldOptions, hasExistingValue bool) bo
fs = strategy.Strategy fs = strategy.Strategy
} }
return fs
}
func shouldSetSingleValueField(strategy *FieldOptions, hasExistingValue bool) bool {
// if unset then default to MERGE
fs := getFieldStrategy(strategy)
if fs == FieldStrategyIgnore { if fs == FieldStrategyIgnore {
return false return false
} }

View file

@ -11,6 +11,7 @@ import (
"github.com/stashapp/stash/pkg/models/mocks" "github.com/stashapp/stash/pkg/models/mocks"
"github.com/stashapp/stash/pkg/scraper" "github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock" "github.com/stretchr/testify/mock"
) )
@ -108,8 +109,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
} }
mockSceneReaderWriter := &mocks.SceneReaderWriter{} mockSceneReaderWriter := &mocks.SceneReaderWriter{}
mockTagFinderCreator := &mocks.TagReaderWriter{} mockSceneReaderWriter.On("GetURLs", mock.Anything, mock.Anything).Return(nil, nil)
mockSceneReaderWriter.On("UpdatePartial", mock.Anything, mock.MatchedBy(func(id int) bool { mockSceneReaderWriter.On("UpdatePartial", mock.Anything, mock.MatchedBy(func(id int) bool {
return id == errUpdateID return id == errUpdateID
}), mock.Anything).Return(nil, errors.New("update error")) }), mock.Anything).Return(nil, errors.New("update error"))
@ -117,6 +117,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
return id != errUpdateID return id != errUpdateID
}), mock.Anything).Return(nil, nil) }), mock.Anything).Return(nil, nil)
mockTagFinderCreator := &mocks.TagReaderWriter{}
mockTagFinderCreator.On("Find", mock.Anything, skipMultipleTagID).Return(&models.Tag{ mockTagFinderCreator.On("Find", mock.Anything, skipMultipleTagID).Return(&models.Tag{
ID: skipMultipleTagID, ID: skipMultipleTagID,
Name: skipMultipleTagIDStr, Name: skipMultipleTagIDStr,
@ -236,6 +237,7 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
"empty update", "empty update",
args{ args{
&models.Scene{ &models.Scene{
URLs: models.NewRelatedStrings([]string{}),
PerformerIDs: models.NewRelatedIDs([]int{}), PerformerIDs: models.NewRelatedIDs([]int{}),
TagIDs: models.NewRelatedIDs([]int{}), TagIDs: models.NewRelatedIDs([]int{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}), StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
@ -351,33 +353,44 @@ func Test_getScenePartial(t *testing.T) {
Title: originalTitle, Title: originalTitle,
Date: &originalDateObj, Date: &originalDateObj,
Details: originalDetails, Details: originalDetails,
URL: originalURL, URLs: models.NewRelatedStrings([]string{originalURL}),
} }
organisedScene := *originalScene organisedScene := *originalScene
organisedScene.Organized = true organisedScene.Organized = true
emptyScene := &models.Scene{} emptyScene := &models.Scene{
URLs: models.NewRelatedStrings([]string{}),
}
postPartial := models.ScenePartial{ postPartial := models.ScenePartial{
Title: models.NewOptionalString(scrapedTitle), Title: models.NewOptionalString(scrapedTitle),
Date: models.NewOptionalDate(scrapedDateObj), Date: models.NewOptionalDate(scrapedDateObj),
Details: models.NewOptionalString(scrapedDetails), Details: models.NewOptionalString(scrapedDetails),
URL: models.NewOptionalString(scrapedURL), URLs: &models.UpdateStrings{
Values: []string{scrapedURL},
Mode: models.RelationshipUpdateModeSet,
},
}
postPartialMerge := postPartial
postPartialMerge.URLs = &models.UpdateStrings{
Values: []string{scrapedURL},
Mode: models.RelationshipUpdateModeSet,
} }
scrapedScene := &scraper.ScrapedScene{ scrapedScene := &scraper.ScrapedScene{
Title: &scrapedTitle, Title: &scrapedTitle,
Date: &scrapedDate, Date: &scrapedDate,
Details: &scrapedDetails, Details: &scrapedDetails,
URL: &scrapedURL, URLs: []string{scrapedURL},
} }
scrapedUnchangedScene := &scraper.ScrapedScene{ scrapedUnchangedScene := &scraper.ScrapedScene{
Title: &originalTitle, Title: &originalTitle,
Date: &originalDate, Date: &originalDate,
Details: &originalDetails, Details: &originalDetails,
URL: &originalURL, URLs: []string{originalURL},
} }
makeFieldOptions := func(input *FieldOptions) map[string]*FieldOptions { makeFieldOptions := func(input *FieldOptions) map[string]*FieldOptions {
@ -440,7 +453,12 @@ func Test_getScenePartial(t *testing.T) {
mergeAll, mergeAll,
false, false,
}, },
models.ScenePartial{}, models.ScenePartial{
URLs: &models.UpdateStrings{
Values: []string{originalURL, scrapedURL},
Mode: models.RelationshipUpdateModeSet,
},
},
}, },
{ {
"merge (empty values)", "merge (empty values)",
@ -450,7 +468,7 @@ func Test_getScenePartial(t *testing.T) {
mergeAll, mergeAll,
false, false,
}, },
postPartial, postPartialMerge,
}, },
{ {
"unchanged", "unchanged",
@ -487,9 +505,9 @@ func Test_getScenePartial(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
if got := getScenePartial(tt.args.scene, tt.args.scraped, tt.args.fieldOptions, tt.args.setOrganized); !reflect.DeepEqual(got, tt.want) { got := getScenePartial(tt.args.scene, tt.args.scraped, tt.args.fieldOptions, tt.args.setOrganized)
t.Errorf("getScenePartial() = %v, want %v", got, tt.want)
} assert.Equal(t, tt.want, got)
}) })
} }
} }

View file

@ -24,6 +24,7 @@ type SceneReaderUpdater interface {
models.PerformerIDLoader models.PerformerIDLoader
models.TagIDLoader models.TagIDLoader
models.StashIDLoader models.StashIDLoader
models.URLLoader
} }
type TagCreatorFinder interface { type TagCreatorFinder interface {

View file

@ -29,6 +29,7 @@ type GalleryReaderWriter interface {
type SceneReaderWriter interface { type SceneReaderWriter interface {
models.SceneReaderWriter models.SceneReaderWriter
scene.CreatorUpdater scene.CreatorUpdater
models.URLLoader
GetManyFileIDs(ctx context.Context, ids []int) ([][]file.ID, error) GetManyFileIDs(ctx context.Context, ids []int) ([][]file.ID, error)
} }

View file

@ -42,7 +42,9 @@ type Scene struct {
Title string `json:"title,omitempty"` Title string `json:"title,omitempty"`
Code string `json:"code,omitempty"` Code string `json:"code,omitempty"`
Studio string `json:"studio,omitempty"` Studio string `json:"studio,omitempty"`
// deprecated - for import only
URL string `json:"url,omitempty"` URL string `json:"url,omitempty"`
URLs []string `json:"urls,omitempty"`
Date string `json:"date,omitempty"` Date string `json:"date,omitempty"`
Rating int `json:"rating,omitempty"` Rating int `json:"rating,omitempty"`
Organized bool `json:"organized,omitempty"` Organized bool `json:"organized,omitempty"`

View file

@ -624,6 +624,29 @@ func (_m *SceneReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]in
return r0, r1 return r0, r1
} }
// GetURLs provides a mock function with given fields: ctx, relatedID
func (_m *SceneReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) {
ret := _m.Called(ctx, relatedID)
var r0 []string
if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok {
r0 = rf(ctx, relatedID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
r1 = rf(ctx, relatedID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// HasCover provides a mock function with given fields: ctx, sceneID // HasCover provides a mock function with given fields: ctx, sceneID
func (_m *SceneReaderWriter) HasCover(ctx context.Context, sceneID int) (bool, error) { func (_m *SceneReaderWriter) HasCover(ctx context.Context, sceneID int) (bool, error) {
ret := _m.Called(ctx, sceneID) ret := _m.Called(ctx, sceneID)

View file

@ -17,7 +17,6 @@ type Scene struct {
Code string `json:"code"` Code string `json:"code"`
Details string `json:"details"` Details string `json:"details"`
Director string `json:"director"` Director string `json:"director"`
URL string `json:"url"`
Date *Date `json:"date"` Date *Date `json:"date"`
// Rating expressed in 1-100 scale // Rating expressed in 1-100 scale
Rating *int `json:"rating"` Rating *int `json:"rating"`
@ -43,6 +42,7 @@ type Scene struct {
PlayDuration float64 `json:"play_duration"` PlayDuration float64 `json:"play_duration"`
PlayCount int `json:"play_count"` PlayCount int `json:"play_count"`
URLs RelatedStrings `json:"urls"`
GalleryIDs RelatedIDs `json:"gallery_ids"` GalleryIDs RelatedIDs `json:"gallery_ids"`
TagIDs RelatedIDs `json:"tag_ids"` TagIDs RelatedIDs `json:"tag_ids"`
PerformerIDs RelatedIDs `json:"performer_ids"` PerformerIDs RelatedIDs `json:"performer_ids"`
@ -50,6 +50,12 @@ type Scene struct {
StashIDs RelatedStashIDs `json:"stash_ids"` StashIDs RelatedStashIDs `json:"stash_ids"`
} }
func (s *Scene) LoadURLs(ctx context.Context, l URLLoader) error {
return s.URLs.load(func() ([]string, error) {
return l.GetURLs(ctx, s.ID)
})
}
func (s *Scene) LoadFiles(ctx context.Context, l VideoFileLoader) error { func (s *Scene) LoadFiles(ctx context.Context, l VideoFileLoader) error {
return s.Files.load(func() ([]*file.VideoFile, error) { return s.Files.load(func() ([]*file.VideoFile, error) {
return l.GetFiles(ctx, s.ID) return l.GetFiles(ctx, s.ID)
@ -110,6 +116,10 @@ func (s *Scene) LoadStashIDs(ctx context.Context, l StashIDLoader) error {
} }
func (s *Scene) LoadRelationships(ctx context.Context, l SceneReader) error { func (s *Scene) LoadRelationships(ctx context.Context, l SceneReader) error {
if err := s.LoadURLs(ctx, l); err != nil {
return err
}
if err := s.LoadGalleryIDs(ctx, l); err != nil { if err := s.LoadGalleryIDs(ctx, l); err != nil {
return err return err
} }
@ -144,7 +154,6 @@ type ScenePartial struct {
Code OptionalString Code OptionalString
Details OptionalString Details OptionalString
Director OptionalString Director OptionalString
URL OptionalString
Date OptionalDate Date OptionalDate
// Rating expressed in 1-100 scale // Rating expressed in 1-100 scale
Rating OptionalInt Rating OptionalInt
@ -158,6 +167,7 @@ type ScenePartial struct {
PlayCount OptionalInt PlayCount OptionalInt
LastPlayedAt OptionalTime LastPlayedAt OptionalTime
URLs *UpdateStrings
GalleryIDs *UpdateIDs GalleryIDs *UpdateIDs
TagIDs *UpdateIDs TagIDs *UpdateIDs
PerformerIDs *UpdateIDs PerformerIDs *UpdateIDs
@ -193,6 +203,7 @@ type SceneUpdateInput struct {
Rating100 *int `json:"rating100"` Rating100 *int `json:"rating100"`
OCounter *int `json:"o_counter"` OCounter *int `json:"o_counter"`
Organized *bool `json:"organized"` Organized *bool `json:"organized"`
Urls []string `json:"urls"`
StudioID *string `json:"studio_id"` StudioID *string `json:"studio_id"`
GalleryIds []string `json:"gallery_ids"` GalleryIds []string `json:"gallery_ids"`
PerformerIds []string `json:"performer_ids"` PerformerIds []string `json:"performer_ids"`
@ -227,7 +238,7 @@ func (s ScenePartial) UpdateInput(id int) SceneUpdateInput {
Code: s.Code.Ptr(), Code: s.Code.Ptr(),
Details: s.Details.Ptr(), Details: s.Details.Ptr(),
Director: s.Director.Ptr(), Director: s.Director.Ptr(),
URL: s.URL.Ptr(), Urls: s.URLs.Strings(),
Date: dateStr, Date: dateStr,
Rating100: s.Rating.Ptr(), Rating100: s.Rating.Ptr(),
Organized: s.Organized.Ptr(), Organized: s.Organized.Ptr(),

View file

@ -41,7 +41,10 @@ func TestScenePartial_UpdateInput(t *testing.T) {
Code: NewOptionalString(code), Code: NewOptionalString(code),
Details: NewOptionalString(details), Details: NewOptionalString(details),
Director: NewOptionalString(director), Director: NewOptionalString(director),
URL: NewOptionalString(url), URLs: &UpdateStrings{
Values: []string{url},
Mode: RelationshipUpdateModeSet,
},
Date: NewOptionalDate(dateObj), Date: NewOptionalDate(dateObj),
Rating: NewOptionalInt(rating100), Rating: NewOptionalInt(rating100),
Organized: NewOptionalBool(organized), Organized: NewOptionalBool(organized),
@ -53,7 +56,7 @@ func TestScenePartial_UpdateInput(t *testing.T) {
Code: &code, Code: &code,
Details: &details, Details: &details,
Director: &director, Director: &director,
URL: &url, Urls: []string{url},
Date: &date, Date: &date,
Rating: &ratingLegacy, Rating: &ratingLegacy,
Rating100: &rating100, Rating100: &rating100,

View file

@ -42,6 +42,10 @@ type AliasLoader interface {
GetAliases(ctx context.Context, relatedID int) ([]string, error) GetAliases(ctx context.Context, relatedID int) ([]string, error)
} }
type URLLoader interface {
GetURLs(ctx context.Context, relatedID int) ([]string, error)
}
// RelatedIDs represents a list of related IDs. // RelatedIDs represents a list of related IDs.
// TODO - this can be made generic // TODO - this can be made generic
type RelatedIDs struct { type RelatedIDs struct {

View file

@ -159,6 +159,7 @@ type SceneReader interface {
FindByGalleryID(ctx context.Context, performerID int) ([]*Scene, error) FindByGalleryID(ctx context.Context, performerID int) ([]*Scene, error)
FindDuplicates(ctx context.Context, distance int, durationDiff float64) ([][]*Scene, error) FindDuplicates(ctx context.Context, distance int, durationDiff float64) ([][]*Scene, error)
URLLoader
GalleryIDLoader GalleryIDLoader
PerformerIDLoader PerformerIDLoader
TagIDLoader TagIDLoader

View file

@ -110,3 +110,11 @@ type UpdateStrings struct {
Values []string `json:"values"` Values []string `json:"values"`
Mode RelationshipUpdateMode `json:"mode"` Mode RelationshipUpdateMode `json:"mode"`
} }
func (u *UpdateStrings) Strings() []string {
if u == nil {
return nil
}
return u.Values
}

View file

@ -41,7 +41,7 @@ func ToBasicJSON(ctx context.Context, reader CoverGetter, scene *models.Scene) (
newSceneJSON := jsonschema.Scene{ newSceneJSON := jsonschema.Scene{
Title: scene.Title, Title: scene.Title,
Code: scene.Code, Code: scene.Code,
URL: scene.URL, URLs: scene.URLs.List(),
Details: scene.Details, Details: scene.Details,
Director: scene.Director, Director: scene.Director,
CreatedAt: json.JSONTime{Time: scene.CreatedAt}, CreatedAt: json.JSONTime{Time: scene.CreatedAt},
@ -86,53 +86,6 @@ func ToBasicJSON(ctx context.Context, reader CoverGetter, scene *models.Scene) (
return &newSceneJSON, nil return &newSceneJSON, nil
} }
// func getSceneFileJSON(scene *models.Scene) *jsonschema.SceneFile {
// ret := &jsonschema.SceneFile{}
// TODO
// if scene.FileModTime != nil {
// ret.ModTime = json.JSONTime{Time: *scene.FileModTime}
// }
// if scene.Size != nil {
// ret.Size = *scene.Size
// }
// if scene.Duration != nil {
// ret.Duration = getDecimalString(*scene.Duration)
// }
// if scene.VideoCodec != nil {
// ret.VideoCodec = *scene.VideoCodec
// }
// if scene.AudioCodec != nil {
// ret.AudioCodec = *scene.AudioCodec
// }
// if scene.Format != nil {
// ret.Format = *scene.Format
// }
// if scene.Width != nil {
// ret.Width = *scene.Width
// }
// if scene.Height != nil {
// ret.Height = *scene.Height
// }
// if scene.Framerate != nil {
// ret.Framerate = getDecimalString(*scene.Framerate)
// }
// if scene.Bitrate != nil {
// ret.Bitrate = int(*scene.Bitrate)
// }
// return ret
// }
// GetStudioName returns the name of the provided scene's studio. It returns an // GetStudioName returns the name of the provided scene's studio. It returns an
// empty string if there is no studio assigned to the scene. // empty string if there is no studio assigned to the scene.
func GetStudioName(ctx context.Context, reader studio.Finder, scene *models.Scene) (string, error) { func GetStudioName(ctx context.Context, reader studio.Finder, scene *models.Scene) (string, error) {

View file

@ -92,7 +92,7 @@ func createFullScene(id int) models.Scene {
OCounter: ocounter, OCounter: ocounter,
Rating: &rating, Rating: &rating,
Organized: organized, Organized: organized,
URL: url, URLs: models.NewRelatedStrings([]string{url}),
Files: models.NewRelatedVideoFiles([]*file.VideoFile{ Files: models.NewRelatedVideoFiles([]*file.VideoFile{
{ {
BaseFile: &file.BaseFile{ BaseFile: &file.BaseFile{
@ -118,6 +118,7 @@ func createEmptyScene(id int) models.Scene {
}, },
}, },
}), }),
URLs: models.NewRelatedStrings([]string{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}), StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
CreatedAt: createTime, CreatedAt: createTime,
UpdatedAt: updateTime, UpdatedAt: updateTime,
@ -133,7 +134,7 @@ func createFullJSONScene(image string) *jsonschema.Scene {
OCounter: ocounter, OCounter: ocounter,
Rating: rating, Rating: rating,
Organized: organized, Organized: organized,
URL: url, URLs: []string{url},
CreatedAt: json.JSONTime{ CreatedAt: json.JSONTime{
Time: createTime, Time: createTime,
}, },
@ -149,6 +150,7 @@ func createFullJSONScene(image string) *jsonschema.Scene {
func createEmptyJSONScene() *jsonschema.Scene { func createEmptyJSONScene() *jsonschema.Scene {
return &jsonschema.Scene{ return &jsonschema.Scene{
URLs: []string{},
Files: []string{path}, Files: []string{path},
CreatedAt: json.JSONTime{ CreatedAt: json.JSONTime{
Time: createTime, Time: createTime,

View file

@ -80,12 +80,10 @@ func (i *Importer) PreImport(ctx context.Context) error {
func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene { func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
newScene := models.Scene{ newScene := models.Scene{
// Path: i.Path,
Title: sceneJSON.Title, Title: sceneJSON.Title,
Code: sceneJSON.Code, Code: sceneJSON.Code,
Details: sceneJSON.Details, Details: sceneJSON.Details,
Director: sceneJSON.Director, Director: sceneJSON.Director,
URL: sceneJSON.URL,
PerformerIDs: models.NewRelatedIDs([]int{}), PerformerIDs: models.NewRelatedIDs([]int{}),
TagIDs: models.NewRelatedIDs([]int{}), TagIDs: models.NewRelatedIDs([]int{}),
GalleryIDs: models.NewRelatedIDs([]int{}), GalleryIDs: models.NewRelatedIDs([]int{}),
@ -93,6 +91,12 @@ func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
StashIDs: models.NewRelatedStashIDs(sceneJSON.StashIDs), StashIDs: models.NewRelatedStashIDs(sceneJSON.StashIDs),
} }
if len(sceneJSON.URLs) > 0 {
newScene.URLs = models.NewRelatedStrings(sceneJSON.URLs)
} else if sceneJSON.URL != "" {
newScene.URLs = models.NewRelatedStrings([]string{sceneJSON.URL})
}
if sceneJSON.Date != "" { if sceneJSON.Date != "" {
d := models.NewDate(sceneJSON.Date) d := models.NewDate(sceneJSON.Date)
newScene.Date = &d newScene.Date = &d

View file

@ -52,6 +52,11 @@ func isCDPPathWS(c GlobalConfig) bool {
return strings.HasPrefix(c.GetScraperCDPPath(), "ws://") return strings.HasPrefix(c.GetScraperCDPPath(), "ws://")
} }
type SceneFinder interface {
scene.IDFinder
models.URLLoader
}
type PerformerFinder interface { type PerformerFinder interface {
match.PerformerAutoTagQueryer match.PerformerAutoTagQueryer
match.PerformerFinder match.PerformerFinder
@ -73,7 +78,7 @@ type GalleryFinder interface {
} }
type Repository struct { type Repository struct {
SceneFinder scene.IDFinder SceneFinder SceneFinder
GalleryFinder GalleryFinder GalleryFinder GalleryFinder
TagFinder TagFinder TagFinder TagFinder
PerformerFinder PerformerFinder PerformerFinder PerformerFinder
@ -240,7 +245,19 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
return nil, fmt.Errorf("%w: cannot use scraper %s to scrape by name", ErrNotSupported, id) return nil, fmt.Errorf("%w: cannot use scraper %s to scrape by name", ErrNotSupported, id)
} }
return ns.viaName(ctx, c.client, query, ty) content, err := ns.viaName(ctx, c.client, query, ty)
if err != nil {
return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
}
for i, cc := range content {
content[i], err = c.postScrape(ctx, cc)
if err != nil {
return nil, fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
}
}
return content, nil
} }
// ScrapeFragment uses the given fragment input to scrape // ScrapeFragment uses the given fragment input to scrape
@ -361,7 +378,7 @@ func (c Cache) getScene(ctx context.Context, sceneID int) (*models.Scene, error)
return fmt.Errorf("scene with id %d not found", sceneID) return fmt.Errorf("scene with id %d not found", sceneID)
} }
return nil return ret.LoadURLs(ctx, c.repository.SceneFinder)
}); err != nil { }); err != nil {
return nil, err return nil, err
} }

View file

@ -106,6 +106,14 @@ func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPer
} }
func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (ScrapedContent, error) { func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (ScrapedContent, error) {
// set the URL/URLs field
if scene.URL == nil && len(scene.URLs) > 0 {
scene.URL = &scene.URLs[0]
}
if scene.URL != nil && len(scene.URLs) == 0 {
scene.URLs = []string{*scene.URL}
}
if err := txn.WithReadTxn(ctx, c.txnManager, func(ctx context.Context) error { if err := txn.WithReadTxn(ctx, c.txnManager, func(ctx context.Context) error {
pqb := c.repository.PerformerFinder pqb := c.repository.PerformerFinder
mqb := c.repository.MovieFinder mqb := c.repository.MovieFinder

View file

@ -20,8 +20,8 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
if scene.Title != "" { if scene.Title != "" {
ret["title"] = scene.Title ret["title"] = scene.Title
} }
if scene.URL != "" { if len(scene.URLs.List()) > 0 {
ret["url"] = scene.URL ret["url"] = scene.URLs.List()[0]
} }
return ret return ret
} }
@ -37,7 +37,11 @@ func queryURLParametersFromScrapedScene(scene ScrapedSceneInput) queryURLParamet
setField("title", scene.Title) setField("title", scene.Title)
setField("code", scene.Code) setField("code", scene.Code)
if len(scene.URLs) > 0 {
setField("url", &scene.URLs[0])
} else {
setField("url", scene.URL) setField("url", scene.URL)
}
setField("date", scene.Date) setField("date", scene.Date)
setField("details", scene.Details) setField("details", scene.Details)
setField("director", scene.Director) setField("director", scene.Director)

View file

@ -10,6 +10,7 @@ type ScrapedScene struct {
Details *string `json:"details"` Details *string `json:"details"`
Director *string `json:"director"` Director *string `json:"director"`
URL *string `json:"url"` URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"` Date *string `json:"date"`
// This should be a base64 encoded data URL // This should be a base64 encoded data URL
Image *string `json:"image"` Image *string `json:"image"`
@ -31,6 +32,7 @@ type ScrapedSceneInput struct {
Details *string `json:"details"` Details *string `json:"details"`
Director *string `json:"director"` Director *string `json:"director"`
URL *string `json:"url"` URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"` Date *string `json:"date"`
RemoteSiteID *string `json:"remote_site_id"` RemoteSiteID *string `json:"remote_site_id"`
} }

View file

@ -328,7 +328,7 @@ func sceneToUpdateInput(scene *models.Scene) models.SceneUpdateInput {
ID: strconv.Itoa(scene.ID), ID: strconv.Itoa(scene.ID),
Title: &title, Title: &title,
Details: &scene.Details, Details: &scene.Details,
URL: &scene.URL, Urls: scene.URLs.List(),
Date: dateToStringPtr(scene.Date), Date: dateToStringPtr(scene.Date),
} }
} }

View file

@ -684,6 +684,7 @@ func getFingerprints(scene *graphql.SceneFragment) []*models.StashBoxFingerprint
func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.SceneFragment) (*scraper.ScrapedScene, error) { func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.SceneFragment) (*scraper.ScrapedScene, error) {
stashID := s.ID stashID := s.ID
ss := &scraper.ScrapedScene{ ss := &scraper.ScrapedScene{
Title: s.Title, Title: s.Title,
Code: s.Code, Code: s.Code,
@ -698,6 +699,14 @@ func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.Scen
// stash_id // stash_id
} }
for _, u := range s.Urls {
ss.URLs = append(ss.URLs, u.URL)
}
if len(ss.URLs) > 0 {
ss.URL = &ss.URLs[0]
}
if len(s.Images) > 0 { if len(s.Images) > 0 {
// TODO - #454 code sorts images by aspect ratio according to a wanted // TODO - #454 code sorts images by aspect ratio according to a wanted
// orientation. I'm just grabbing the first for now // orientation. I'm just grabbing the first for now
@ -823,8 +832,9 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
if scene.Director != "" { if scene.Director != "" {
draft.Director = &scene.Director draft.Director = &scene.Director
} }
if scene.URL != "" && len(strings.TrimSpace(scene.URL)) > 0 { // TODO - draft does not accept multiple URLs. Use single URL for now.
url := strings.TrimSpace(scene.URL) if len(scene.URLs.List()) > 0 {
url := strings.TrimSpace(scene.URLs.List()[0])
draft.URL = &url draft.URL = &url
} }
if scene.Date != nil { if scene.Date != nil {

View file

@ -2,6 +2,53 @@ package sliceutil
import "reflect" import "reflect"
// Exclude removes all instances of any value in toExclude from the vs
// slice. It returns the new or unchanged slice.
func Exclude[T comparable](vs []T, toExclude []T) []T {
var ret []T
for _, v := range vs {
if !Include(toExclude, v) {
ret = append(ret, v)
}
}
return ret
}
func Index[T comparable](vs []T, t T) int {
for i, v := range vs {
if v == t {
return i
}
}
return -1
}
func Include[T comparable](vs []T, t T) bool {
return Index(vs, t) >= 0
}
// IntAppendUnique appends toAdd to the vs int slice if toAdd does not already
// exist in the slice. It returns the new or unchanged int slice.
func AppendUnique[T comparable](vs []T, toAdd T) []T {
if Include(vs, toAdd) {
return vs
}
return append(vs, toAdd)
}
// IntAppendUniques appends a slice of values to the vs slice. It only
// appends values that do not already exist in the slice. It returns the new or
// unchanged slice.
func AppendUniques[T comparable](vs []T, toAdd []T) []T {
for _, v := range toAdd {
vs = AppendUnique(vs, v)
}
return vs
}
// SliceSame returns true if the two provided lists have the same elements, // SliceSame returns true if the two provided lists have the same elements,
// regardless of order. Panics if either parameter is not a slice. // regardless of order. Panics if either parameter is not a slice.
func SliceSame(a, b interface{}) bool { func SliceSame(a, b interface{}) bool {

View file

@ -230,7 +230,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
table.Col(idColumn), table.Col(idColumn),
table.Col("title"), table.Col("title"),
table.Col("details"), table.Col("details"),
table.Col("url"),
table.Col("code"), table.Col("code"),
table.Col("director"), table.Col("director"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000) ).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
@ -243,7 +242,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
id int id int
title sql.NullString title sql.NullString
details sql.NullString details sql.NullString
url sql.NullString
code sql.NullString code sql.NullString
director sql.NullString director sql.NullString
) )
@ -252,7 +250,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
&id, &id,
&title, &title,
&details, &details,
&url,
&code, &code,
&director, &director,
); err != nil { ); err != nil {
@ -264,7 +261,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
// if title set set new title // if title set set new title
db.obfuscateNullString(set, "title", title) db.obfuscateNullString(set, "title", title)
db.obfuscateNullString(set, "details", details) db.obfuscateNullString(set, "details", details)
db.obfuscateNullString(set, "url", url)
if len(set) > 0 { if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id)) stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id))
@ -301,6 +297,10 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
} }
} }
if err := db.anonymiseURLs(ctx, goqu.T(scenesURLsTable), "scene_id"); err != nil {
return err
}
return nil return nil
} }
@ -704,6 +704,68 @@ func (db *Anonymiser) anonymiseAliases(ctx context.Context, table exp.Identifier
return nil return nil
} }
func (db *Anonymiser) anonymiseURLs(ctx context.Context, table exp.IdentifierExpression, idColumn string) error {
lastID := 0
lastURL := ""
total := 0
const logEvery = 10000
for gotSome := true; gotSome; {
if err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("url"),
).Where(goqu.L("(" + idColumn + ", url)").Gt(goqu.L("(?, ?)", lastID, lastURL))).Limit(1000)
gotSome = false
const single = false
return queryFunc(ctx, query, single, func(rows *sqlx.Rows) error {
var (
id int
url sql.NullString
)
if err := rows.Scan(
&id,
&url,
); err != nil {
return err
}
set := goqu.Record{}
db.obfuscateNullString(set, "url", url)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(
table.Col(idColumn).Eq(id),
table.Col("url").Eq(url),
)
if _, err := exec(ctx, stmt); err != nil {
return fmt.Errorf("anonymising %s: %w", table.GetTable(), err)
}
}
lastID = id
lastURL = url.String
gotSome = true
total++
if total%logEvery == 0 {
logger.Infof("Anonymised %d %s URLs", total, table.GetTable())
}
return nil
})
}); err != nil {
return err
}
}
return nil
}
func (db *Anonymiser) anonymiseTags(ctx context.Context) error { func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
logger.Infof("Anonymising tags") logger.Infof("Anonymising tags")
table := tagTableMgr.table table := tagTableMgr.table

View file

@ -32,7 +32,7 @@ const (
dbConnTimeout = 30 dbConnTimeout = 30
) )
var appSchemaVersion uint = 46 var appSchemaVersion uint = 47
//go:embed migrations/*.sql //go:embed migrations/*.sql
var migrationsBox embed.FS var migrationsBox embed.FS

View file

@ -0,0 +1,94 @@
PRAGMA foreign_keys=OFF;
CREATE TABLE `scene_urls` (
`scene_id` integer NOT NULL,
`position` integer NOT NULL,
`url` varchar(255) NOT NULL,
foreign key(`scene_id`) references `scenes`(`id`) on delete CASCADE,
PRIMARY KEY(`scene_id`, `position`, `url`)
);
CREATE INDEX `scene_urls_url` on `scene_urls` (`url`);
-- drop url
CREATE TABLE "scenes_new" (
`id` integer not null primary key autoincrement,
`title` varchar(255),
`details` text,
`date` date,
`rating` tinyint,
`studio_id` integer,
`o_counter` tinyint not null default 0,
`organized` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
`code` text,
`director` text,
`resume_time` float not null default 0,
`last_played_at` datetime default null,
`play_count` tinyint not null default 0,
`play_duration` float not null default 0,
`cover_blob` varchar(255) REFERENCES `blobs`(`checksum`),
foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL
);
INSERT INTO `scenes_new`
(
`id`,
`title`,
`details`,
`date`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`code`,
`director`,
`resume_time`,
`last_played_at`,
`play_count`,
`play_duration`,
`cover_blob`
)
SELECT
`id`,
`title`,
`details`,
`date`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`code`,
`director`,
`resume_time`,
`last_played_at`,
`play_count`,
`play_duration`,
`cover_blob`
FROM `scenes`;
INSERT INTO `scene_urls`
(
`scene_id`,
`position`,
`url`
)
SELECT
`id`,
'0',
`url`
FROM `scenes`
WHERE `scenes`.`url` IS NOT NULL AND `scenes`.`url` != '';
DROP INDEX `index_scenes_on_studio_id`;
DROP TABLE `scenes`;
ALTER TABLE `scenes_new` rename to `scenes`;
CREATE INDEX `index_scenes_on_studio_id` on `scenes` (`studio_id`);
PRAGMA foreign_keys=ON;

View file

@ -31,6 +31,8 @@ const (
scenesTagsTable = "scenes_tags" scenesTagsTable = "scenes_tags"
scenesGalleriesTable = "scenes_galleries" scenesGalleriesTable = "scenes_galleries"
moviesScenesTable = "movies_scenes" moviesScenesTable = "movies_scenes"
scenesURLsTable = "scene_urls"
sceneURLColumn = "url"
sceneCoverBlobColumn = "cover_blob" sceneCoverBlobColumn = "cover_blob"
) )
@ -76,7 +78,6 @@ type sceneRow struct {
Code zero.String `db:"code"` Code zero.String `db:"code"`
Details zero.String `db:"details"` Details zero.String `db:"details"`
Director zero.String `db:"director"` Director zero.String `db:"director"`
URL zero.String `db:"url"`
Date NullDate `db:"date"` Date NullDate `db:"date"`
// expressed as 1-100 // expressed as 1-100
Rating null.Int `db:"rating"` Rating null.Int `db:"rating"`
@ -100,7 +101,6 @@ func (r *sceneRow) fromScene(o models.Scene) {
r.Code = zero.StringFrom(o.Code) r.Code = zero.StringFrom(o.Code)
r.Details = zero.StringFrom(o.Details) r.Details = zero.StringFrom(o.Details)
r.Director = zero.StringFrom(o.Director) r.Director = zero.StringFrom(o.Director)
r.URL = zero.StringFrom(o.URL)
r.Date = NullDateFromDatePtr(o.Date) r.Date = NullDateFromDatePtr(o.Date)
r.Rating = intFromPtr(o.Rating) r.Rating = intFromPtr(o.Rating)
r.Organized = o.Organized r.Organized = o.Organized
@ -130,7 +130,6 @@ func (r *sceneQueryRow) resolve() *models.Scene {
Code: r.Code.String, Code: r.Code.String,
Details: r.Details.String, Details: r.Details.String,
Director: r.Director.String, Director: r.Director.String,
URL: r.URL.String,
Date: r.Date.DatePtr(), Date: r.Date.DatePtr(),
Rating: nullIntPtr(r.Rating), Rating: nullIntPtr(r.Rating),
Organized: r.Organized, Organized: r.Organized,
@ -166,7 +165,6 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) {
r.setNullString("code", o.Code) r.setNullString("code", o.Code)
r.setNullString("details", o.Details) r.setNullString("details", o.Details)
r.setNullString("director", o.Director) r.setNullString("director", o.Director)
r.setNullString("url", o.URL)
r.setNullDate("date", o.Date) r.setNullDate("date", o.Date)
r.setNullInt("rating", o.Rating) r.setNullInt("rating", o.Rating)
r.setBool("organized", o.Organized) r.setBool("organized", o.Organized)
@ -268,6 +266,13 @@ func (qb *SceneStore) Create(ctx context.Context, newObject *models.Scene, fileI
} }
} }
if newObject.URLs.Loaded() {
const startPos = 0
if err := scenesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
return err
}
}
if newObject.PerformerIDs.Loaded() { if newObject.PerformerIDs.Loaded() {
if err := scenesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil { if err := scenesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil {
return err return err
@ -322,6 +327,11 @@ func (qb *SceneStore) UpdatePartial(ctx context.Context, id int, partial models.
} }
} }
if partial.URLs != nil {
if err := scenesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
return nil, err
}
}
if partial.PerformerIDs != nil { if partial.PerformerIDs != nil {
if err := scenesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil { if err := scenesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil {
return nil, err return nil, err
@ -364,6 +374,12 @@ func (qb *SceneStore) Update(ctx context.Context, updatedObject *models.Scene) e
return err return err
} }
if updatedObject.URLs.Loaded() {
if err := scenesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
return err
}
}
if updatedObject.PerformerIDs.Loaded() { if updatedObject.PerformerIDs.Loaded() {
if err := scenesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil { if err := scenesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil {
return err return err
@ -974,7 +990,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers)) query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing)) query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.URL, "scenes.url")) query.handleCriterion(ctx, sceneURLsCriterionHandler(sceneFilter.URL))
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) { query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
if sceneFilter.StashID != nil { if sceneFilter.StashID != nil {
@ -1308,6 +1324,18 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion
} }
} }
func sceneURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
joinTable: scenesURLsTable,
stringColumn: sceneURLColumn,
addJoinTable: func(f *filterBuilder) {
scenesURLsTableMgr.join(f, "", "scenes.id")
},
}
return h.handler(url)
}
func (qb *SceneStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder { func (qb *SceneStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder {
return multiCriterionHandlerBuilder{ return multiCriterionHandlerBuilder{
primaryTable: sceneTable, primaryTable: sceneTable,
@ -1637,6 +1665,10 @@ func (qb *SceneStore) IncrementWatchCount(ctx context.Context, id int) (int, err
return qb.getPlayCount(ctx, id) return qb.getPlayCount(ctx, id)
} }
func (qb *SceneStore) GetURLs(ctx context.Context, sceneID int) ([]string, error) {
return scenesURLsTableMgr.get(ctx, sceneID)
}
func (qb *SceneStore) GetCover(ctx context.Context, sceneID int) ([]byte, error) { func (qb *SceneStore) GetCover(ctx context.Context, sceneID int) ([]byte, error) {
return qb.GetImage(ctx, sceneID, sceneCoverBlobColumn) return qb.GetImage(ctx, sceneID, sceneCoverBlobColumn)
} }

View file

@ -21,6 +21,12 @@ import (
) )
func loadSceneRelationships(ctx context.Context, expected models.Scene, actual *models.Scene) error { func loadSceneRelationships(ctx context.Context, expected models.Scene, actual *models.Scene) error {
if expected.URLs.Loaded() {
if err := actual.LoadURLs(ctx, db.Scene); err != nil {
return err
}
}
if expected.GalleryIDs.Loaded() { if expected.GalleryIDs.Loaded() {
if err := actual.LoadGalleryIDs(ctx, db.Scene); err != nil { if err := actual.LoadGalleryIDs(ctx, db.Scene); err != nil {
return err return err
@ -108,7 +114,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
Code: code, Code: code,
Details: details, Details: details,
Director: director, Director: director,
URL: url, URLs: models.NewRelatedStrings([]string{url}),
Date: &date, Date: &date,
Rating: &rating, Rating: &rating,
Organized: true, Organized: true,
@ -153,7 +159,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
Code: code, Code: code,
Details: details, Details: details,
Director: director, Director: director,
URL: url, URLs: models.NewRelatedStrings([]string{url}),
Date: &date, Date: &date,
Rating: &rating, Rating: &rating,
Organized: true, Organized: true,
@ -346,7 +352,7 @@ func Test_sceneQueryBuilder_Update(t *testing.T) {
Code: code, Code: code,
Details: details, Details: details,
Director: director, Director: director,
URL: url, URLs: models.NewRelatedStrings([]string{url}),
Date: &date, Date: &date,
Rating: &rating, Rating: &rating,
Organized: true, Organized: true,
@ -513,7 +519,7 @@ func clearScenePartial() models.ScenePartial {
Code: models.OptionalString{Set: true, Null: true}, Code: models.OptionalString{Set: true, Null: true},
Details: models.OptionalString{Set: true, Null: true}, Details: models.OptionalString{Set: true, Null: true},
Director: models.OptionalString{Set: true, Null: true}, Director: models.OptionalString{Set: true, Null: true},
URL: models.OptionalString{Set: true, Null: true}, URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
Date: models.OptionalDate{Set: true, Null: true}, Date: models.OptionalDate{Set: true, Null: true},
Rating: models.OptionalInt{Set: true, Null: true}, Rating: models.OptionalInt{Set: true, Null: true},
StudioID: models.OptionalInt{Set: true, Null: true}, StudioID: models.OptionalInt{Set: true, Null: true},
@ -564,7 +570,10 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
Code: models.NewOptionalString(code), Code: models.NewOptionalString(code),
Details: models.NewOptionalString(details), Details: models.NewOptionalString(details),
Director: models.NewOptionalString(director), Director: models.NewOptionalString(director),
URL: models.NewOptionalString(url), URLs: &models.UpdateStrings{
Values: []string{url},
Mode: models.RelationshipUpdateModeSet,
},
Date: models.NewOptionalDate(date), Date: models.NewOptionalDate(date),
Rating: models.NewOptionalInt(rating), Rating: models.NewOptionalInt(rating),
Organized: models.NewOptionalBool(true), Organized: models.NewOptionalBool(true),
@ -624,7 +633,7 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
Code: code, Code: code,
Details: details, Details: details,
Director: director, Director: director,
URL: url, URLs: models.NewRelatedStrings([]string{url}),
Date: &date, Date: &date,
Rating: &rating, Rating: &rating,
Organized: true, Organized: true,
@ -2400,7 +2409,14 @@ func TestSceneQueryURL(t *testing.T) {
verifyFn := func(s *models.Scene) { verifyFn := func(s *models.Scene) {
t.Helper() t.Helper()
verifyString(t, s.URL, urlCriterion)
urls := s.URLs.List()
var url string
if len(urls) > 0 {
url = urls[0]
}
verifyString(t, url, urlCriterion)
} }
verifySceneQuery(t, filter, verifyFn) verifySceneQuery(t, filter, verifyFn)
@ -2576,6 +2592,12 @@ func verifySceneQuery(t *testing.T, filter models.SceneFilterType, verifyFn func
scenes := queryScene(ctx, t, sqb, &filter, nil) scenes := queryScene(ctx, t, sqb, &filter, nil)
for _, scene := range scenes {
if err := scene.LoadRelationships(ctx, sqb); err != nil {
t.Errorf("Error loading scene relationships: %v", err)
}
}
// assume it should find at least one // assume it should find at least one
assert.Greater(t, len(scenes), 0) assert.Greater(t, len(scenes), 0)

View file

@ -1067,7 +1067,9 @@ func makeScene(i int) *models.Scene {
return &models.Scene{ return &models.Scene{
Title: title, Title: title,
Details: details, Details: details,
URL: getSceneEmptyString(i, urlField), URLs: models.NewRelatedStrings([]string{
getSceneEmptyString(i, urlField),
}),
Rating: getIntPtr(rating), Rating: getIntPtr(rating),
OCounter: getOCounter(i), OCounter: getOCounter(i),
Date: getObjectDate(i), Date: getObjectDate(i),

View file

@ -14,6 +14,7 @@ import (
"github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/sliceutil/stringslice"
) )
@ -472,6 +473,113 @@ func (t *stringTable) modifyJoins(ctx context.Context, id int, v []string, mode
return nil return nil
} }
type orderedValueTable[T comparable] struct {
table
valueColumn exp.IdentifierExpression
}
func (t *orderedValueTable[T]) positionColumn() exp.IdentifierExpression {
const positionColumn = "position"
return t.table.table.Col(positionColumn)
}
func (t *orderedValueTable[T]) get(ctx context.Context, id int) ([]T, error) {
q := dialect.Select(t.valueColumn).From(t.table.table).Where(t.idColumn.Eq(id)).Order(t.positionColumn().Asc())
const single = false
var ret []T
if err := queryFunc(ctx, q, single, func(rows *sqlx.Rows) error {
var v T
if err := rows.Scan(&v); err != nil {
return err
}
ret = append(ret, v)
return nil
}); err != nil {
return nil, fmt.Errorf("getting stash ids from %s: %w", t.table.table.GetTable(), err)
}
return ret, nil
}
func (t *orderedValueTable[T]) insertJoin(ctx context.Context, id int, position int, v T) (sql.Result, error) {
q := dialect.Insert(t.table.table).Cols(t.idColumn.GetCol(), t.positionColumn().GetCol(), t.valueColumn.GetCol()).Vals(
goqu.Vals{id, position, v},
)
ret, err := exec(ctx, q)
if err != nil {
return nil, fmt.Errorf("inserting into %s: %w", t.table.table.GetTable(), err)
}
return ret, nil
}
func (t *orderedValueTable[T]) insertJoins(ctx context.Context, id int, startPos int, v []T) error {
for i, fk := range v {
if _, err := t.insertJoin(ctx, id, i+startPos, fk); err != nil {
return err
}
}
return nil
}
func (t *orderedValueTable[T]) replaceJoins(ctx context.Context, id int, v []T) error {
if err := t.destroy(ctx, []int{id}); err != nil {
return err
}
const startPos = 0
return t.insertJoins(ctx, id, startPos, v)
}
func (t *orderedValueTable[T]) addJoins(ctx context.Context, id int, v []T) error {
// get existing foreign keys
existing, err := t.get(ctx, id)
if err != nil {
return err
}
// only add values that are not already present
filtered := sliceutil.Exclude(v, existing)
if len(filtered) == 0 {
return nil
}
startPos := len(existing)
return t.insertJoins(ctx, id, startPos, filtered)
}
func (t *orderedValueTable[T]) destroyJoins(ctx context.Context, id int, v []T) error {
existing, err := t.get(ctx, id)
if err != nil {
return fmt.Errorf("getting existing %s: %w", t.table.table.GetTable(), err)
}
newValue := sliceutil.Exclude(existing, v)
if len(newValue) == len(existing) {
return nil
}
return t.replaceJoins(ctx, id, newValue)
}
func (t *orderedValueTable[T]) modifyJoins(ctx context.Context, id int, v []T, mode models.RelationshipUpdateMode) error {
switch mode {
case models.RelationshipUpdateModeSet:
return t.replaceJoins(ctx, id, v)
case models.RelationshipUpdateModeAdd:
return t.addJoins(ctx, id, v)
case models.RelationshipUpdateModeRemove:
return t.destroyJoins(ctx, id, v)
}
return nil
}
type scenesMoviesTable struct { type scenesMoviesTable struct {
table table
} }

View file

@ -24,6 +24,7 @@ var (
scenesPerformersJoinTable = goqu.T(performersScenesTable) scenesPerformersJoinTable = goqu.T(performersScenesTable)
scenesStashIDsJoinTable = goqu.T("scene_stash_ids") scenesStashIDsJoinTable = goqu.T("scene_stash_ids")
scenesMoviesJoinTable = goqu.T(moviesScenesTable) scenesMoviesJoinTable = goqu.T(moviesScenesTable)
scenesURLsJoinTable = goqu.T(scenesURLsTable)
performersAliasesJoinTable = goqu.T(performersAliasesTable) performersAliasesJoinTable = goqu.T(performersAliasesTable)
performersTagsJoinTable = goqu.T(performersTagsTable) performersTagsJoinTable = goqu.T(performersTagsTable)
@ -160,6 +161,14 @@ var (
idColumn: scenesMoviesJoinTable.Col(sceneIDColumn), idColumn: scenesMoviesJoinTable.Col(sceneIDColumn),
}, },
} }
scenesURLsTableMgr = &orderedValueTable[string]{
table: table{
table: scenesURLsJoinTable,
idColumn: scenesURLsJoinTable.Col(sceneIDColumn),
},
valueColumn: scenesURLsJoinTable.Col(sceneURLColumn),
}
) )
var ( var (

View file

@ -29,7 +29,7 @@ import {
import { Icon } from "src/components/Shared/Icon"; import { Icon } from "src/components/Shared/Icon";
import { LoadingIndicator } from "src/components/Shared/LoadingIndicator"; import { LoadingIndicator } from "src/components/Shared/LoadingIndicator";
import { ImageInput } from "src/components/Shared/ImageInput"; import { ImageInput } from "src/components/Shared/ImageInput";
import { URLField } from "src/components/Shared/URLField"; import { URLListInput } from "src/components/Shared/URLField";
import { useToast } from "src/hooks/Toast"; import { useToast } from "src/hooks/Toast";
import ImageUtils from "src/utils/image"; import ImageUtils from "src/utils/image";
import FormUtils from "src/utils/form"; import FormUtils from "src/utils/form";
@ -106,7 +106,25 @@ export const SceneEditPanel: React.FC<IProps> = ({
const schema = yup.object({ const schema = yup.object({
title: yup.string().ensure(), title: yup.string().ensure(),
code: yup.string().ensure(), code: yup.string().ensure(),
url: yup.string().ensure(), urls: yup
.array(yup.string().required())
.defined()
.test({
name: "unique",
test: (value) => {
const dupes = value
.map((e, i, a) => {
if (a.indexOf(e) !== i) {
return String(i - 1);
} else {
return null;
}
})
.filter((e) => e !== null) as string[];
if (dupes.length === 0) return true;
return new yup.ValidationError(dupes.join(" "), value, "urls");
},
}),
date: yup date: yup
.string() .string()
.ensure() .ensure()
@ -143,7 +161,7 @@ export const SceneEditPanel: React.FC<IProps> = ({
() => ({ () => ({
title: scene.title ?? "", title: scene.title ?? "",
code: scene.code ?? "", code: scene.code ?? "",
url: scene.url ?? "", urls: scene.urls ?? [],
date: scene.date ?? "", date: scene.date ?? "",
director: scene.director ?? "", director: scene.director ?? "",
rating100: scene.rating100 ?? null, rating100: scene.rating100 ?? null,
@ -333,7 +351,7 @@ export const SceneEditPanel: React.FC<IProps> = ({
director: fragment.director, director: fragment.director,
remote_site_id: fragment.remote_site_id, remote_site_id: fragment.remote_site_id,
title: fragment.title, title: fragment.title,
url: fragment.url, urls: fragment.urls,
}; };
const result = await queryScrapeSceneQueryFragment(s, input); const result = await queryScrapeSceneQueryFragment(s, input);
@ -549,8 +567,8 @@ export const SceneEditPanel: React.FC<IProps> = ({
formik.setFieldValue("date", updatedScene.date); formik.setFieldValue("date", updatedScene.date);
} }
if (updatedScene.url) { if (updatedScene.urls) {
formik.setFieldValue("url", updatedScene.url); formik.setFieldValue("urls", updatedScene.urls);
} }
if (updatedScene.studio && updatedScene.studio.stored_id) { if (updatedScene.studio && updatedScene.studio.stored_id) {
@ -624,13 +642,13 @@ export const SceneEditPanel: React.FC<IProps> = ({
} }
} }
async function onScrapeSceneURL() { async function onScrapeSceneURL(url: string) {
if (!formik.values.url) { if (!url) {
return; return;
} }
setIsLoading(true); setIsLoading(true);
try { try {
const result = await queryScrapeSceneURL(formik.values.url); const result = await queryScrapeSceneURL(url);
if (!result.data || !result.data.scrapeSceneURL) { if (!result.data || !result.data.scrapeSceneURL) {
return; return;
} }
@ -683,6 +701,14 @@ export const SceneEditPanel: React.FC<IProps> = ({
if (isLoading) return <LoadingIndicator />; if (isLoading) return <LoadingIndicator />;
const urlsErrors = Array.isArray(formik.errors.urls)
? formik.errors.urls[0]
: formik.errors.urls;
const urlsErrorMsg = urlsErrors
? intl.formatMessage({ id: "validation.urls_must_be_unique" })
: undefined;
const urlsErrorIdx = urlsErrors?.split(" ").map((e) => parseInt(e));
return ( return (
<div id="scene-edit-details"> <div id="scene-edit-details">
<Prompt <Prompt
@ -728,18 +754,20 @@ export const SceneEditPanel: React.FC<IProps> = ({
<div className="col-12 col-lg-7 col-xl-12"> <div className="col-12 col-lg-7 col-xl-12">
{renderTextField("title", intl.formatMessage({ id: "title" }))} {renderTextField("title", intl.formatMessage({ id: "title" }))}
{renderTextField("code", intl.formatMessage({ id: "scene_code" }))} {renderTextField("code", intl.formatMessage({ id: "scene_code" }))}
<Form.Group controlId="url" as={Row}> <Form.Group controlId="urls" as={Row}>
<Col xs={3} className="pr-0 url-label"> <Col xs={3} className="pr-0 url-label">
<Form.Label className="col-form-label"> <Form.Label className="col-form-label">
<FormattedMessage id="url" /> <FormattedMessage id="urls" />
</Form.Label> </Form.Label>
</Col> </Col>
<Col xs={9}> <Col xs={9}>
<URLField <URLListInput
{...formik.getFieldProps("url")} value={formik.values.urls ?? []}
onScrapeClick={onScrapeSceneURL} setValue={(value) => formik.setFieldValue("urls", value)}
errors={urlsErrorMsg}
errorIdx={urlsErrorIdx}
onScrapeClick={(url) => onScrapeSceneURL(url)}
urlScrapable={urlScrapable} urlScrapable={urlScrapable}
isInvalid={!!formik.getFieldMeta("url").error}
/> />
</Col> </Col>
</Form.Group> </Form.Group>

View file

@ -16,7 +16,7 @@ import { useToast } from "src/hooks/Toast";
import NavUtils from "src/utils/navigation"; import NavUtils from "src/utils/navigation";
import TextUtils from "src/utils/text"; import TextUtils from "src/utils/text";
import { getStashboxBase } from "src/utils/stashbox"; import { getStashboxBase } from "src/utils/stashbox";
import { TextField, URLField } from "src/utils/field"; import { TextField, URLField, URLsField } from "src/utils/field";
interface IFileInfoPanelProps { interface IFileInfoPanelProps {
sceneID: string; sceneID: string;
@ -316,12 +316,7 @@ export const SceneFileInfoPanel: React.FC<ISceneFileInfoPanelProps> = (
)} )}
{renderFunscript()} {renderFunscript()}
{renderInteractiveSpeed()} {renderInteractiveSpeed()}
<URLField <URLsField id="urls" urls={props.scene.urls} truncate />
id="media_info.downloaded_from"
url={props.scene.url}
value={props.scene.url}
truncate
/>
{renderStashIDs()} {renderStashIDs()}
<TextField <TextField
id="media_info.play_count" id="media_info.play_count"

View file

@ -14,6 +14,7 @@ import {
ScrapedTextAreaRow, ScrapedTextAreaRow,
ScrapedImageRow, ScrapedImageRow,
IHasName, IHasName,
ScrapedStringListRow,
} from "src/components/Shared/ScrapeDialog"; } from "src/components/Shared/ScrapeDialog";
import clone from "lodash-es/clone"; import clone from "lodash-es/clone";
import { import {
@ -26,6 +27,7 @@ import {
import { useToast } from "src/hooks/Toast"; import { useToast } from "src/hooks/Toast";
import DurationUtils from "src/utils/duration"; import DurationUtils from "src/utils/duration";
import { useIntl } from "react-intl"; import { useIntl } from "react-intl";
import { uniq } from "lodash-es";
interface IScrapedStudioRow { interface IScrapedStudioRow {
title: string; title: string;
@ -295,9 +297,16 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
const [code, setCode] = useState<ScrapeResult<string>>( const [code, setCode] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(scene.code, scraped.code) new ScrapeResult<string>(scene.code, scraped.code)
); );
const [url, setURL] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(scene.url, scraped.url) const [urls, setURLs] = useState<ScrapeResult<string[]>>(
new ScrapeResult<string[]>(
scene.urls,
scraped.urls
? uniq((scene.urls ?? []).concat(scraped.urls ?? []))
: undefined
)
); );
const [date, setDate] = useState<ScrapeResult<string>>( const [date, setDate] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(scene.date, scraped.date) new ScrapeResult<string>(scene.date, scraped.date)
); );
@ -407,7 +416,7 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
[ [
title, title,
code, code,
url, urls,
date, date,
director, director,
studio, studio,
@ -581,7 +590,7 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
return { return {
title: title.getNewValue(), title: title.getNewValue(),
code: code.getNewValue(), code: code.getNewValue(),
url: url.getNewValue(), urls: urls.getNewValue(),
date: date.getNewValue(), date: date.getNewValue(),
director: director.getNewValue(), director: director.getNewValue(),
studio: newStudioValue studio: newStudioValue
@ -627,10 +636,10 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
result={code} result={code}
onChange={(value) => setCode(value)} onChange={(value) => setCode(value)}
/> />
<ScrapedInputGroupRow <ScrapedStringListRow
title={intl.formatMessage({ id: "url" })} title={intl.formatMessage({ id: "urls" })}
result={url} result={urls}
onChange={(value) => setURL(value)} onChange={(value) => setURLs(value)}
/> />
<ScrapedInputGroupRow <ScrapedInputGroupRow
title={intl.formatMessage({ id: "date" })} title={intl.formatMessage({ id: "date" })}

View file

@ -17,6 +17,7 @@ import {
ScrapeDialogRow, ScrapeDialogRow,
ScrapedImageRow, ScrapedImageRow,
ScrapedInputGroupRow, ScrapedInputGroupRow,
ScrapedStringListRow,
ScrapedTextAreaRow, ScrapedTextAreaRow,
ScrapeResult, ScrapeResult,
ZeroableScrapeResult, ZeroableScrapeResult,
@ -61,8 +62,8 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
const [code, setCode] = useState<ScrapeResult<string>>( const [code, setCode] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(dest.code) new ScrapeResult<string>(dest.code)
); );
const [url, setURL] = useState<ScrapeResult<string>>( const [url, setURL] = useState<ScrapeResult<string[]>>(
new ScrapeResult<string>(dest.url) new ScrapeResult<string[]>(dest.urls)
); );
const [date, setDate] = useState<ScrapeResult<string>>( const [date, setDate] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(dest.date) new ScrapeResult<string>(dest.date)
@ -164,7 +165,7 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
new ScrapeResult(dest.code, sources.find((s) => s.code)?.code, !dest.code) new ScrapeResult(dest.code, sources.find((s) => s.code)?.code, !dest.code)
); );
setURL( setURL(
new ScrapeResult(dest.url, sources.find((s) => s.url)?.url, !dest.url) new ScrapeResult(dest.urls, sources.find((s) => s.urls)?.urls, !dest.urls)
); );
setDate( setDate(
new ScrapeResult(dest.date, sources.find((s) => s.date)?.date, !dest.date) new ScrapeResult(dest.date, sources.find((s) => s.date)?.date, !dest.date)
@ -361,8 +362,8 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
result={code} result={code}
onChange={(value) => setCode(value)} onChange={(value) => setCode(value)}
/> />
<ScrapedInputGroupRow <ScrapedStringListRow
title={intl.formatMessage({ id: "url" })} title={intl.formatMessage({ id: "urls" })}
result={url} result={url}
onChange={(value) => setURL(value)} onChange={(value) => setURL(value)}
/> />
@ -546,7 +547,7 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
id: dest.id, id: dest.id,
title: title.getNewValue(), title: title.getNewValue(),
code: code.getNewValue(), code: code.getNewValue(),
url: url.getNewValue(), urls: url.getNewValue(),
date: date.getNewValue(), date: date.getNewValue(),
rating100: rating.getNewValue(), rating100: rating.getNewValue(),
o_counter: oCounter.getNewValue(), o_counter: oCounter.getNewValue(),

View file

@ -22,6 +22,7 @@ import {
} from "@fortawesome/free-solid-svg-icons"; } from "@fortawesome/free-solid-svg-icons";
import { getCountryByISO } from "src/utils/country"; import { getCountryByISO } from "src/utils/country";
import { CountrySelect } from "./CountrySelect"; import { CountrySelect } from "./CountrySelect";
import { StringListInput } from "./StringListInput";
export class ScrapeResult<T> { export class ScrapeResult<T> {
public newValue?: T; public newValue?: T;
@ -102,6 +103,7 @@ interface IScrapedFieldProps<T> {
interface IScrapedRowProps<T, V extends IHasName> interface IScrapedRowProps<T, V extends IHasName>
extends IScrapedFieldProps<T> { extends IScrapedFieldProps<T> {
className?: string;
title: string; title: string;
renderOriginalField: (result: ScrapeResult<T>) => JSX.Element | undefined; renderOriginalField: (result: ScrapeResult<T>) => JSX.Element | undefined;
renderNewField: (result: ScrapeResult<T>) => JSX.Element | undefined; renderNewField: (result: ScrapeResult<T>) => JSX.Element | undefined;
@ -175,7 +177,7 @@ export const ScrapeDialogRow = <T, V extends IHasName>(
} }
return ( return (
<Row className="px-3 pt-3"> <Row className={`px-3 pt-3 ${props.className ?? ""}`}>
<Form.Label column lg="3"> <Form.Label column lg="3">
{props.title} {props.title}
</Form.Label> </Form.Label>
@ -276,6 +278,71 @@ export const ScrapedInputGroupRow: React.FC<IScrapedInputGroupRowProps> = (
); );
}; };
interface IScrapedStringListProps {
isNew?: boolean;
placeholder?: string;
locked?: boolean;
result: ScrapeResult<string[]>;
onChange?: (value: string[]) => void;
}
const ScrapedStringList: React.FC<IScrapedStringListProps> = (props) => {
const value = props.isNew
? props.result.newValue
: props.result.originalValue;
return (
<StringListInput
value={value ?? []}
setValue={(v) => {
if (props.isNew && props.onChange) {
props.onChange(v);
}
}}
placeholder={props.placeholder}
readOnly={!props.isNew || props.locked}
/>
);
};
interface IScrapedStringListRowProps {
title: string;
placeholder?: string;
result: ScrapeResult<string[]>;
locked?: boolean;
onChange: (value: ScrapeResult<string[]>) => void;
}
export const ScrapedStringListRow: React.FC<IScrapedStringListRowProps> = (
props
) => {
return (
<ScrapeDialogRow
className="string-list-row"
title={props.title}
result={props.result}
renderOriginalField={() => (
<ScrapedStringList
placeholder={props.placeholder || props.title}
result={props.result}
/>
)}
renderNewField={() => (
<ScrapedStringList
placeholder={props.placeholder || props.title}
result={props.result}
isNew
locked={props.locked}
onChange={(value) =>
props.onChange(props.result.cloneWithValue(value))
}
/>
)}
onChange={props.onChange}
/>
);
};
const ScrapedTextArea: React.FC<IScrapedInputGroupProps> = (props) => { const ScrapedTextArea: React.FC<IScrapedInputGroupProps> = (props) => {
return ( return (
<FormControl <FormControl

View file

@ -1,18 +1,55 @@
import { faMinus } from "@fortawesome/free-solid-svg-icons"; import { faMinus } from "@fortawesome/free-solid-svg-icons";
import React from "react"; import React, { ComponentType } from "react";
import { Button, Form, InputGroup } from "react-bootstrap"; import { Button, Form, InputGroup } from "react-bootstrap";
import { Icon } from "./Icon"; import { Icon } from "./Icon";
interface IStringListInputProps { interface IListInputComponentProps {
value: string;
setValue: (value: string) => void;
placeholder?: string;
className?: string;
readOnly?: boolean;
}
interface IListInputAppendProps {
value: string;
}
export interface IStringListInputProps {
value: string[]; value: string[];
setValue: (value: string[]) => void; setValue: (value: string[]) => void;
inputComponent?: ComponentType<IListInputComponentProps>;
appendComponent?: ComponentType<IListInputAppendProps>;
placeholder?: string; placeholder?: string;
className?: string; className?: string;
errors?: string; errors?: string;
errorIdx?: number[]; errorIdx?: number[];
readOnly?: boolean;
} }
export const StringInput: React.FC<IListInputComponentProps> = ({
className,
placeholder,
value,
setValue,
readOnly = false,
}) => {
return (
<Form.Control
className={`text-input ${className ?? ""}`}
value={value}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setValue(e.currentTarget.value)
}
placeholder={placeholder}
readOnly={readOnly}
/>
);
};
export const StringListInput: React.FC<IStringListInputProps> = (props) => { export const StringListInput: React.FC<IStringListInputProps> = (props) => {
const Input = props.inputComponent ?? StringInput;
const AppendComponent = props.appendComponent;
const values = props.value.concat(""); const values = props.value.concat("");
function valueChanged(idx: number, value: string) { function valueChanged(idx: number, value: string) {
@ -37,17 +74,16 @@ export const StringListInput: React.FC<IStringListInputProps> = (props) => {
<Form.Group> <Form.Group>
{values.map((v, i) => ( {values.map((v, i) => (
<InputGroup className={props.className} key={i}> <InputGroup className={props.className} key={i}>
<Form.Control <Input
className={`text-input ${
props.errorIdx?.includes(i) ? "is-invalid" : ""
}`}
value={v} value={v}
onChange={(e: React.ChangeEvent<HTMLInputElement>) => setValue={(value) => valueChanged(i, value)}
valueChanged(i, e.currentTarget.value)
}
placeholder={props.placeholder} placeholder={props.placeholder}
className={props.errorIdx?.includes(i) ? "is-invalid" : ""}
readOnly={props.readOnly}
/> />
<InputGroup.Append> <InputGroup.Append>
{AppendComponent && <AppendComponent value={v} />}
{!props.readOnly && (
<Button <Button
variant="danger" variant="danger"
onClick={() => removeValue(i)} onClick={() => removeValue(i)}
@ -55,6 +91,7 @@ export const StringListInput: React.FC<IStringListInputProps> = (props) => {
> >
<Icon icon={faMinus} /> <Icon icon={faMinus} />
</Button> </Button>
)}
</InputGroup.Append> </InputGroup.Append>
</InputGroup> </InputGroup>
))} ))}

View file

@ -4,6 +4,11 @@ import { Button, InputGroup, Form } from "react-bootstrap";
import { Icon } from "./Icon"; import { Icon } from "./Icon";
import { FormikHandlers } from "formik"; import { FormikHandlers } from "formik";
import { faFileDownload } from "@fortawesome/free-solid-svg-icons"; import { faFileDownload } from "@fortawesome/free-solid-svg-icons";
import {
IStringListInputProps,
StringInput,
StringListInput,
} from "./StringListInput";
interface IProps { interface IProps {
value: string; value: string;
@ -43,3 +48,33 @@ export const URLField: React.FC<IProps> = (props: IProps) => {
</InputGroup> </InputGroup>
); );
}; };
interface IURLListProps extends IStringListInputProps {
onScrapeClick(url: string): void;
urlScrapable(url: string): boolean;
}
export const URLListInput: React.FC<IURLListProps> = (
listProps: IURLListProps
) => {
const intl = useIntl();
const { onScrapeClick, urlScrapable } = listProps;
return (
<StringListInput
{...listProps}
placeholder={intl.formatMessage({ id: "url" })}
inputComponent={StringInput}
appendComponent={(props) => (
<Button
className="scrape-url-button text-input"
variant="secondary"
onClick={() => onScrapeClick(props.value)}
disabled={!props.value || !urlScrapable(props.value)}
title={intl.formatMessage({ id: "actions.scrape" })}
>
<Icon icon={faFileDownload} />
</Button>
)}
/>
);
};

View file

@ -441,3 +441,7 @@ div.react-datepicker {
right: 0; right: 0;
z-index: 4; z-index: 4;
} }
.string-list-row .input-group {
flex-wrap: nowrap;
}

View file

@ -408,7 +408,7 @@ export const TaggerContext: React.FC = ({ children }) => {
details: scene.details, details: scene.details,
remote_site_id: scene.remote_site_id, remote_site_id: scene.remote_site_id,
title: scene.title, title: scene.title,
url: scene.url, urls: scene.urls,
}; };
const result = await queryScrapeSceneQueryFragment( const result = await queryScrapeSceneQueryFragment(

View file

@ -360,7 +360,7 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
), ),
studio_id: studioID, studio_id: studioID,
cover_image: resolveField("cover_image", undefined, imgData), cover_image: resolveField("cover_image", undefined, imgData),
url: resolveField("url", stashScene.url, scene.url), urls: resolveField("url", stashScene.urls, scene.urls),
tag_ids: tagIDs, tag_ids: tagIDs,
stash_ids: stashScene.stash_ids ?? [], stash_ids: stashScene.stash_ids ?? [],
code: resolveField("code", stashScene.code, scene.code), code: resolveField("code", stashScene.code, scene.code),
@ -462,9 +462,11 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
); );
} }
const sceneTitleEl = scene.url ? ( const url = scene.urls?.length ? scene.urls[0] : null;
const sceneTitleEl = url ? (
<a <a
href={scene.url} href={url}
target="_blank" target="_blank"
rel="noopener noreferrer" rel="noopener noreferrer"
className="scene-link" className="scene-link"
@ -558,16 +560,20 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
}; };
const maybeRenderURL = () => { const maybeRenderURL = () => {
if (scene.url) { if (scene.urls) {
return ( return (
<div className="scene-details"> <div className="scene-details">
<OptionalField <OptionalField
exclude={excludedFields[fields.url]} exclude={excludedFields[fields.url]}
setExclude={(v) => setExcludedField(fields.url, v)} setExclude={(v) => setExcludedField(fields.url, v)}
> >
<a href={scene.url} target="_blank" rel="noopener noreferrer"> {scene.urls.map((url) => (
{scene.url} <div key={url}>
<a href={url} target="_blank" rel="noopener noreferrer">
{url}
</a> </a>
</div>
))}
</OptionalField> </OptionalField>
</div> </div>
); );

View file

@ -1270,10 +1270,12 @@
"type": "Type", "type": "Type",
"updated_at": "Updated At", "updated_at": "Updated At",
"url": "URL", "url": "URL",
"urls": "URLs",
"validation": { "validation": {
"aliases_must_be_unique": "aliases must be unique", "aliases_must_be_unique": "aliases must be unique",
"date_invalid_form": "${path} must be in YYYY-MM-DD form", "date_invalid_form": "${path} must be in YYYY-MM-DD form",
"required": "${path} is a required field" "required": "${path} is a required field",
"urls_must_be_unique": "URLs must be unique"
}, },
"videos": "Videos", "videos": "Videos",
"video_codec": "Video Codec", "video_codec": "Video Codec",

View file

@ -90,3 +90,50 @@ export const URLField: React.FC<IURLField> = ({
</> </>
); );
}; };
interface IURLsField {
id?: string;
name?: string;
abbr?: string | null;
urls?: string[] | null;
truncate?: boolean | null;
target?: string;
// use for internal links
trusted?: boolean;
}
export const URLsField: React.FC<IURLsField> = ({
id,
name,
urls,
abbr,
truncate,
target,
trusted,
}) => {
const values = urls ?? [];
if (!values.length) {
return null;
}
const message = (
<>{id ? <FormattedMessage id={id} defaultMessage={name} /> : name}:</>
);
const rel = !trusted ? "noopener noreferrer" : undefined;
return (
<>
<dt>{abbr ? <abbr title={abbr}>{message}</abbr> : message}</dt>
<dd>
<dl>
{values.map((url, i) => (
<a key={i} href={url} target={target || "_blank"} rel={rel}>
{truncate ? <TruncatedText text={url} /> : url}
</a>
))}
</dl>
</dd>
</>
);
};