Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 08:26:00 +01:00
Code Issues Projects Releases Wiki Activity

Multiple scene URLs (#3852)

Browse source 
* Add URLs scene relationship
* Update unit tests
* Update scene edit and details pages
* Update scrapers to use urls
* Post-process scenes during query scrape
* Update UI for URLs
* Change urls label
This commit is contained in:
WithoutPants 2023-07-12 11:51:52 +10:00 committed by GitHub
parent 76a4bfa49a
commit 67d4f9729a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 978 additions and 205 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
graphql/documents/data/scene-slim.graphql
View file

@ -4,7 +4,7 @@ fragment SlimSceneData on Scene {
code
details
director
url
urls
date
rating100
o_counter

2
graphql/documents/data/scene.graphql
View file

@ -4,7 +4,7 @@ fragment SceneData on Scene {
code
details
director
url
urls
date
rating100
o_counter

2
graphql/documents/data/scrapers.graphql
View file

@ -114,7 +114,7 @@ fragment ScrapedSceneData on ScrapedScene {
code
details
director
url
urls
date
image
remote_site_id

12
graphql/schema/types/scene.graphql
View file

@ -40,7 +40,8 @@ type Scene {
code: String
details: String
director: String
url: String
url: String @deprecated(reason: "Use urls")
urls: [String!]
date: String
# rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100")
@ -91,7 +92,8 @@ input SceneCreateInput {
code: String
details: String
director: String
url: String
url: String @deprecated(reason: "Use urls")
urls: [String!]
date: String
# rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100")
@ -119,7 +121,8 @@ input SceneUpdateInput {
code: String
details: String
director: String
url: String
url: String @deprecated(reason: "Use urls")
urls: [String!]
date: String
# rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100")
@ -164,7 +167,8 @@ input BulkSceneUpdateInput {
code: String
details: String
director: String
url: String
url: String @deprecated(reason: "Use urls")
urls: BulkUpdateStrings
date: String
# rating expressed as 1-5
rating: Int @deprecated(reason: "Use 1-100 range with rating100")

6
graphql/schema/types/scraper.graphql
View file

@ -64,7 +64,8 @@ type ScrapedScene {
code: String
details: String
director: String
url: String
url: String @deprecated(reason: "use urls")
urls: [String!]
date: String
"""This should be a base64 encoded data URL"""
@ -87,7 +88,8 @@ input ScrapedSceneInput {
code: String
details: String
director: String
url: String
url: String @deprecated(reason: "use urls")
urls: [String!]
date: String
# no image, file, duration or relationships

29
internal/api/resolver_model_scene.go
View file

@ -405,3 +405,32 @@ func (r *sceneResolver) InteractiveSpeed(ctx context.Context, obj *models.Scene)
return primaryFile.InteractiveSpeed, nil
}
func (r *sceneResolver) URL(ctx context.Context, obj *models.Scene) (*string, error) {
if !obj.URLs.Loaded() {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
return obj.LoadURLs(ctx, r.repository.Scene)
}); err != nil {
return nil, err
}
}
urls := obj.URLs.List()
if len(urls) == 0 {
return nil, nil
}
return &urls[0], nil
}
func (r *sceneResolver) Urls(ctx context.Context, obj *models.Scene) ([]string, error) {
if !obj.URLs.Loaded() {
if err := r.withReadTxn(ctx, func(ctx context.Context) error {
return obj.LoadURLs(ctx, r.repository.Scene)
}); err != nil {
return nil, err
}
}
return obj.URLs.List(), nil
}

33
internal/api/resolver_mutation_scene.go
View file

@ -67,7 +67,6 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp
Code: translator.string(input.Code, "code"),
Details: translator.string(input.Details, "details"),
Director: translator.string(input.Director, "director"),
URL: translator.string(input.URL, "url"),
Date: translator.datePtr(input.Date, "date"),
Rating: translator.ratingConversionInt(input.Rating, input.Rating100),
Organized: translator.bool(input.Organized, "organized"),
@ -83,6 +82,12 @@ func (r *mutationResolver) SceneCreate(ctx context.Context, input SceneCreateInp
return nil, fmt.Errorf("converting studio id: %w", err)
}
if input.Urls != nil {
newScene.URLs = models.NewRelatedStrings(input.Urls)
} else if input.URL != nil {
newScene.URLs = models.NewRelatedStrings([]string{*input.URL})
}
var coverImageData []byte
if input.CoverImage != nil && *input.CoverImage != "" {
var err error
@ -168,7 +173,6 @@ func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTr
updatedScene.Code = translator.optionalString(input.Code, "code")
updatedScene.Details = translator.optionalString(input.Details, "details")
updatedScene.Director = translator.optionalString(input.Director, "director")
updatedScene.URL = translator.optionalString(input.URL, "url")
updatedScene.Date = translator.optionalDate(input.Date, "date")
updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100)
updatedScene.OCounter = translator.optionalInt(input.OCounter, "o_counter")
@ -182,6 +186,18 @@ func scenePartialFromInput(input models.SceneUpdateInput, translator changesetTr
updatedScene.Organized = translator.optionalBool(input.Organized, "organized")
if translator.hasField("urls") {
updatedScene.URLs = &models.UpdateStrings{
Values: input.Urls,
Mode: models.RelationshipUpdateModeSet,
}
} else if translator.hasField("url") {
updatedScene.URLs = &models.UpdateStrings{
Values: []string{*input.URL},
Mode: models.RelationshipUpdateModeSet,
}
}
if input.PrimaryFileID != nil {
primaryFileID, err := strconv.Atoi(*input.PrimaryFileID)
if err != nil {
@ -339,7 +355,6 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU
updatedScene.Code = translator.optionalString(input.Code, "code")
updatedScene.Details = translator.optionalString(input.Details, "details")
updatedScene.Director = translator.optionalString(input.Director, "director")
updatedScene.URL = translator.optionalString(input.URL, "url")
updatedScene.Date = translator.optionalDate(input.Date, "date")
updatedScene.Rating = translator.ratingConversionOptional(input.Rating, input.Rating100)
updatedScene.StudioID, err = translator.optionalIntFromString(input.StudioID, "studio_id")
@ -349,6 +364,18 @@ func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input BulkSceneU
updatedScene.Organized = translator.optionalBool(input.Organized, "organized")
if translator.hasField("urls") {
updatedScene.URLs = &models.UpdateStrings{
Values: input.Urls.Values,
Mode: input.Urls.Mode,
}
} else if translator.hasField("url") {
updatedScene.URLs = &models.UpdateStrings{
Values: []string{*input.URL},
Mode: models.RelationshipUpdateModeSet,
}
}
if translator.hasField("performer_ids") {
updatedScene.PerformerIDs, err = translateUpdateIDs(input.PerformerIds.Ids, input.PerformerIds.Mode)
if err != nil {

4
internal/api/resolver_mutation_stash_box.go
View file

@ -68,6 +68,10 @@ func (r *mutationResolver) SubmitStashBoxSceneDraft(ctx context.Context, input S
logger.Errorf("Error getting scene cover: %v", err)
}
if err := scene.LoadURLs(ctx, r.repository.Scene); err != nil {
return fmt.Errorf("loading scene URLs: %w", err)
}
res, err = client.SubmitSceneDraft(ctx, scene, boxes[input.StashBoxIndex].Endpoint, cover)
return err
})

4
internal/autotag/integration_test.go
View file

@ -176,7 +176,7 @@ func createScenes(ctx context.Context, sqb models.SceneReaderWriter, folderStore
s := &models.Scene{
Title: expectedMatchTitle,
URL: existingStudioSceneName,
Code: existingStudioSceneName,
StudioID: &existingStudioID,
}
if err := createScene(ctx, sqb, s, f); err != nil {
@ -625,7 +625,7 @@ func TestParseStudioScenes(t *testing.T) {
for _, scene := range scenes {
// check for existing studio id scene first
if scene.URL == existingStudioSceneName {
if scene.Code == existingStudioSceneName {
if scene.StudioID == nil || *scene.StudioID != existingStudioID {
t.Error("Incorrectly overwrote studio ID for scene with existing studio ID")
}

42
internal/identify/identify.go
View file

@ -10,7 +10,7 @@ import (
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/txn"
"github.com/stashapp/stash/pkg/utils"
)
@ -226,7 +226,7 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene,
return nil, fmt.Errorf("error converting tag ID %s: %w", *options.SkipSingleNamePerformerTag, err)
}
tagIDs = intslice.IntAppendUnique(tagIDs, int(tagID))
tagIDs = sliceutil.AppendUnique(tagIDs, int(tagID))
}
if tagIDs != nil {
ret.Partial.TagIDs = &models.UpdateIDs{
@ -260,6 +260,9 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage
var updater *scene.UpdateSet
if err := txn.WithTxn(ctx, txnManager, func(ctx context.Context) error {
// load scene relationships
if err := s.LoadURLs(ctx, t.SceneReaderUpdater); err != nil {
return err
}
if err := s.LoadPerformerIDs(ctx, t.SceneReaderUpdater); err != nil {
return err
}
@ -320,7 +323,7 @@ func (t *SceneIdentifier) addTagToScene(ctx context.Context, txnManager txn.Mana
}
existing := s.TagIDs.List()
if intslice.IntInclude(existing, tagID) {
if sliceutil.Include(existing, tagID) {
// skip if the scene was already tagged
return nil
}
@ -376,9 +379,27 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp
partial.Details = models.NewOptionalString(*scraped.Details)
}
}
if scraped.URL != nil && (scene.URL != *scraped.URL) {
if shouldSetSingleValueField(fieldOptions["url"], scene.URL != "") {
partial.URL = models.NewOptionalString(*scraped.URL)
if len(scraped.URLs) > 0 && shouldSetSingleValueField(fieldOptions["url"], false) {
// if overwrite, then set over the top
switch getFieldStrategy(fieldOptions["url"]) {
case FieldStrategyOverwrite:
// only overwrite if not equal
if len(sliceutil.Exclude(scene.URLs.List(), scraped.URLs)) != 0 {
partial.URLs = &models.UpdateStrings{
Values: scraped.URLs,
Mode: models.RelationshipUpdateModeSet,
}
}
case FieldStrategyMerge:
// if merge, add if not already present
urls := sliceutil.AppendUniques(scene.URLs.List(), scraped.URLs)
if len(urls) != len(scene.URLs.List()) {
partial.URLs = &models.UpdateStrings{
Values: urls,
Mode: models.RelationshipUpdateModeSet,
}
}
}
}
if scraped.Director != nil && (scene.Director != *scraped.Director) {
@ -399,7 +420,7 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp
return partial
}
func shouldSetSingleValueField(strategy *FieldOptions, hasExistingValue bool) bool {
func getFieldStrategy(strategy *FieldOptions) FieldStrategy {
// if unset then default to MERGE
fs := FieldStrategyMerge
@ -407,6 +428,13 @@ func shouldSetSingleValueField(strategy *FieldOptions, hasExistingValue bool) bo
fs = strategy.Strategy
}
return fs
}
func shouldSetSingleValueField(strategy *FieldOptions, hasExistingValue bool) bool {
// if unset then default to MERGE
fs := getFieldStrategy(strategy)
if fs == FieldStrategyIgnore {
return false
}

42
internal/identify/identify_test.go
View file

@ -11,6 +11,7 @@ import (
"github.com/stashapp/stash/pkg/models/mocks"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@ -108,8 +109,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
}
mockSceneReaderWriter := &mocks.SceneReaderWriter{}
mockTagFinderCreator := &mocks.TagReaderWriter{}
mockSceneReaderWriter.On("GetURLs", mock.Anything, mock.Anything).Return(nil, nil)
mockSceneReaderWriter.On("UpdatePartial", mock.Anything, mock.MatchedBy(func(id int) bool {
return id == errUpdateID
}), mock.Anything).Return(nil, errors.New("update error"))
@ -117,6 +117,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
return id != errUpdateID
}), mock.Anything).Return(nil, nil)
mockTagFinderCreator := &mocks.TagReaderWriter{}
mockTagFinderCreator.On("Find", mock.Anything, skipMultipleTagID).Return(&models.Tag{
ID: skipMultipleTagID,
Name: skipMultipleTagIDStr,
@ -236,6 +237,7 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
"empty update",
args{
&models.Scene{
URLs: models.NewRelatedStrings([]string{}),
PerformerIDs: models.NewRelatedIDs([]int{}),
TagIDs: models.NewRelatedIDs([]int{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
@ -351,33 +353,44 @@ func Test_getScenePartial(t *testing.T) {
Title: originalTitle,
Date: &originalDateObj,
Details: originalDetails,
URL: originalURL,
URLs: models.NewRelatedStrings([]string{originalURL}),
}
organisedScene := *originalScene
organisedScene.Organized = true
emptyScene := &models.Scene{}
emptyScene := &models.Scene{
URLs: models.NewRelatedStrings([]string{}),
}
postPartial := models.ScenePartial{
Title: models.NewOptionalString(scrapedTitle),
Date: models.NewOptionalDate(scrapedDateObj),
Details: models.NewOptionalString(scrapedDetails),
URL: models.NewOptionalString(scrapedURL),
URLs: &models.UpdateStrings{
Values: []string{scrapedURL},
Mode: models.RelationshipUpdateModeSet,
},
}
postPartialMerge := postPartial
postPartialMerge.URLs = &models.UpdateStrings{
Values: []string{scrapedURL},
Mode: models.RelationshipUpdateModeSet,
}
scrapedScene := &scraper.ScrapedScene{
Title: &scrapedTitle,
Date: &scrapedDate,
Details: &scrapedDetails,
URL: &scrapedURL,
URLs: []string{scrapedURL},
}
scrapedUnchangedScene := &scraper.ScrapedScene{
Title: &originalTitle,
Date: &originalDate,
Details: &originalDetails,
URL: &originalURL,
URLs: []string{originalURL},
}
makeFieldOptions := func(input *FieldOptions) map[string]*FieldOptions {
@ -440,7 +453,12 @@ func Test_getScenePartial(t *testing.T) {
mergeAll,
false,
},
models.ScenePartial{},
models.ScenePartial{
URLs: &models.UpdateStrings{
Values: []string{originalURL, scrapedURL},
Mode: models.RelationshipUpdateModeSet,
},
},
},
{
"merge (empty values)",
@ -450,7 +468,7 @@ func Test_getScenePartial(t *testing.T) {
mergeAll,
false,
},
postPartial,
postPartialMerge,
},
{
"unchanged",
@ -487,9 +505,9 @@ func Test_getScenePartial(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getScenePartial(tt.args.scene, tt.args.scraped, tt.args.fieldOptions, tt.args.setOrganized); !reflect.DeepEqual(got, tt.want) {
t.Errorf("getScenePartial() = %v, want %v", got, tt.want)
}
got := getScenePartial(tt.args.scene, tt.args.scraped, tt.args.fieldOptions, tt.args.setOrganized)
assert.Equal(t, tt.want, got)
})
}
}

1
internal/identify/scene.go
View file

@ -24,6 +24,7 @@ type SceneReaderUpdater interface {
models.PerformerIDLoader
models.TagIDLoader
models.StashIDLoader
models.URLLoader
}
type TagCreatorFinder interface {

1
internal/manager/repository.go
View file

@ -29,6 +29,7 @@ type GalleryReaderWriter interface {
type SceneReaderWriter interface {
models.SceneReaderWriter
scene.CreatorUpdater
models.URLLoader
GetManyFileIDs(ctx context.Context, ids []int) ([][]file.ID, error)
}

2
pkg/models/jsonschema/scene.go
View file

@ -42,7 +42,9 @@ type Scene struct {
Title string `json:"title,omitempty"`
Code string `json:"code,omitempty"`
Studio string `json:"studio,omitempty"`
// deprecated - for import only
URL string `json:"url,omitempty"`
URLs []string `json:"urls,omitempty"`
Date string `json:"date,omitempty"`
Rating int `json:"rating,omitempty"`
Organized bool `json:"organized,omitempty"`

23
pkg/models/mocks/SceneReaderWriter.go
View file

@ -624,6 +624,29 @@ func (_m *SceneReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]in
return r0, r1
}
// GetURLs provides a mock function with given fields: ctx, relatedID
func (_m *SceneReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) {
ret := _m.Called(ctx, relatedID)
var r0 []string
if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok {
r0 = rf(ctx, relatedID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, int) error); ok {
r1 = rf(ctx, relatedID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// HasCover provides a mock function with given fields: ctx, sceneID
func (_m *SceneReaderWriter) HasCover(ctx context.Context, sceneID int) (bool, error) {
ret := _m.Called(ctx, sceneID)

17
pkg/models/model_scene.go
View file

@ -17,7 +17,6 @@ type Scene struct {
Code string `json:"code"`
Details string `json:"details"`
Director string `json:"director"`
URL string `json:"url"`
Date *Date `json:"date"`
// Rating expressed in 1-100 scale
Rating *int `json:"rating"`
@ -43,6 +42,7 @@ type Scene struct {
PlayDuration float64 `json:"play_duration"`
PlayCount int `json:"play_count"`
URLs RelatedStrings `json:"urls"`
GalleryIDs RelatedIDs `json:"gallery_ids"`
TagIDs RelatedIDs `json:"tag_ids"`
PerformerIDs RelatedIDs `json:"performer_ids"`
@ -50,6 +50,12 @@ type Scene struct {
StashIDs RelatedStashIDs `json:"stash_ids"`
}
func (s *Scene) LoadURLs(ctx context.Context, l URLLoader) error {
return s.URLs.load(func() ([]string, error) {
return l.GetURLs(ctx, s.ID)
})
}
func (s *Scene) LoadFiles(ctx context.Context, l VideoFileLoader) error {
return s.Files.load(func() ([]*file.VideoFile, error) {
return l.GetFiles(ctx, s.ID)
@ -110,6 +116,10 @@ func (s *Scene) LoadStashIDs(ctx context.Context, l StashIDLoader) error {
}
func (s *Scene) LoadRelationships(ctx context.Context, l SceneReader) error {
if err := s.LoadURLs(ctx, l); err != nil {
return err
}
if err := s.LoadGalleryIDs(ctx, l); err != nil {
return err
}
@ -144,7 +154,6 @@ type ScenePartial struct {
Code OptionalString
Details OptionalString
Director OptionalString
URL OptionalString
Date OptionalDate
// Rating expressed in 1-100 scale
Rating OptionalInt
@ -158,6 +167,7 @@ type ScenePartial struct {
PlayCount OptionalInt
LastPlayedAt OptionalTime
URLs *UpdateStrings
GalleryIDs *UpdateIDs
TagIDs *UpdateIDs
PerformerIDs *UpdateIDs
@ -193,6 +203,7 @@ type SceneUpdateInput struct {
Rating100 *int `json:"rating100"`
OCounter *int `json:"o_counter"`
Organized *bool `json:"organized"`
Urls []string `json:"urls"`
StudioID *string `json:"studio_id"`
GalleryIds []string `json:"gallery_ids"`
PerformerIds []string `json:"performer_ids"`
@ -227,7 +238,7 @@ func (s ScenePartial) UpdateInput(id int) SceneUpdateInput {
Code: s.Code.Ptr(),
Details: s.Details.Ptr(),
Director: s.Director.Ptr(),
URL: s.URL.Ptr(),
Urls: s.URLs.Strings(),
Date: dateStr,
Rating100: s.Rating.Ptr(),
Organized: s.Organized.Ptr(),

7
pkg/models/model_scene_test.go
View file

@ -41,7 +41,10 @@ func TestScenePartial_UpdateInput(t *testing.T) {
Code: NewOptionalString(code),
Details: NewOptionalString(details),
Director: NewOptionalString(director),
URL: NewOptionalString(url),
URLs: &UpdateStrings{
Values: []string{url},
Mode: RelationshipUpdateModeSet,
},
Date: NewOptionalDate(dateObj),
Rating: NewOptionalInt(rating100),
Organized: NewOptionalBool(organized),
@ -53,7 +56,7 @@ func TestScenePartial_UpdateInput(t *testing.T) {
Code: &code,
Details: &details,
Director: &director,
URL: &url,
Urls: []string{url},
Date: &date,
Rating: &ratingLegacy,
Rating100: &rating100,

4
pkg/models/relationships.go
View file

@ -42,6 +42,10 @@ type AliasLoader interface {
GetAliases(ctx context.Context, relatedID int) ([]string, error)
}
type URLLoader interface {
GetURLs(ctx context.Context, relatedID int) ([]string, error)
}
// RelatedIDs represents a list of related IDs.
// TODO - this can be made generic
type RelatedIDs struct {

1
pkg/models/scene.go
View file

@ -159,6 +159,7 @@ type SceneReader interface {
FindByGalleryID(ctx context.Context, performerID int) ([]*Scene, error)
FindDuplicates(ctx context.Context, distance int, durationDiff float64) ([][]*Scene, error)
URLLoader
GalleryIDLoader
PerformerIDLoader
TagIDLoader

8
pkg/models/update.go
View file

@ -110,3 +110,11 @@ type UpdateStrings struct {
Values []string `json:"values"`
Mode RelationshipUpdateMode `json:"mode"`
}
func (u *UpdateStrings) Strings() []string {
if u == nil {
return nil
}
return u.Values
}

49
pkg/scene/export.go
View file

@ -41,7 +41,7 @@ func ToBasicJSON(ctx context.Context, reader CoverGetter, scene *models.Scene) (
newSceneJSON := jsonschema.Scene{
Title: scene.Title,
Code: scene.Code,
URL: scene.URL,
URLs: scene.URLs.List(),
Details: scene.Details,
Director: scene.Director,
CreatedAt: json.JSONTime{Time: scene.CreatedAt},
@ -86,53 +86,6 @@ func ToBasicJSON(ctx context.Context, reader CoverGetter, scene *models.Scene) (
return &newSceneJSON, nil
}
// func getSceneFileJSON(scene *models.Scene) *jsonschema.SceneFile {
// ret := &jsonschema.SceneFile{}
// TODO
// if scene.FileModTime != nil {
// ret.ModTime = json.JSONTime{Time: *scene.FileModTime}
// }
// if scene.Size != nil {
// ret.Size = *scene.Size
// }
// if scene.Duration != nil {
// ret.Duration = getDecimalString(*scene.Duration)
// }
// if scene.VideoCodec != nil {
// ret.VideoCodec = *scene.VideoCodec
// }
// if scene.AudioCodec != nil {
// ret.AudioCodec = *scene.AudioCodec
// }
// if scene.Format != nil {
// ret.Format = *scene.Format
// }
// if scene.Width != nil {
// ret.Width = *scene.Width
// }
// if scene.Height != nil {
// ret.Height = *scene.Height
// }
// if scene.Framerate != nil {
// ret.Framerate = getDecimalString(*scene.Framerate)
// }
// if scene.Bitrate != nil {
// ret.Bitrate = int(*scene.Bitrate)
// }
// return ret
// }
// GetStudioName returns the name of the provided scene's studio. It returns an
// empty string if there is no studio assigned to the scene.
func GetStudioName(ctx context.Context, reader studio.Finder, scene *models.Scene) (string, error) {

6
pkg/scene/export_test.go
View file

@ -92,7 +92,7 @@ func createFullScene(id int) models.Scene {
OCounter: ocounter,
Rating: &rating,
Organized: organized,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Files: models.NewRelatedVideoFiles([]*file.VideoFile{
{
BaseFile: &file.BaseFile{
@ -118,6 +118,7 @@ func createEmptyScene(id int) models.Scene {
},
},
}),
URLs: models.NewRelatedStrings([]string{}),
StashIDs: models.NewRelatedStashIDs([]models.StashID{}),
CreatedAt: createTime,
UpdatedAt: updateTime,
@ -133,7 +134,7 @@ func createFullJSONScene(image string) *jsonschema.Scene {
OCounter: ocounter,
Rating: rating,
Organized: organized,
URL: url,
URLs: []string{url},
CreatedAt: json.JSONTime{
Time: createTime,
},
@ -149,6 +150,7 @@ func createFullJSONScene(image string) *jsonschema.Scene {
func createEmptyJSONScene() *jsonschema.Scene {
return &jsonschema.Scene{
URLs: []string{},
Files: []string{path},
CreatedAt: json.JSONTime{
Time: createTime,

8
pkg/scene/import.go
View file

@ -80,12 +80,10 @@ func (i *Importer) PreImport(ctx context.Context) error {
func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
newScene := models.Scene{
// Path: i.Path,
Title: sceneJSON.Title,
Code: sceneJSON.Code,
Details: sceneJSON.Details,
Director: sceneJSON.Director,
URL: sceneJSON.URL,
PerformerIDs: models.NewRelatedIDs([]int{}),
TagIDs: models.NewRelatedIDs([]int{}),
GalleryIDs: models.NewRelatedIDs([]int{}),
@ -93,6 +91,12 @@ func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
StashIDs: models.NewRelatedStashIDs(sceneJSON.StashIDs),
}
if len(sceneJSON.URLs) > 0 {
newScene.URLs = models.NewRelatedStrings(sceneJSON.URLs)
} else if sceneJSON.URL != "" {
newScene.URLs = models.NewRelatedStrings([]string{sceneJSON.URL})
}
if sceneJSON.Date != "" {
d := models.NewDate(sceneJSON.Date)
newScene.Date = &d

23
pkg/scraper/cache.go
View file

@ -52,6 +52,11 @@ func isCDPPathWS(c GlobalConfig) bool {
return strings.HasPrefix(c.GetScraperCDPPath(), "ws://")
}
type SceneFinder interface {
scene.IDFinder
models.URLLoader
}
type PerformerFinder interface {
match.PerformerAutoTagQueryer
match.PerformerFinder
@ -73,7 +78,7 @@ type GalleryFinder interface {
}
type Repository struct {
SceneFinder scene.IDFinder
SceneFinder SceneFinder
GalleryFinder GalleryFinder
TagFinder TagFinder
PerformerFinder PerformerFinder
@ -240,7 +245,19 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
return nil, fmt.Errorf("%w: cannot use scraper %s to scrape by name", ErrNotSupported, id)
}
return ns.viaName(ctx, c.client, query, ty)
content, err := ns.viaName(ctx, c.client, query, ty)
if err != nil {
return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
}
for i, cc := range content {
content[i], err = c.postScrape(ctx, cc)
if err != nil {
return nil, fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
}
}
return content, nil
}
// ScrapeFragment uses the given fragment input to scrape
@ -361,7 +378,7 @@ func (c Cache) getScene(ctx context.Context, sceneID int) (*models.Scene, error)
return fmt.Errorf("scene with id %d not found", sceneID)
}
return nil
return ret.LoadURLs(ctx, c.repository.SceneFinder)
}); err != nil {
return nil, err
}

8
pkg/scraper/postprocessing.go
View file

@ -106,6 +106,14 @@ func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPer
}
func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene) (ScrapedContent, error) {
// set the URL/URLs field
if scene.URL == nil && len(scene.URLs) > 0 {
scene.URL = &scene.URLs[0]
}
if scene.URL != nil && len(scene.URLs) == 0 {
scene.URLs = []string{*scene.URL}
}
if err := txn.WithReadTxn(ctx, c.txnManager, func(ctx context.Context) error {
pqb := c.repository.PerformerFinder
mqb := c.repository.MovieFinder

8
pkg/scraper/query_url.go
View file

@ -20,8 +20,8 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
if scene.Title != "" {
ret["title"] = scene.Title
}
if scene.URL != "" {
ret["url"] = scene.URL
if len(scene.URLs.List()) > 0 {
ret["url"] = scene.URLs.List()[0]
}
return ret
}
@ -37,7 +37,11 @@ func queryURLParametersFromScrapedScene(scene ScrapedSceneInput) queryURLParamet
setField("title", scene.Title)
setField("code", scene.Code)
if len(scene.URLs) > 0 {
setField("url", &scene.URLs[0])
} else {
setField("url", scene.URL)
}
setField("date", scene.Date)
setField("details", scene.Details)
setField("director", scene.Director)

2
pkg/scraper/scene.go
View file

@ -10,6 +10,7 @@ type ScrapedScene struct {
Details *string `json:"details"`
Director *string `json:"director"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
// This should be a base64 encoded data URL
Image *string `json:"image"`
@ -31,6 +32,7 @@ type ScrapedSceneInput struct {
Details *string `json:"details"`
Director *string `json:"director"`
URL *string `json:"url"`
URLs []string `json:"urls"`
Date *string `json:"date"`
RemoteSiteID *string `json:"remote_site_id"`
}

2
pkg/scraper/stash.go
View file

@ -328,7 +328,7 @@ func sceneToUpdateInput(scene *models.Scene) models.SceneUpdateInput {
ID: strconv.Itoa(scene.ID),
Title: &title,
Details: &scene.Details,
URL: &scene.URL,
Urls: scene.URLs.List(),
Date: dateToStringPtr(scene.Date),
}
}

14
pkg/scraper/stashbox/stash_box.go
View file

@ -684,6 +684,7 @@ func getFingerprints(scene *graphql.SceneFragment) []*models.StashBoxFingerprint
func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.SceneFragment) (*scraper.ScrapedScene, error) {
stashID := s.ID
ss := &scraper.ScrapedScene{
Title: s.Title,
Code: s.Code,
@ -698,6 +699,14 @@ func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.Scen
// stash_id
}
for _, u := range s.Urls {
ss.URLs = append(ss.URLs, u.URL)
}
if len(ss.URLs) > 0 {
ss.URL = &ss.URLs[0]
}
if len(s.Images) > 0 {
// TODO - #454 code sorts images by aspect ratio according to a wanted
// orientation. I'm just grabbing the first for now
@ -823,8 +832,9 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
if scene.Director != "" {
draft.Director = &scene.Director
}
if scene.URL != "" && len(strings.TrimSpace(scene.URL)) > 0 {
url := strings.TrimSpace(scene.URL)
// TODO - draft does not accept multiple URLs. Use single URL for now.
if len(scene.URLs.List()) > 0 {
url := strings.TrimSpace(scene.URLs.List()[0])
draft.URL = &url
}
if scene.Date != nil {

47
pkg/sliceutil/collections.go
View file

@ -2,6 +2,53 @@ package sliceutil
import "reflect"
// Exclude removes all instances of any value in toExclude from the vs
// slice. It returns the new or unchanged slice.
func Exclude[T comparable](vs []T, toExclude []T) []T {
var ret []T
for _, v := range vs {
if !Include(toExclude, v) {
ret = append(ret, v)
}
}
return ret
}
func Index[T comparable](vs []T, t T) int {
for i, v := range vs {
if v == t {
return i
}
}
return -1
}
func Include[T comparable](vs []T, t T) bool {
return Index(vs, t) >= 0
}
// IntAppendUnique appends toAdd to the vs int slice if toAdd does not already
// exist in the slice. It returns the new or unchanged int slice.
func AppendUnique[T comparable](vs []T, toAdd T) []T {
if Include(vs, toAdd) {
return vs
}
return append(vs, toAdd)
}
// IntAppendUniques appends a slice of values to the vs slice. It only
// appends values that do not already exist in the slice. It returns the new or
// unchanged slice.
func AppendUniques[T comparable](vs []T, toAdd []T) []T {
for _, v := range toAdd {
vs = AppendUnique(vs, v)
}
return vs
}
// SliceSame returns true if the two provided lists have the same elements,
// regardless of order. Panics if either parameter is not a slice.
func SliceSame(a, b interface{}) bool {

70
pkg/sqlite/anonymise.go
View file

@ -230,7 +230,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
table.Col(idColumn),
table.Col("title"),
table.Col("details"),
table.Col("url"),
table.Col("code"),
table.Col("director"),
).Where(table.Col(idColumn).Gt(lastID)).Limit(1000)
@ -243,7 +242,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
id int
title sql.NullString
details sql.NullString
url sql.NullString
code sql.NullString
director sql.NullString
)
@ -252,7 +250,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
&id,
&title,
&details,
&url,
&code,
&director,
); err != nil {
@ -264,7 +261,6 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
// if title set set new title
db.obfuscateNullString(set, "title", title)
db.obfuscateNullString(set, "details", details)
db.obfuscateNullString(set, "url", url)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(table.Col(idColumn).Eq(id))
@ -301,6 +297,10 @@ func (db *Anonymiser) anonymiseScenes(ctx context.Context) error {
}
}
if err := db.anonymiseURLs(ctx, goqu.T(scenesURLsTable), "scene_id"); err != nil {
return err
}
return nil
}
@ -704,6 +704,68 @@ func (db *Anonymiser) anonymiseAliases(ctx context.Context, table exp.Identifier
return nil
}
func (db *Anonymiser) anonymiseURLs(ctx context.Context, table exp.IdentifierExpression, idColumn string) error {
lastID := 0
lastURL := ""
total := 0
const logEvery = 10000
for gotSome := true; gotSome; {
if err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
query := dialect.From(table).Select(
table.Col(idColumn),
table.Col("url"),
).Where(goqu.L("(" + idColumn + ", url)").Gt(goqu.L("(?, ?)", lastID, lastURL))).Limit(1000)
gotSome = false
const single = false
return queryFunc(ctx, query, single, func(rows *sqlx.Rows) error {
var (
id int
url sql.NullString
)
if err := rows.Scan(
&id,
&url,
); err != nil {
return err
}
set := goqu.Record{}
db.obfuscateNullString(set, "url", url)
if len(set) > 0 {
stmt := dialect.Update(table).Set(set).Where(
table.Col(idColumn).Eq(id),
table.Col("url").Eq(url),
)
if _, err := exec(ctx, stmt); err != nil {
return fmt.Errorf("anonymising %s: %w", table.GetTable(), err)
}
}
lastID = id
lastURL = url.String
gotSome = true
total++
if total%logEvery == 0 {
logger.Infof("Anonymised %d %s URLs", total, table.GetTable())
}
return nil
})
}); err != nil {
return err
}
}
return nil
}
func (db *Anonymiser) anonymiseTags(ctx context.Context) error {
logger.Infof("Anonymising tags")
table := tagTableMgr.table

2
pkg/sqlite/database.go
View file

@ -32,7 +32,7 @@ const (
dbConnTimeout = 30
)
var appSchemaVersion uint = 46
var appSchemaVersion uint = 47
//go:embed migrations/*.sql
var migrationsBox embed.FS

94
pkg/sqlite/migrations/47_scene_urls.up.sql Normal file
View file

@ -0,0 +1,94 @@
PRAGMA foreign_keys=OFF;
CREATE TABLE `scene_urls` (
`scene_id` integer NOT NULL,
`position` integer NOT NULL,
`url` varchar(255) NOT NULL,
foreign key(`scene_id`) references `scenes`(`id`) on delete CASCADE,
PRIMARY KEY(`scene_id`, `position`, `url`)
);
CREATE INDEX `scene_urls_url` on `scene_urls` (`url`);
-- drop url
CREATE TABLE "scenes_new" (
`id` integer not null primary key autoincrement,
`title` varchar(255),
`details` text,
`date` date,
`rating` tinyint,
`studio_id` integer,
`o_counter` tinyint not null default 0,
`organized` boolean not null default '0',
`created_at` datetime not null,
`updated_at` datetime not null,
`code` text,
`director` text,
`resume_time` float not null default 0,
`last_played_at` datetime default null,
`play_count` tinyint not null default 0,
`play_duration` float not null default 0,
`cover_blob` varchar(255) REFERENCES `blobs`(`checksum`),
foreign key(`studio_id`) references `studios`(`id`) on delete SET NULL
);
INSERT INTO `scenes_new`
(
`id`,
`title`,
`details`,
`date`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`code`,
`director`,
`resume_time`,
`last_played_at`,
`play_count`,
`play_duration`,
`cover_blob`
)
SELECT
`id`,
`title`,
`details`,
`date`,
`rating`,
`studio_id`,
`o_counter`,
`organized`,
`created_at`,
`updated_at`,
`code`,
`director`,
`resume_time`,
`last_played_at`,
`play_count`,
`play_duration`,
`cover_blob`
FROM `scenes`;
INSERT INTO `scene_urls`
(
`scene_id`,
`position`,
`url`
)
SELECT
`id`,
'0',
`url`
FROM `scenes`
WHERE `scenes`.`url` IS NOT NULL AND `scenes`.`url` != '';
DROP INDEX `index_scenes_on_studio_id`;
DROP TABLE `scenes`;
ALTER TABLE `scenes_new` rename to `scenes`;
CREATE INDEX `index_scenes_on_studio_id` on `scenes` (`studio_id`);
PRAGMA foreign_keys=ON;

42
pkg/sqlite/scene.go
View file

@ -31,6 +31,8 @@ const (
scenesTagsTable = "scenes_tags"
scenesGalleriesTable = "scenes_galleries"
moviesScenesTable = "movies_scenes"
scenesURLsTable = "scene_urls"
sceneURLColumn = "url"
sceneCoverBlobColumn = "cover_blob"
)
@ -76,7 +78,6 @@ type sceneRow struct {
Code zero.String `db:"code"`
Details zero.String `db:"details"`
Director zero.String `db:"director"`
URL zero.String `db:"url"`
Date NullDate `db:"date"`
// expressed as 1-100
Rating null.Int `db:"rating"`
@ -100,7 +101,6 @@ func (r *sceneRow) fromScene(o models.Scene) {
r.Code = zero.StringFrom(o.Code)
r.Details = zero.StringFrom(o.Details)
r.Director = zero.StringFrom(o.Director)
r.URL = zero.StringFrom(o.URL)
r.Date = NullDateFromDatePtr(o.Date)
r.Rating = intFromPtr(o.Rating)
r.Organized = o.Organized
@ -130,7 +130,6 @@ func (r *sceneQueryRow) resolve() *models.Scene {
Code: r.Code.String,
Details: r.Details.String,
Director: r.Director.String,
URL: r.URL.String,
Date: r.Date.DatePtr(),
Rating: nullIntPtr(r.Rating),
Organized: r.Organized,
@ -166,7 +165,6 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) {
r.setNullString("code", o.Code)
r.setNullString("details", o.Details)
r.setNullString("director", o.Director)
r.setNullString("url", o.URL)
r.setNullDate("date", o.Date)
r.setNullInt("rating", o.Rating)
r.setBool("organized", o.Organized)
@ -268,6 +266,13 @@ func (qb *SceneStore) Create(ctx context.Context, newObject *models.Scene, fileI
}
}
if newObject.URLs.Loaded() {
const startPos = 0
if err := scenesURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil {
return err
}
}
if newObject.PerformerIDs.Loaded() {
if err := scenesPerformersTableMgr.insertJoins(ctx, id, newObject.PerformerIDs.List()); err != nil {
return err
@ -322,6 +327,11 @@ func (qb *SceneStore) UpdatePartial(ctx context.Context, id int, partial models.
}
}
if partial.URLs != nil {
if err := scenesURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil {
return nil, err
}
}
if partial.PerformerIDs != nil {
if err := scenesPerformersTableMgr.modifyJoins(ctx, id, partial.PerformerIDs.IDs, partial.PerformerIDs.Mode); err != nil {
return nil, err
@ -364,6 +374,12 @@ func (qb *SceneStore) Update(ctx context.Context, updatedObject *models.Scene) e
return err
}
if updatedObject.URLs.Loaded() {
if err := scenesURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil {
return err
}
}
if updatedObject.PerformerIDs.Loaded() {
if err := scenesPerformersTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.PerformerIDs.List()); err != nil {
return err
@ -974,7 +990,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
query.handleCriterion(ctx, hasMarkersCriterionHandler(sceneFilter.HasMarkers))
query.handleCriterion(ctx, sceneIsMissingCriterionHandler(qb, sceneFilter.IsMissing))
query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.URL, "scenes.url"))
query.handleCriterion(ctx, sceneURLsCriterionHandler(sceneFilter.URL))
query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
if sceneFilter.StashID != nil {
@ -1308,6 +1324,18 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion
}
}
func sceneURLsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc {
h := stringListCriterionHandlerBuilder{
joinTable: scenesURLsTable,
stringColumn: sceneURLColumn,
addJoinTable: func(f *filterBuilder) {
scenesURLsTableMgr.join(f, "", "scenes.id")
},
}
return h.handler(url)
}
func (qb *SceneStore) getMultiCriterionHandlerBuilder(foreignTable, joinTable, foreignFK string, addJoinsFunc func(f *filterBuilder)) multiCriterionHandlerBuilder {
return multiCriterionHandlerBuilder{
primaryTable: sceneTable,
@ -1637,6 +1665,10 @@ func (qb *SceneStore) IncrementWatchCount(ctx context.Context, id int) (int, err
return qb.getPlayCount(ctx, id)
}
func (qb *SceneStore) GetURLs(ctx context.Context, sceneID int) ([]string, error) {
return scenesURLsTableMgr.get(ctx, sceneID)
}
func (qb *SceneStore) GetCover(ctx context.Context, sceneID int) ([]byte, error) {
return qb.GetImage(ctx, sceneID, sceneCoverBlobColumn)
}

36
pkg/sqlite/scene_test.go
View file

@ -21,6 +21,12 @@ import (
)
func loadSceneRelationships(ctx context.Context, expected models.Scene, actual *models.Scene) error {
if expected.URLs.Loaded() {
if err := actual.LoadURLs(ctx, db.Scene); err != nil {
return err
}
}
if expected.GalleryIDs.Loaded() {
if err := actual.LoadGalleryIDs(ctx, db.Scene); err != nil {
return err
@ -108,7 +114,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@ -153,7 +159,7 @@ func Test_sceneQueryBuilder_Create(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@ -346,7 +352,7 @@ func Test_sceneQueryBuilder_Update(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@ -513,7 +519,7 @@ func clearScenePartial() models.ScenePartial {
Code: models.OptionalString{Set: true, Null: true},
Details: models.OptionalString{Set: true, Null: true},
Director: models.OptionalString{Set: true, Null: true},
URL: models.OptionalString{Set: true, Null: true},
URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet},
Date: models.OptionalDate{Set: true, Null: true},
Rating: models.OptionalInt{Set: true, Null: true},
StudioID: models.OptionalInt{Set: true, Null: true},
@ -564,7 +570,10 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
Code: models.NewOptionalString(code),
Details: models.NewOptionalString(details),
Director: models.NewOptionalString(director),
URL: models.NewOptionalString(url),
URLs: &models.UpdateStrings{
Values: []string{url},
Mode: models.RelationshipUpdateModeSet,
},
Date: models.NewOptionalDate(date),
Rating: models.NewOptionalInt(rating),
Organized: models.NewOptionalBool(true),
@ -624,7 +633,7 @@ func Test_sceneQueryBuilder_UpdatePartial(t *testing.T) {
Code: code,
Details: details,
Director: director,
URL: url,
URLs: models.NewRelatedStrings([]string{url}),
Date: &date,
Rating: &rating,
Organized: true,
@ -2400,7 +2409,14 @@ func TestSceneQueryURL(t *testing.T) {
verifyFn := func(s *models.Scene) {
t.Helper()
verifyString(t, s.URL, urlCriterion)
urls := s.URLs.List()
var url string
if len(urls) > 0 {
url = urls[0]
}
verifyString(t, url, urlCriterion)
}
verifySceneQuery(t, filter, verifyFn)
@ -2576,6 +2592,12 @@ func verifySceneQuery(t *testing.T, filter models.SceneFilterType, verifyFn func
scenes := queryScene(ctx, t, sqb, &filter, nil)
for _, scene := range scenes {
if err := scene.LoadRelationships(ctx, sqb); err != nil {
t.Errorf("Error loading scene relationships: %v", err)
}
}
// assume it should find at least one
assert.Greater(t, len(scenes), 0)

4
pkg/sqlite/setup_test.go
View file

@ -1067,7 +1067,9 @@ func makeScene(i int) *models.Scene {
return &models.Scene{
Title: title,
Details: details,
URL: getSceneEmptyString(i, urlField),
URLs: models.NewRelatedStrings([]string{
getSceneEmptyString(i, urlField),
}),
Rating: getIntPtr(rating),
OCounter: getOCounter(i),
Date: getObjectDate(i),

108
pkg/sqlite/table.go
View file

@ -14,6 +14,7 @@ import (
"github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
)
@ -472,6 +473,113 @@ func (t *stringTable) modifyJoins(ctx context.Context, id int, v []string, mode
return nil
}
type orderedValueTable[T comparable] struct {
table
valueColumn exp.IdentifierExpression
}
func (t *orderedValueTable[T]) positionColumn() exp.IdentifierExpression {
const positionColumn = "position"
return t.table.table.Col(positionColumn)
}
func (t *orderedValueTable[T]) get(ctx context.Context, id int) ([]T, error) {
q := dialect.Select(t.valueColumn).From(t.table.table).Where(t.idColumn.Eq(id)).Order(t.positionColumn().Asc())
const single = false
var ret []T
if err := queryFunc(ctx, q, single, func(rows *sqlx.Rows) error {
var v T
if err := rows.Scan(&v); err != nil {
return err
}
ret = append(ret, v)
return nil
}); err != nil {
return nil, fmt.Errorf("getting stash ids from %s: %w", t.table.table.GetTable(), err)
}
return ret, nil
}
func (t *orderedValueTable[T]) insertJoin(ctx context.Context, id int, position int, v T) (sql.Result, error) {
q := dialect.Insert(t.table.table).Cols(t.idColumn.GetCol(), t.positionColumn().GetCol(), t.valueColumn.GetCol()).Vals(
goqu.Vals{id, position, v},
)
ret, err := exec(ctx, q)
if err != nil {
return nil, fmt.Errorf("inserting into %s: %w", t.table.table.GetTable(), err)
}
return ret, nil
}
func (t *orderedValueTable[T]) insertJoins(ctx context.Context, id int, startPos int, v []T) error {
for i, fk := range v {
if _, err := t.insertJoin(ctx, id, i+startPos, fk); err != nil {
return err
}
}
return nil
}
func (t *orderedValueTable[T]) replaceJoins(ctx context.Context, id int, v []T) error {
if err := t.destroy(ctx, []int{id}); err != nil {
return err
}
const startPos = 0
return t.insertJoins(ctx, id, startPos, v)
}
func (t *orderedValueTable[T]) addJoins(ctx context.Context, id int, v []T) error {
// get existing foreign keys
existing, err := t.get(ctx, id)
if err != nil {
return err
}
// only add values that are not already present
filtered := sliceutil.Exclude(v, existing)
if len(filtered) == 0 {
return nil
}
startPos := len(existing)
return t.insertJoins(ctx, id, startPos, filtered)
}
func (t *orderedValueTable[T]) destroyJoins(ctx context.Context, id int, v []T) error {
existing, err := t.get(ctx, id)
if err != nil {
return fmt.Errorf("getting existing %s: %w", t.table.table.GetTable(), err)
}
newValue := sliceutil.Exclude(existing, v)
if len(newValue) == len(existing) {
return nil
}
return t.replaceJoins(ctx, id, newValue)
}
func (t *orderedValueTable[T]) modifyJoins(ctx context.Context, id int, v []T, mode models.RelationshipUpdateMode) error {
switch mode {
case models.RelationshipUpdateModeSet:
return t.replaceJoins(ctx, id, v)
case models.RelationshipUpdateModeAdd:
return t.addJoins(ctx, id, v)
case models.RelationshipUpdateModeRemove:
return t.destroyJoins(ctx, id, v)
}
return nil
}
type scenesMoviesTable struct {
table
}

9
pkg/sqlite/tables.go
View file

@ -24,6 +24,7 @@ var (
scenesPerformersJoinTable = goqu.T(performersScenesTable)
scenesStashIDsJoinTable = goqu.T("scene_stash_ids")
scenesMoviesJoinTable = goqu.T(moviesScenesTable)
scenesURLsJoinTable = goqu.T(scenesURLsTable)
performersAliasesJoinTable = goqu.T(performersAliasesTable)
performersTagsJoinTable = goqu.T(performersTagsTable)
@ -160,6 +161,14 @@ var (
idColumn: scenesMoviesJoinTable.Col(sceneIDColumn),
},
}
scenesURLsTableMgr = &orderedValueTable[string]{
table: table{
table: scenesURLsJoinTable,
idColumn: scenesURLsJoinTable.Col(sceneIDColumn),
},
valueColumn: scenesURLsJoinTable.Col(sceneURLColumn),
}
)
var (

58
ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx
View file

@ -29,7 +29,7 @@ import {
import { Icon } from "src/components/Shared/Icon";
import { LoadingIndicator } from "src/components/Shared/LoadingIndicator";
import { ImageInput } from "src/components/Shared/ImageInput";
import { URLField } from "src/components/Shared/URLField";
import { URLListInput } from "src/components/Shared/URLField";
import { useToast } from "src/hooks/Toast";
import ImageUtils from "src/utils/image";
import FormUtils from "src/utils/form";
@ -106,7 +106,25 @@ export const SceneEditPanel: React.FC<IProps> = ({
const schema = yup.object({
title: yup.string().ensure(),
code: yup.string().ensure(),
url: yup.string().ensure(),
urls: yup
.array(yup.string().required())
.defined()
.test({
name: "unique",
test: (value) => {
const dupes = value
.map((e, i, a) => {
if (a.indexOf(e) !== i) {
return String(i - 1);
} else {
return null;
}
})
.filter((e) => e !== null) as string[];
if (dupes.length === 0) return true;
return new yup.ValidationError(dupes.join(" "), value, "urls");
},
}),
date: yup
.string()
.ensure()
@ -143,7 +161,7 @@ export const SceneEditPanel: React.FC<IProps> = ({
() => ({
title: scene.title ?? "",
code: scene.code ?? "",
url: scene.url ?? "",
urls: scene.urls ?? [],
date: scene.date ?? "",
director: scene.director ?? "",
rating100: scene.rating100 ?? null,
@ -333,7 +351,7 @@ export const SceneEditPanel: React.FC<IProps> = ({
director: fragment.director,
remote_site_id: fragment.remote_site_id,
title: fragment.title,
url: fragment.url,
urls: fragment.urls,
};
const result = await queryScrapeSceneQueryFragment(s, input);
@ -549,8 +567,8 @@ export const SceneEditPanel: React.FC<IProps> = ({
formik.setFieldValue("date", updatedScene.date);
}
if (updatedScene.url) {
formik.setFieldValue("url", updatedScene.url);
if (updatedScene.urls) {
formik.setFieldValue("urls", updatedScene.urls);
}
if (updatedScene.studio && updatedScene.studio.stored_id) {
@ -624,13 +642,13 @@ export const SceneEditPanel: React.FC<IProps> = ({
}
}
async function onScrapeSceneURL() {
if (!formik.values.url) {
async function onScrapeSceneURL(url: string) {
if (!url) {
return;
}
setIsLoading(true);
try {
const result = await queryScrapeSceneURL(formik.values.url);
const result = await queryScrapeSceneURL(url);
if (!result.data || !result.data.scrapeSceneURL) {
return;
}
@ -683,6 +701,14 @@ export const SceneEditPanel: React.FC<IProps> = ({
if (isLoading) return <LoadingIndicator />;
const urlsErrors = Array.isArray(formik.errors.urls)
? formik.errors.urls[0]
: formik.errors.urls;
const urlsErrorMsg = urlsErrors
? intl.formatMessage({ id: "validation.urls_must_be_unique" })
: undefined;
const urlsErrorIdx = urlsErrors?.split(" ").map((e) => parseInt(e));
return (
<div id="scene-edit-details">
<Prompt
@ -728,18 +754,20 @@ export const SceneEditPanel: React.FC<IProps> = ({
<div className="col-12 col-lg-7 col-xl-12">
{renderTextField("title", intl.formatMessage({ id: "title" }))}
{renderTextField("code", intl.formatMessage({ id: "scene_code" }))}
<Form.Group controlId="url" as={Row}>
<Form.Group controlId="urls" as={Row}>
<Col xs={3} className="pr-0 url-label">
<Form.Label className="col-form-label">
<FormattedMessage id="url" />
<FormattedMessage id="urls" />
</Form.Label>
</Col>
<Col xs={9}>
<URLField
{...formik.getFieldProps("url")}
onScrapeClick={onScrapeSceneURL}
<URLListInput
value={formik.values.urls ?? []}
setValue={(value) => formik.setFieldValue("urls", value)}
errors={urlsErrorMsg}
errorIdx={urlsErrorIdx}
onScrapeClick={(url) => onScrapeSceneURL(url)}
urlScrapable={urlScrapable}
isInvalid={!!formik.getFieldMeta("url").error}
/>
</Col>
</Form.Group>

9
ui/v2.5/src/components/Scenes/SceneDetails/SceneFileInfoPanel.tsx
View file

@ -16,7 +16,7 @@ import { useToast } from "src/hooks/Toast";
import NavUtils from "src/utils/navigation";
import TextUtils from "src/utils/text";
import { getStashboxBase } from "src/utils/stashbox";
import { TextField, URLField } from "src/utils/field";
import { TextField, URLField, URLsField } from "src/utils/field";
interface IFileInfoPanelProps {
sceneID: string;
@ -316,12 +316,7 @@ export const SceneFileInfoPanel: React.FC<ISceneFileInfoPanelProps> = (
)}
{renderFunscript()}
{renderInteractiveSpeed()}
<URLField
id="media_info.downloaded_from"
url={props.scene.url}
value={props.scene.url}
truncate
/>
<URLsField id="urls" urls={props.scene.urls} truncate />
{renderStashIDs()}
<TextField
id="media_info.play_count"

25
ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx
View file

@ -14,6 +14,7 @@ import {
ScrapedTextAreaRow,
ScrapedImageRow,
IHasName,
ScrapedStringListRow,
} from "src/components/Shared/ScrapeDialog";
import clone from "lodash-es/clone";
import {
@ -26,6 +27,7 @@ import {
import { useToast } from "src/hooks/Toast";
import DurationUtils from "src/utils/duration";
import { useIntl } from "react-intl";
import { uniq } from "lodash-es";
interface IScrapedStudioRow {
title: string;
@ -295,9 +297,16 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
const [code, setCode] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(scene.code, scraped.code)
);
const [url, setURL] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(scene.url, scraped.url)
const [urls, setURLs] = useState<ScrapeResult<string[]>>(
new ScrapeResult<string[]>(
scene.urls,
scraped.urls
? uniq((scene.urls ?? []).concat(scraped.urls ?? []))
: undefined
)
);
const [date, setDate] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(scene.date, scraped.date)
);
@ -407,7 +416,7 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
[
title,
code,
url,
urls,
date,
director,
studio,
@ -581,7 +590,7 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
return {
title: title.getNewValue(),
code: code.getNewValue(),
url: url.getNewValue(),
urls: urls.getNewValue(),
date: date.getNewValue(),
director: director.getNewValue(),
studio: newStudioValue
@ -627,10 +636,10 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
result={code}
onChange={(value) => setCode(value)}
/>
<ScrapedInputGroupRow
title={intl.formatMessage({ id: "url" })}
result={url}
onChange={(value) => setURL(value)}
<ScrapedStringListRow
title={intl.formatMessage({ id: "urls" })}
result={urls}
onChange={(value) => setURLs(value)}
/>
<ScrapedInputGroupRow
title={intl.formatMessage({ id: "date" })}

13
ui/v2.5/src/components/Scenes/SceneMergeDialog.tsx
View file

@ -17,6 +17,7 @@ import {
ScrapeDialogRow,
ScrapedImageRow,
ScrapedInputGroupRow,
ScrapedStringListRow,
ScrapedTextAreaRow,
ScrapeResult,
ZeroableScrapeResult,
@ -61,8 +62,8 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
const [code, setCode] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(dest.code)
);
const [url, setURL] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(dest.url)
const [url, setURL] = useState<ScrapeResult<string[]>>(
new ScrapeResult<string[]>(dest.urls)
);
const [date, setDate] = useState<ScrapeResult<string>>(
new ScrapeResult<string>(dest.date)
@ -164,7 +165,7 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
new ScrapeResult(dest.code, sources.find((s) => s.code)?.code, !dest.code)
);
setURL(
new ScrapeResult(dest.url, sources.find((s) => s.url)?.url, !dest.url)
new ScrapeResult(dest.urls, sources.find((s) => s.urls)?.urls, !dest.urls)
);
setDate(
new ScrapeResult(dest.date, sources.find((s) => s.date)?.date, !dest.date)
@ -361,8 +362,8 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
result={code}
onChange={(value) => setCode(value)}
/>
<ScrapedInputGroupRow
title={intl.formatMessage({ id: "url" })}
<ScrapedStringListRow
title={intl.formatMessage({ id: "urls" })}
result={url}
onChange={(value) => setURL(value)}
/>
@ -546,7 +547,7 @@ const SceneMergeDetails: React.FC<ISceneMergeDetailsProps> = ({
id: dest.id,
title: title.getNewValue(),
code: code.getNewValue(),
url: url.getNewValue(),
urls: url.getNewValue(),
date: date.getNewValue(),
rating100: rating.getNewValue(),
o_counter: oCounter.getNewValue(),

69
ui/v2.5/src/components/Shared/ScrapeDialog.tsx
View file

@ -22,6 +22,7 @@ import {
} from "@fortawesome/free-solid-svg-icons";
import { getCountryByISO } from "src/utils/country";
import { CountrySelect } from "./CountrySelect";
import { StringListInput } from "./StringListInput";
export class ScrapeResult<T> {
public newValue?: T;
@ -102,6 +103,7 @@ interface IScrapedFieldProps<T> {
interface IScrapedRowProps<T, V extends IHasName>
extends IScrapedFieldProps<T> {
className?: string;
title: string;
renderOriginalField: (result: ScrapeResult<T>) => JSX.Element | undefined;
renderNewField: (result: ScrapeResult<T>) => JSX.Element | undefined;
@ -175,7 +177,7 @@ export const ScrapeDialogRow = <T, V extends IHasName>(
}
return (
<Row className="px-3 pt-3">
<Row className={`px-3 pt-3 ${props.className ?? ""}`}>
<Form.Label column lg="3">
{props.title}
</Form.Label>
@ -276,6 +278,71 @@ export const ScrapedInputGroupRow: React.FC<IScrapedInputGroupRowProps> = (
);
};
interface IScrapedStringListProps {
isNew?: boolean;
placeholder?: string;
locked?: boolean;
result: ScrapeResult<string[]>;
onChange?: (value: string[]) => void;
}
const ScrapedStringList: React.FC<IScrapedStringListProps> = (props) => {
const value = props.isNew
? props.result.newValue
: props.result.originalValue;
return (
<StringListInput
value={value ?? []}
setValue={(v) => {
if (props.isNew && props.onChange) {
props.onChange(v);
}
}}
placeholder={props.placeholder}
readOnly={!props.isNew || props.locked}
/>
);
};
interface IScrapedStringListRowProps {
title: string;
placeholder?: string;
result: ScrapeResult<string[]>;
locked?: boolean;
onChange: (value: ScrapeResult<string[]>) => void;
}
export const ScrapedStringListRow: React.FC<IScrapedStringListRowProps> = (
props
) => {
return (
<ScrapeDialogRow
className="string-list-row"
title={props.title}
result={props.result}
renderOriginalField={() => (
<ScrapedStringList
placeholder={props.placeholder || props.title}
result={props.result}
/>
)}
renderNewField={() => (
<ScrapedStringList
placeholder={props.placeholder || props.title}
result={props.result}
isNew
locked={props.locked}
onChange={(value) =>
props.onChange(props.result.cloneWithValue(value))
}
/>
)}
onChange={props.onChange}
/>
);
};
const ScrapedTextArea: React.FC<IScrapedInputGroupProps> = (props) => {
return (
<FormControl

55
ui/v2.5/src/components/Shared/StringListInput.tsx
View file

@ -1,18 +1,55 @@
import { faMinus } from "@fortawesome/free-solid-svg-icons";
import React from "react";
import React, { ComponentType } from "react";
import { Button, Form, InputGroup } from "react-bootstrap";
import { Icon } from "./Icon";
interface IStringListInputProps {
interface IListInputComponentProps {
value: string;
setValue: (value: string) => void;
placeholder?: string;
className?: string;
readOnly?: boolean;
}
interface IListInputAppendProps {
value: string;
}
export interface IStringListInputProps {
value: string[];
setValue: (value: string[]) => void;
inputComponent?: ComponentType<IListInputComponentProps>;
appendComponent?: ComponentType<IListInputAppendProps>;
placeholder?: string;
className?: string;
errors?: string;
errorIdx?: number[];
readOnly?: boolean;
}
export const StringInput: React.FC<IListInputComponentProps> = ({
className,
placeholder,
value,
setValue,
readOnly = false,
}) => {
return (
<Form.Control
className={`text-input ${className ?? ""}`}
value={value}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setValue(e.currentTarget.value)
}
placeholder={placeholder}
readOnly={readOnly}
/>
);
};
export const StringListInput: React.FC<IStringListInputProps> = (props) => {
const Input = props.inputComponent ?? StringInput;
const AppendComponent = props.appendComponent;
const values = props.value.concat("");
function valueChanged(idx: number, value: string) {
@ -37,17 +74,16 @@ export const StringListInput: React.FC<IStringListInputProps> = (props) => {
<Form.Group>
{values.map((v, i) => (
<InputGroup className={props.className} key={i}>
<Form.Control
className={`text-input ${
props.errorIdx?.includes(i) ? "is-invalid" : ""
}`}
<Input
value={v}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
valueChanged(i, e.currentTarget.value)
}
setValue={(value) => valueChanged(i, value)}
placeholder={props.placeholder}
className={props.errorIdx?.includes(i) ? "is-invalid" : ""}
readOnly={props.readOnly}
/>
<InputGroup.Append>
{AppendComponent && <AppendComponent value={v} />}
{!props.readOnly && (
<Button
variant="danger"
onClick={() => removeValue(i)}
@ -55,6 +91,7 @@ export const StringListInput: React.FC<IStringListInputProps> = (props) => {
>
<Icon icon={faMinus} />
</Button>
)}
</InputGroup.Append>
</InputGroup>
))}

35
ui/v2.5/src/components/Shared/URLField.tsx
View file

@ -4,6 +4,11 @@ import { Button, InputGroup, Form } from "react-bootstrap";
import { Icon } from "./Icon";
import { FormikHandlers } from "formik";
import { faFileDownload } from "@fortawesome/free-solid-svg-icons";
import {
IStringListInputProps,
StringInput,
StringListInput,
} from "./StringListInput";
interface IProps {
value: string;
@ -43,3 +48,33 @@ export const URLField: React.FC<IProps> = (props: IProps) => {
</InputGroup>
);
};
interface IURLListProps extends IStringListInputProps {
onScrapeClick(url: string): void;
urlScrapable(url: string): boolean;
}
export const URLListInput: React.FC<IURLListProps> = (
listProps: IURLListProps
) => {
const intl = useIntl();
const { onScrapeClick, urlScrapable } = listProps;
return (
<StringListInput
{...listProps}
placeholder={intl.formatMessage({ id: "url" })}
inputComponent={StringInput}
appendComponent={(props) => (
<Button
className="scrape-url-button text-input"
variant="secondary"
onClick={() => onScrapeClick(props.value)}
disabled={!props.value || !urlScrapable(props.value)}
title={intl.formatMessage({ id: "actions.scrape" })}
>
<Icon icon={faFileDownload} />
</Button>
)}
/>
);
};

4
ui/v2.5/src/components/Shared/styles.scss
View file

@ -441,3 +441,7 @@ div.react-datepicker {
right: 0;
z-index: 4;
}
.string-list-row .input-group {
flex-wrap: nowrap;
}

2
ui/v2.5/src/components/Tagger/context.tsx
View file

@ -408,7 +408,7 @@ export const TaggerContext: React.FC = ({ children }) => {
details: scene.details,
remote_site_id: scene.remote_site_id,
title: scene.title,
url: scene.url,
urls: scene.urls,
};
const result = await queryScrapeSceneQueryFragment(

18
ui/v2.5/src/components/Tagger/scenes/StashSearchResult.tsx
View file

@ -360,7 +360,7 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
),
studio_id: studioID,
cover_image: resolveField("cover_image", undefined, imgData),
url: resolveField("url", stashScene.url, scene.url),
urls: resolveField("url", stashScene.urls, scene.urls),
tag_ids: tagIDs,
stash_ids: stashScene.stash_ids ?? [],
code: resolveField("code", stashScene.code, scene.code),
@ -462,9 +462,11 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
);
}
const sceneTitleEl = scene.url ? (
const url = scene.urls?.length ? scene.urls[0] : null;
const sceneTitleEl = url ? (
<a
href={scene.url}
href={url}
target="_blank"
rel="noopener noreferrer"
className="scene-link"
@ -558,16 +560,20 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
};
const maybeRenderURL = () => {
if (scene.url) {
if (scene.urls) {
return (
<div className="scene-details">
<OptionalField
exclude={excludedFields[fields.url]}
setExclude={(v) => setExcludedField(fields.url, v)}
>
<a href={scene.url} target="_blank" rel="noopener noreferrer">
{scene.url}
{scene.urls.map((url) => (
<div key={url}>
<a href={url} target="_blank" rel="noopener noreferrer">
{url}
</a>
</div>
))}
</OptionalField>
</div>
);

4
ui/v2.5/src/locales/en-GB.json
View file

@ -1270,10 +1270,12 @@
"type": "Type",
"updated_at": "Updated At",
"url": "URL",
"urls": "URLs",
"validation": {
"aliases_must_be_unique": "aliases must be unique",
"date_invalid_form": "${path} must be in YYYY-MM-DD form",
"required": "${path} is a required field"
"required": "${path} is a required field",
"urls_must_be_unique": "URLs must be unique"
},
"videos": "Videos",
"video_codec": "Video Codec",

47
ui/v2.5/src/utils/field.tsx
View file

@ -90,3 +90,50 @@ export const URLField: React.FC<IURLField> = ({
</>
);
};
interface IURLsField {
id?: string;
name?: string;
abbr?: string | null;
urls?: string[] | null;
truncate?: boolean | null;
target?: string;
// use for internal links
trusted?: boolean;
}
export const URLsField: React.FC<IURLsField> = ({
id,
name,
urls,
abbr,
truncate,
target,
trusted,
}) => {
const values = urls ?? [];
if (!values.length) {
return null;
}
const message = (
<>{id ? <FormattedMessage id={id} defaultMessage={name} /> : name}:</>
);
const rel = !trusted ? "noopener noreferrer" : undefined;
return (
<>
<dt>{abbr ? <abbr title={abbr}>{message}</abbr> : message}</dt>
<dd>
<dl>
{values.map((url, i) => (
<a key={i} href={url} target={target || "_blank"} rel={rel}>
{truncate ? <TruncatedText text={url} /> : url}
</a>
))}
</dl>
</dd>
</>
);
};