From e51083c26d7151600f870c41faadac08d51877b1 Mon Sep 17 00:00:00 2001 From: InfiniteTF Date: Wed, 1 Jun 2022 04:59:06 +0200 Subject: [PATCH] Update stash-box fingerprint query to fully support distance matching (#2509) --- graphql/schema/schema.graphql | 6 - graphql/stash-box/query.graphql | 6 + internal/api/resolver_query_scraper.go | 63 +- internal/manager/task_identify.go | 3 +- .../stashbox/graphql/generated_client.go | 547 +++++++++++------- .../stashbox/graphql/generated_models.go | 116 ++-- pkg/scraper/stashbox/stash_box.go | 169 ++---- 7 files changed, 466 insertions(+), 444 deletions(-) diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 9b5bf6ed7..b81168a9a 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -114,12 +114,6 @@ type Query { """Scrape a list of performers from a query""" scrapeFreeonesPerformerList(query: String!): [String!]! @deprecated(reason: "use scrapeSinglePerformer with scraper_id = builtin_freeones") - """Query StashBox for scenes""" - queryStashBoxScene(input: StashBoxSceneQueryInput!): [ScrapedScene!]! @deprecated(reason: "use scrapeSingleScene or scrapeMultiScenes") - """Query StashBox for performers""" - queryStashBoxPerformer(input: StashBoxPerformerQueryInput!): [StashBoxPerformerQueryResult!]! @deprecated(reason: "use scrapeSinglePerformer or scrapeMultiPerformers") - # === end deprecated methods === - # Plugins """List loaded plugins""" plugins: [Plugin!] diff --git a/graphql/stash-box/query.graphql b/graphql/stash-box/query.graphql index 39bce5d3c..12f12d2a5 100644 --- a/graphql/stash-box/query.graphql +++ b/graphql/stash-box/query.graphql @@ -129,6 +129,12 @@ query FindScenesByFullFingerprints($fingerprints: [FingerprintQueryInput!]!) { } } +query FindScenesBySceneFingerprints($fingerprints: [[FingerprintQueryInput!]!]!) { + findScenesBySceneFingerprints(fingerprints: $fingerprints) { + ...SceneFragment + } +} + query SearchScene($term: String!) { searchScene(term: $term) { ...SceneFragment diff --git a/internal/api/resolver_query_scraper.go b/internal/api/resolver_query_scraper.go index 2208628d5..8fd6c345a 100644 --- a/internal/api/resolver_query_scraper.go +++ b/internal/api/resolver_query_scraper.go @@ -227,46 +227,6 @@ func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models return marshalScrapedMovie(content) } -func (r *queryResolver) QueryStashBoxScene(ctx context.Context, input models.StashBoxSceneQueryInput) ([]*models.ScrapedScene, error) { - boxes := config.GetInstance().GetStashBoxes() - - if input.StashBoxIndex < 0 || input.StashBoxIndex >= len(boxes) { - return nil, fmt.Errorf("%w: invalid stash_box_index %d", ErrInput, input.StashBoxIndex) - } - - client := stashbox.NewClient(*boxes[input.StashBoxIndex], r.txnManager) - - if len(input.SceneIds) > 0 { - return client.FindStashBoxScenesByFingerprintsFlat(ctx, input.SceneIds) - } - - if input.Q != nil { - return client.QueryStashBoxScene(ctx, *input.Q) - } - - return nil, nil -} - -func (r *queryResolver) QueryStashBoxPerformer(ctx context.Context, input models.StashBoxPerformerQueryInput) ([]*models.StashBoxPerformerQueryResult, error) { - boxes := config.GetInstance().GetStashBoxes() - - if input.StashBoxIndex < 0 || input.StashBoxIndex >= len(boxes) { - return nil, fmt.Errorf("%w: invalid stash_box_index %d", ErrInput, input.StashBoxIndex) - } - - client := stashbox.NewClient(*boxes[input.StashBoxIndex], r.txnManager) - - if len(input.PerformerIds) > 0 { - return client.FindStashBoxPerformersByNames(ctx, input.PerformerIds) - } - - if input.Q != nil { - return client.QueryStashBoxPerformer(ctx, *input.Q) - } - - return nil, nil -} - func (r *queryResolver) getStashBoxClient(index int) (*stashbox.Client, error) { boxes := config.GetInstance().GetStashBoxes() @@ -280,6 +240,15 @@ func (r *queryResolver) getStashBoxClient(index int) (*stashbox.Client, error) { func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) { var ret []*models.ScrapedScene + var sceneID int + if input.SceneID != nil { + var err error + sceneID, err = strconv.Atoi(*input.SceneID) + if err != nil { + return nil, fmt.Errorf("%w: sceneID is not an integer: '%s'", ErrInput, *input.SceneID) + } + } + switch { case source.ScraperID != nil: var err error @@ -288,11 +257,6 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr switch { case input.SceneID != nil: - var sceneID int - sceneID, err = strconv.Atoi(*input.SceneID) - if err != nil { - return nil, fmt.Errorf("%w: sceneID is not an integer: '%s'", ErrInput, *input.SceneID) - } c, err = r.scraperCache().ScrapeID(ctx, *source.ScraperID, sceneID, models.ScrapeContentTypeScene) if c != nil { content = []models.ScrapedContent{c} @@ -324,7 +288,7 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr switch { case input.SceneID != nil: - ret, err = client.FindStashBoxScenesByFingerprintsFlat(ctx, []string{*input.SceneID}) + ret, err = client.FindStashBoxSceneByFingerprints(ctx, sceneID) case input.Query != nil: ret, err = client.QueryStashBoxScene(ctx, *input.Query) default: @@ -352,7 +316,12 @@ func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source models.Scr return nil, err } - return client.FindStashBoxScenesByFingerprints(ctx, input.SceneIds) + sceneIDs, err := stringslice.StringSliceToIntSlice(input.SceneIds) + if err != nil { + return nil, err + } + + return client.FindStashBoxScenesByFingerprints(ctx, sceneIDs) } return nil, errors.New("scraper_id or stash_box_index must be set") diff --git a/internal/manager/task_identify.go b/internal/manager/task_identify.go index 54bb063e4..678d0c7b3 100644 --- a/internal/manager/task_identify.go +++ b/internal/manager/task_identify.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strconv" "github.com/stashapp/stash/internal/identify" "github.com/stashapp/stash/pkg/job" @@ -212,7 +211,7 @@ type stashboxSource struct { } func (s stashboxSource) ScrapeScene(ctx context.Context, sceneID int) (*models.ScrapedScene, error) { - results, err := s.FindStashBoxScenesByFingerprintsFlat(ctx, []string{strconv.Itoa(sceneID)}) + results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID) if err != nil { return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err) } diff --git a/pkg/scraper/stashbox/graphql/generated_client.go b/pkg/scraper/stashbox/graphql/generated_client.go index 37985a20f..248f3e711 100644 --- a/pkg/scraper/stashbox/graphql/generated_client.go +++ b/pkg/scraper/stashbox/graphql/generated_client.go @@ -12,6 +12,7 @@ import ( type StashBoxGraphQLClient interface { FindSceneByFingerprint(ctx context.Context, fingerprint FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindSceneByFingerprint, error) FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error) + FindScenesBySceneFingerprints(ctx context.Context, fingerprints [][]*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesBySceneFingerprints, error) SearchScene(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchScene, error) SearchPerformer(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchPerformer, error) FindPerformerByID(ctx context.Context, id string, httpRequestOptions ...client.HTTPRequestOption) (*FindPerformerByID, error) @@ -31,32 +32,33 @@ func NewClient(cli *http.Client, baseURL string, options ...client.HTTPRequestOp } type Query struct { - FindPerformer *Performer "json:\"findPerformer\" graphql:\"findPerformer\"" - QueryPerformers QueryPerformersResultType "json:\"queryPerformers\" graphql:\"queryPerformers\"" - FindStudio *Studio "json:\"findStudio\" graphql:\"findStudio\"" - QueryStudios QueryStudiosResultType "json:\"queryStudios\" graphql:\"queryStudios\"" - FindTag *Tag "json:\"findTag\" graphql:\"findTag\"" - QueryTags QueryTagsResultType "json:\"queryTags\" graphql:\"queryTags\"" - FindTagCategory *TagCategory "json:\"findTagCategory\" graphql:\"findTagCategory\"" - QueryTagCategories QueryTagCategoriesResultType "json:\"queryTagCategories\" graphql:\"queryTagCategories\"" - FindScene *Scene "json:\"findScene\" graphql:\"findScene\"" - FindSceneByFingerprint []*Scene "json:\"findSceneByFingerprint\" graphql:\"findSceneByFingerprint\"" - FindScenesByFingerprints []*Scene "json:\"findScenesByFingerprints\" graphql:\"findScenesByFingerprints\"" - FindScenesByFullFingerprints []*Scene "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\"" - QueryScenes QueryScenesResultType "json:\"queryScenes\" graphql:\"queryScenes\"" - FindSite *Site "json:\"findSite\" graphql:\"findSite\"" - QuerySites QuerySitesResultType "json:\"querySites\" graphql:\"querySites\"" - FindEdit *Edit "json:\"findEdit\" graphql:\"findEdit\"" - QueryEdits QueryEditsResultType "json:\"queryEdits\" graphql:\"queryEdits\"" - FindUser *User "json:\"findUser\" graphql:\"findUser\"" - QueryUsers QueryUsersResultType "json:\"queryUsers\" graphql:\"queryUsers\"" - Me *User "json:\"me\" graphql:\"me\"" - SearchPerformer []*Performer "json:\"searchPerformer\" graphql:\"searchPerformer\"" - SearchScene []*Scene "json:\"searchScene\" graphql:\"searchScene\"" - FindDraft *Draft "json:\"findDraft\" graphql:\"findDraft\"" - FindDrafts []*Draft "json:\"findDrafts\" graphql:\"findDrafts\"" - Version Version "json:\"version\" graphql:\"version\"" - GetConfig StashBoxConfig "json:\"getConfig\" graphql:\"getConfig\"" + FindPerformer *Performer "json:\"findPerformer\" graphql:\"findPerformer\"" + QueryPerformers QueryPerformersResultType "json:\"queryPerformers\" graphql:\"queryPerformers\"" + FindStudio *Studio "json:\"findStudio\" graphql:\"findStudio\"" + QueryStudios QueryStudiosResultType "json:\"queryStudios\" graphql:\"queryStudios\"" + FindTag *Tag "json:\"findTag\" graphql:\"findTag\"" + QueryTags QueryTagsResultType "json:\"queryTags\" graphql:\"queryTags\"" + FindTagCategory *TagCategory "json:\"findTagCategory\" graphql:\"findTagCategory\"" + QueryTagCategories QueryTagCategoriesResultType "json:\"queryTagCategories\" graphql:\"queryTagCategories\"" + FindScene *Scene "json:\"findScene\" graphql:\"findScene\"" + FindSceneByFingerprint []*Scene "json:\"findSceneByFingerprint\" graphql:\"findSceneByFingerprint\"" + FindScenesByFingerprints []*Scene "json:\"findScenesByFingerprints\" graphql:\"findScenesByFingerprints\"" + FindScenesByFullFingerprints []*Scene "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\"" + FindScenesBySceneFingerprints [][]*Scene "json:\"findScenesBySceneFingerprints\" graphql:\"findScenesBySceneFingerprints\"" + QueryScenes QueryScenesResultType "json:\"queryScenes\" graphql:\"queryScenes\"" + FindSite *Site "json:\"findSite\" graphql:\"findSite\"" + QuerySites QuerySitesResultType "json:\"querySites\" graphql:\"querySites\"" + FindEdit *Edit "json:\"findEdit\" graphql:\"findEdit\"" + QueryEdits QueryEditsResultType "json:\"queryEdits\" graphql:\"queryEdits\"" + FindUser *User "json:\"findUser\" graphql:\"findUser\"" + QueryUsers QueryUsersResultType "json:\"queryUsers\" graphql:\"queryUsers\"" + Me *User "json:\"me\" graphql:\"me\"" + SearchPerformer []*Performer "json:\"searchPerformer\" graphql:\"searchPerformer\"" + SearchScene []*Scene "json:\"searchScene\" graphql:\"searchScene\"" + FindDraft *Draft "json:\"findDraft\" graphql:\"findDraft\"" + FindDrafts []*Draft "json:\"findDrafts\" graphql:\"findDrafts\"" + Version Version "json:\"version\" graphql:\"version\"" + GetConfig StashBoxConfig "json:\"getConfig\" graphql:\"getConfig\"" } type Mutation struct { SceneCreate *Scene "json:\"sceneCreate\" graphql:\"sceneCreate\"" @@ -95,6 +97,10 @@ type Mutation struct { PerformerEdit Edit "json:\"performerEdit\" graphql:\"performerEdit\"" StudioEdit Edit "json:\"studioEdit\" graphql:\"studioEdit\"" TagEdit Edit "json:\"tagEdit\" graphql:\"tagEdit\"" + SceneEditUpdate Edit "json:\"sceneEditUpdate\" graphql:\"sceneEditUpdate\"" + PerformerEditUpdate Edit "json:\"performerEditUpdate\" graphql:\"performerEditUpdate\"" + StudioEditUpdate Edit "json:\"studioEditUpdate\" graphql:\"studioEditUpdate\"" + TagEditUpdate Edit "json:\"tagEditUpdate\" graphql:\"tagEditUpdate\"" EditVote Edit "json:\"editVote\" graphql:\"editVote\"" EditComment Edit "json:\"editComment\" graphql:\"editComment\"" ApplyEdit Edit "json:\"applyEdit\" graphql:\"applyEdit\"" @@ -190,6 +196,9 @@ type FindSceneByFingerprint struct { type FindScenesByFullFingerprints struct { FindScenesByFullFingerprints []*SceneFragment "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\"" } +type FindScenesBySceneFingerprints struct { + FindScenesBySceneFingerprints [][]*SceneFragment "json:\"findScenesBySceneFingerprints\" graphql:\"findScenesBySceneFingerprints\"" +} type SearchScene struct { SearchScene []*SceneFragment "json:\"searchScene\" graphql:\"searchScene\"" } @@ -240,6 +249,10 @@ fragment StudioFragment on Studio { ... ImageFragment } } +fragment TagFragment on Tag { + name + id +} fragment PerformerFragment on Performer { id name @@ -274,16 +287,6 @@ fragment PerformerFragment on Performer { ... BodyModificationFragment } } -fragment FuzzyDateFragment on FuzzyDate { - date - accuracy -} -fragment MeasurementsFragment on Measurements { - band_size - cup_size - waist - hip -} fragment BodyModificationFragment on BodyModification { location description @@ -324,16 +327,22 @@ fragment ImageFragment on Image { width height } -fragment TagFragment on Tag { - name - id -} fragment PerformerAppearanceFragment on PerformerAppearance { as performer { ... PerformerFragment } } +fragment FuzzyDateFragment on FuzzyDate { + date + accuracy +} +fragment MeasurementsFragment on Measurements { + band_size + cup_size + waist + hip +} ` func (c *Client) FindSceneByFingerprint(ctx context.Context, fingerprint FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindSceneByFingerprint, error) { @@ -354,10 +363,6 @@ const FindScenesByFullFingerprintsDocument = `query FindScenesByFullFingerprints ... SceneFragment } } -fragment URLFragment on URL { - url - type -} fragment StudioFragment on Studio { name id @@ -368,11 +373,9 @@ fragment StudioFragment on Studio { ... ImageFragment } } -fragment PerformerAppearanceFragment on PerformerAppearance { - as - performer { - ... PerformerFragment - } +fragment TagFragment on Tag { + name + id } fragment PerformerFragment on Performer { id @@ -412,6 +415,15 @@ fragment FuzzyDateFragment on FuzzyDate { date accuracy } +fragment BodyModificationFragment on BodyModification { + location + description +} +fragment FingerprintFragment on Fingerprint { + algorithm + hash + duration +} fragment SceneFragment on Scene { id title @@ -437,9 +449,81 @@ fragment SceneFragment on Scene { ... FingerprintFragment } } -fragment TagFragment on Tag { - name +fragment ImageFragment on Image { id + url + width + height +} +fragment MeasurementsFragment on Measurements { + band_size + cup_size + waist + hip +} +fragment URLFragment on URL { + url + type +} +fragment PerformerAppearanceFragment on PerformerAppearance { + as + performer { + ... PerformerFragment + } +} +` + +func (c *Client) FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error) { + vars := map[string]interface{}{ + "fingerprints": fingerprints, + } + + var res FindScenesByFullFingerprints + if err := c.Client.Post(ctx, "FindScenesByFullFingerprints", FindScenesByFullFingerprintsDocument, &res, vars, httpRequestOptions...); err != nil { + return nil, err + } + + return &res, nil +} + +const FindScenesBySceneFingerprintsDocument = `query FindScenesBySceneFingerprints ($fingerprints: [[FingerprintQueryInput!]!]!) { + findScenesBySceneFingerprints(fingerprints: $fingerprints) { + ... SceneFragment + } +} +fragment PerformerFragment on Performer { + id + name + disambiguation + aliases + gender + merged_ids + urls { + ... URLFragment + } + images { + ... ImageFragment + } + birthdate { + ... FuzzyDateFragment + } + ethnicity + country + eye_color + hair_color + height + measurements { + ... MeasurementsFragment + } + breast_type + career_start_year + career_end_year + tattoos { + ... BodyModificationFragment + } + piercings { + ... BodyModificationFragment + } } fragment MeasurementsFragment on Measurements { band_size @@ -462,15 +546,68 @@ fragment ImageFragment on Image { width height } +fragment URLFragment on URL { + url + type +} +fragment StudioFragment on Studio { + name + id + urls { + ... URLFragment + } + images { + ... ImageFragment + } +} +fragment TagFragment on Tag { + name + id +} +fragment PerformerAppearanceFragment on PerformerAppearance { + as + performer { + ... PerformerFragment + } +} +fragment FuzzyDateFragment on FuzzyDate { + date + accuracy +} +fragment SceneFragment on Scene { + id + title + details + duration + date + urls { + ... URLFragment + } + images { + ... ImageFragment + } + studio { + ... StudioFragment + } + tags { + ... TagFragment + } + performers { + ... PerformerAppearanceFragment + } + fingerprints { + ... FingerprintFragment + } +} ` -func (c *Client) FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error) { +func (c *Client) FindScenesBySceneFingerprints(ctx context.Context, fingerprints [][]*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesBySceneFingerprints, error) { vars := map[string]interface{}{ "fingerprints": fingerprints, } - var res FindScenesByFullFingerprints - if err := c.Client.Post(ctx, "FindScenesByFullFingerprints", FindScenesByFullFingerprintsDocument, &res, vars, httpRequestOptions...); err != nil { + var res FindScenesBySceneFingerprints + if err := c.Client.Post(ctx, "FindScenesBySceneFingerprints", FindScenesBySceneFingerprintsDocument, &res, vars, httpRequestOptions...); err != nil { return nil, err } @@ -490,72 +627,6 @@ fragment TagFragment on Tag { name id } -fragment FuzzyDateFragment on FuzzyDate { - date - accuracy -} -fragment MeasurementsFragment on Measurements { - band_size - cup_size - waist - hip -} -fragment BodyModificationFragment on BodyModification { - location - description -} -fragment FingerprintFragment on Fingerprint { - algorithm - hash - duration -} -fragment SceneFragment on Scene { - id - title - details - duration - date - urls { - ... URLFragment - } - images { - ... ImageFragment - } - studio { - ... StudioFragment - } - tags { - ... TagFragment - } - performers { - ... PerformerAppearanceFragment - } - fingerprints { - ... FingerprintFragment - } -} -fragment ImageFragment on Image { - id - url - width - height -} -fragment StudioFragment on Studio { - name - id - urls { - ... URLFragment - } - images { - ... ImageFragment - } -} -fragment PerformerAppearanceFragment on PerformerAppearance { - as - performer { - ... PerformerFragment - } -} fragment PerformerFragment on Performer { id name @@ -590,6 +661,72 @@ fragment PerformerFragment on Performer { ... BodyModificationFragment } } +fragment SceneFragment on Scene { + id + title + details + duration + date + urls { + ... URLFragment + } + images { + ... ImageFragment + } + studio { + ... StudioFragment + } + tags { + ... TagFragment + } + performers { + ... PerformerAppearanceFragment + } + fingerprints { + ... FingerprintFragment + } +} +fragment StudioFragment on Studio { + name + id + urls { + ... URLFragment + } + images { + ... ImageFragment + } +} +fragment PerformerAppearanceFragment on PerformerAppearance { + as + performer { + ... PerformerFragment + } +} +fragment FuzzyDateFragment on FuzzyDate { + date + accuracy +} +fragment MeasurementsFragment on Measurements { + band_size + cup_size + waist + hip +} +fragment BodyModificationFragment on BodyModification { + location + description +} +fragment FingerprintFragment on Fingerprint { + algorithm + hash + duration +} +fragment ImageFragment on Image { + id + url + width + height +} ` func (c *Client) SearchScene(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchScene, error) { @@ -610,16 +747,6 @@ const SearchPerformerDocument = `query SearchPerformer ($term: String!) { ... PerformerFragment } } -fragment URLFragment on URL { - url - type -} -fragment ImageFragment on Image { - id - url - width - height -} fragment FuzzyDateFragment on FuzzyDate { date accuracy @@ -668,6 +795,16 @@ fragment PerformerFragment on Performer { ... BodyModificationFragment } } +fragment URLFragment on URL { + url + type +} +fragment ImageFragment on Image { + id + url + width + height +} ` func (c *Client) SearchPerformer(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchPerformer, error) { @@ -688,6 +825,16 @@ const FindPerformerByIDDocument = `query FindPerformerByID ($id: ID!) { ... PerformerFragment } } +fragment FuzzyDateFragment on FuzzyDate { + date + accuracy +} +fragment MeasurementsFragment on Measurements { + band_size + cup_size + waist + hip +} fragment BodyModificationFragment on BodyModification { location description @@ -736,16 +883,6 @@ fragment ImageFragment on Image { width height } -fragment FuzzyDateFragment on FuzzyDate { - date - accuracy -} -fragment MeasurementsFragment on Measurements { - band_size - cup_size - waist - hip -} ` func (c *Client) FindPerformerByID(ctx context.Context, id string, httpRequestOptions ...client.HTTPRequestOption) (*FindPerformerByID, error) { @@ -766,63 +903,11 @@ const FindSceneByIDDocument = `query FindSceneByID ($id: ID!) { ... SceneFragment } } -fragment PerformerFragment on Performer { +fragment ImageFragment on Image { id - name - disambiguation - aliases - gender - merged_ids - urls { - ... URLFragment - } - images { - ... ImageFragment - } - birthdate { - ... FuzzyDateFragment - } - ethnicity - country - eye_color - hair_color - height - measurements { - ... MeasurementsFragment - } - breast_type - career_start_year - career_end_year - tattoos { - ... BodyModificationFragment - } - piercings { - ... BodyModificationFragment - } -} -fragment MeasurementsFragment on Measurements { - band_size - cup_size - waist - hip -} -fragment BodyModificationFragment on BodyModification { - location - description -} -fragment URLFragment on URL { url - type -} -fragment StudioFragment on Studio { - name - id - urls { - ... URLFragment - } - images { - ... ImageFragment - } + width + height } fragment TagFragment on Tag { name @@ -834,14 +919,15 @@ fragment PerformerAppearanceFragment on PerformerAppearance { ... PerformerFragment } } -fragment FuzzyDateFragment on FuzzyDate { - date - accuracy +fragment MeasurementsFragment on Measurements { + band_size + cup_size + waist + hip } -fragment FingerprintFragment on Fingerprint { - algorithm - hash - duration +fragment BodyModificationFragment on BodyModification { + location + description } fragment SceneFragment on Scene { id @@ -868,11 +954,62 @@ fragment SceneFragment on Scene { ... FingerprintFragment } } -fragment ImageFragment on Image { - id +fragment URLFragment on URL { url - width + type +} +fragment StudioFragment on Studio { + name + id + urls { + ... URLFragment + } + images { + ... ImageFragment + } +} +fragment PerformerFragment on Performer { + id + name + disambiguation + aliases + gender + merged_ids + urls { + ... URLFragment + } + images { + ... ImageFragment + } + birthdate { + ... FuzzyDateFragment + } + ethnicity + country + eye_color + hair_color height + measurements { + ... MeasurementsFragment + } + breast_type + career_start_year + career_end_year + tattoos { + ... BodyModificationFragment + } + piercings { + ... BodyModificationFragment + } +} +fragment FuzzyDateFragment on FuzzyDate { + date + accuracy +} +fragment FingerprintFragment on Fingerprint { + algorithm + hash + duration } ` diff --git a/pkg/scraper/stashbox/graphql/generated_models.go b/pkg/scraper/stashbox/graphql/generated_models.go index 2ce0301bb..cfa893c23 100644 --- a/pkg/scraper/stashbox/graphql/generated_models.go +++ b/pkg/scraper/stashbox/graphql/generated_models.go @@ -88,8 +88,8 @@ type DraftEntity struct { ID *string `json:"id,omitempty"` } -func (DraftEntity) IsSceneDraftPerformer() {} func (DraftEntity) IsSceneDraftStudio() {} +func (DraftEntity) IsSceneDraftPerformer() {} func (DraftEntity) IsSceneDraftTag() {} type DraftEntityInput struct { @@ -130,7 +130,9 @@ type Edit struct { Status VoteStatusEnum `json:"status"` Applied bool `json:"applied"` Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Updated *time.Time `json:"updated,omitempty"` + Closed *time.Time `json:"closed,omitempty"` + Expires *time.Time `json:"expires,omitempty"` } type EditComment struct { @@ -149,8 +151,6 @@ type EditInput struct { // Not required for create type ID *string `json:"id,omitempty"` Operation OperationEnum `json:"operation"` - // Required for amending an existing edit - EditID *string `json:"edit_id,omitempty"` // Only required for merge type MergeSourceIds []string `json:"merge_source_ids,omitempty"` Comment *string `json:"comment,omitempty"` @@ -206,15 +206,13 @@ type Fingerprint struct { } type FingerprintEditInput struct { - UserIds []string `json:"user_ids,omitempty"` - Hash string `json:"hash"` - Algorithm FingerprintAlgorithm `json:"algorithm"` - Duration int `json:"duration"` - Created time.Time `json:"created"` - // @deprecated(reason: "unused") - Submissions *int `json:"submissions,omitempty"` - // @deprecated(reason: "unused") - Updated *time.Time `json:"updated,omitempty"` + UserIds []string `json:"user_ids,omitempty"` + Hash string `json:"hash"` + Algorithm FingerprintAlgorithm `json:"algorithm"` + Duration int `json:"duration"` + Created time.Time `json:"created"` + Submissions *int `json:"submissions,omitempty"` + Updated *time.Time `json:"updated,omitempty"` } type FingerprintInput struct { @@ -241,11 +239,6 @@ type FuzzyDate struct { Accuracy DateAccuracyEnum `json:"accuracy"` } -type FuzzyDateInput struct { - Date string `json:"date"` - Accuracy DateAccuracyEnum `json:"accuracy"` -} - type GrantInviteInput struct { UserID string `json:"user_id"` Amount int `json:"amount"` @@ -294,13 +287,6 @@ type Measurements struct { Hip *int `json:"hip,omitempty"` } -type MeasurementsInput struct { - CupSize *string `json:"cup_size,omitempty"` - BandSize *int `json:"band_size,omitempty"` - Waist *int `json:"waist,omitempty"` - Hip *int `json:"hip,omitempty"` -} - type MultiIDCriterionInput struct { Value []string `json:"value,omitempty"` Modifier CriterionModifier `json:"modifier"` @@ -324,6 +310,7 @@ type Performer struct { Gender *GenderEnum `json:"gender,omitempty"` Urls []*URL `json:"urls,omitempty"` Birthdate *FuzzyDate `json:"birthdate,omitempty"` + BirthDate *string `json:"birth_date,omitempty"` Age *int `json:"age,omitempty"` Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"` Country *string `json:"country,omitempty"` @@ -332,6 +319,10 @@ type Performer struct { // Height in cm Height *int `json:"height,omitempty"` Measurements *Measurements `json:"measurements,omitempty"` + CupSize *string `json:"cup_size,omitempty"` + BandSize *int `json:"band_size,omitempty"` + WaistSize *int `json:"waist_size,omitempty"` + HipSize *int `json:"hip_size,omitempty"` BreastType *BreastTypeEnum `json:"breast_type,omitempty"` CareerStartYear *int `json:"career_start_year,omitempty"` CareerEndYear *int `json:"career_end_year,omitempty"` @@ -348,8 +339,8 @@ type Performer struct { Updated time.Time `json:"updated"` } -func (Performer) IsSceneDraftPerformer() {} func (Performer) IsEditTarget() {} +func (Performer) IsSceneDraftPerformer() {} type PerformerAppearance struct { Performer *Performer `json:"performer,omitempty"` @@ -369,13 +360,16 @@ type PerformerCreateInput struct { Aliases []string `json:"aliases,omitempty"` Gender *GenderEnum `json:"gender,omitempty"` Urls []*URLInput `json:"urls,omitempty"` - Birthdate *FuzzyDateInput `json:"birthdate,omitempty"` + Birthdate *string `json:"birthdate,omitempty"` Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"` Country *string `json:"country,omitempty"` EyeColor *EyeColorEnum `json:"eye_color,omitempty"` HairColor *HairColorEnum `json:"hair_color,omitempty"` Height *int `json:"height,omitempty"` - Measurements *MeasurementsInput `json:"measurements,omitempty"` + CupSize *string `json:"cup_size,omitempty"` + BandSize *int `json:"band_size,omitempty"` + WaistSize *int `json:"waist_size,omitempty"` + HipSize *int `json:"hip_size,omitempty"` BreastType *BreastTypeEnum `json:"breast_type,omitempty"` CareerStartYear *int `json:"career_start_year,omitempty"` CareerEndYear *int `json:"career_end_year,omitempty"` @@ -390,6 +384,7 @@ type PerformerDestroyInput struct { } type PerformerDraft struct { + ID *string `json:"id,omitempty"` Name string `json:"name"` Aliases *string `json:"aliases,omitempty"` Gender *string `json:"gender,omitempty"` @@ -412,6 +407,7 @@ type PerformerDraft struct { func (PerformerDraft) IsDraftData() {} type PerformerDraftInput struct { + ID *string `json:"id,omitempty"` Name string `json:"name"` Aliases *string `json:"aliases,omitempty"` Gender *string `json:"gender,omitempty"` @@ -432,19 +428,18 @@ type PerformerDraftInput struct { } type PerformerEdit struct { - Name *string `json:"name,omitempty"` - Disambiguation *string `json:"disambiguation,omitempty"` - AddedAliases []string `json:"added_aliases,omitempty"` - RemovedAliases []string `json:"removed_aliases,omitempty"` - Gender *GenderEnum `json:"gender,omitempty"` - AddedUrls []*URL `json:"added_urls,omitempty"` - RemovedUrls []*URL `json:"removed_urls,omitempty"` - Birthdate *string `json:"birthdate,omitempty"` - BirthdateAccuracy *string `json:"birthdate_accuracy,omitempty"` - Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"` - Country *string `json:"country,omitempty"` - EyeColor *EyeColorEnum `json:"eye_color,omitempty"` - HairColor *HairColorEnum `json:"hair_color,omitempty"` + Name *string `json:"name,omitempty"` + Disambiguation *string `json:"disambiguation,omitempty"` + AddedAliases []string `json:"added_aliases,omitempty"` + RemovedAliases []string `json:"removed_aliases,omitempty"` + Gender *GenderEnum `json:"gender,omitempty"` + AddedUrls []*URL `json:"added_urls,omitempty"` + RemovedUrls []*URL `json:"removed_urls,omitempty"` + Birthdate *string `json:"birthdate,omitempty"` + Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"` + Country *string `json:"country,omitempty"` + EyeColor *EyeColorEnum `json:"eye_color,omitempty"` + HairColor *HairColorEnum `json:"hair_color,omitempty"` // Height in cm Height *int `json:"height,omitempty"` CupSize *string `json:"cup_size,omitempty"` @@ -461,6 +456,11 @@ type PerformerEdit struct { AddedImages []*Image `json:"added_images,omitempty"` RemovedImages []*Image `json:"removed_images,omitempty"` DraftID *string `json:"draft_id,omitempty"` + Aliases []string `json:"aliases,omitempty"` + Urls []*URL `json:"urls,omitempty"` + Images []*Image `json:"images,omitempty"` + Tattoos []*BodyModification `json:"tattoos,omitempty"` + Piercings []*BodyModification `json:"piercings,omitempty"` } func (PerformerEdit) IsEditDetails() {} @@ -471,13 +471,16 @@ type PerformerEditDetailsInput struct { Aliases []string `json:"aliases,omitempty"` Gender *GenderEnum `json:"gender,omitempty"` Urls []*URLInput `json:"urls,omitempty"` - Birthdate *FuzzyDateInput `json:"birthdate,omitempty"` + Birthdate *string `json:"birthdate,omitempty"` Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"` Country *string `json:"country,omitempty"` EyeColor *EyeColorEnum `json:"eye_color,omitempty"` HairColor *HairColorEnum `json:"hair_color,omitempty"` Height *int `json:"height,omitempty"` - Measurements *MeasurementsInput `json:"measurements,omitempty"` + CupSize *string `json:"cup_size,omitempty"` + BandSize *int `json:"band_size,omitempty"` + WaistSize *int `json:"waist_size,omitempty"` + HipSize *int `json:"hip_size,omitempty"` BreastType *BreastTypeEnum `json:"breast_type,omitempty"` CareerStartYear *int `json:"career_start_year,omitempty"` CareerEndYear *int `json:"career_end_year,omitempty"` @@ -510,7 +513,7 @@ type PerformerEditOptionsInput struct { } type PerformerQueryInput struct { - // Searches name and aliases - assumes like query unless quoted + // Searches name and disambiguation - assumes like query unless quoted Names *string `json:"names,omitempty"` // Searches name only - assumes like query unless quoted Name *string `json:"name,omitempty"` @@ -557,13 +560,16 @@ type PerformerUpdateInput struct { Aliases []string `json:"aliases,omitempty"` Gender *GenderEnum `json:"gender,omitempty"` Urls []*URLInput `json:"urls,omitempty"` - Birthdate *FuzzyDateInput `json:"birthdate,omitempty"` + Birthdate *string `json:"birthdate,omitempty"` Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"` Country *string `json:"country,omitempty"` EyeColor *EyeColorEnum `json:"eye_color,omitempty"` HairColor *HairColorEnum `json:"hair_color,omitempty"` Height *int `json:"height,omitempty"` - Measurements *MeasurementsInput `json:"measurements,omitempty"` + CupSize *string `json:"cup_size,omitempty"` + BandSize *int `json:"band_size,omitempty"` + WaistSize *int `json:"waist_size,omitempty"` + HipSize *int `json:"hip_size,omitempty"` BreastType *BreastTypeEnum `json:"breast_type,omitempty"` CareerStartYear *int `json:"career_start_year,omitempty"` CareerEndYear *int `json:"career_end_year,omitempty"` @@ -631,6 +637,7 @@ type Scene struct { Title *string `json:"title,omitempty"` Details *string `json:"details,omitempty"` Date *string `json:"date,omitempty"` + ReleaseDate *string `json:"release_date,omitempty"` Urls []*URL `json:"urls,omitempty"` Studio *Studio `json:"studio,omitempty"` Tags []*Tag `json:"tags,omitempty"` @@ -652,7 +659,7 @@ type SceneCreateInput struct { Title *string `json:"title,omitempty"` Details *string `json:"details,omitempty"` Urls []*URLInput `json:"urls,omitempty"` - Date *string `json:"date,omitempty"` + Date string `json:"date"` StudioID *string `json:"studio_id,omitempty"` Performers []*PerformerAppearanceInput `json:"performers,omitempty"` TagIds []string `json:"tag_ids,omitempty"` @@ -668,6 +675,7 @@ type SceneDestroyInput struct { } type SceneDraft struct { + ID *string `json:"id,omitempty"` Title *string `json:"title,omitempty"` Details *string `json:"details,omitempty"` URL *URL `json:"url,omitempty"` @@ -701,6 +709,11 @@ type SceneEdit struct { Director *string `json:"director,omitempty"` Code *string `json:"code,omitempty"` DraftID *string `json:"draft_id,omitempty"` + Urls []*URL `json:"urls,omitempty"` + Performers []*PerformerAppearance `json:"performers,omitempty"` + Tags []*Tag `json:"tags,omitempty"` + Images []*Image `json:"images,omitempty"` + Fingerprints []*Fingerprint `json:"fingerprints,omitempty"` } func (SceneEdit) IsEditDetails() {} @@ -855,6 +868,8 @@ type StudioEdit struct { Parent *Studio `json:"parent,omitempty"` AddedImages []*Image `json:"added_images,omitempty"` RemovedImages []*Image `json:"removed_images,omitempty"` + Images []*Image `json:"images,omitempty"` + Urls []*URL `json:"urls,omitempty"` } func (StudioEdit) IsEditDetails() {} @@ -909,8 +924,8 @@ type Tag struct { Updated time.Time `json:"updated"` } -func (Tag) IsSceneDraftTag() {} func (Tag) IsEditTarget() {} +func (Tag) IsSceneDraftTag() {} type TagCategory struct { ID string `json:"id"` @@ -953,6 +968,7 @@ type TagEdit struct { AddedAliases []string `json:"added_aliases,omitempty"` RemovedAliases []string `json:"removed_aliases,omitempty"` Category *TagCategory `json:"category,omitempty"` + Aliases []string `json:"aliases,omitempty"` } func (TagEdit) IsEditDetails() {} @@ -1256,16 +1272,18 @@ type EditSortEnum string const ( EditSortEnumCreatedAt EditSortEnum = "CREATED_AT" EditSortEnumUpdatedAt EditSortEnum = "UPDATED_AT" + EditSortEnumClosedAt EditSortEnum = "CLOSED_AT" ) var AllEditSortEnum = []EditSortEnum{ EditSortEnumCreatedAt, EditSortEnumUpdatedAt, + EditSortEnumClosedAt, } func (e EditSortEnum) IsValid() bool { switch e { - case EditSortEnumCreatedAt, EditSortEnumUpdatedAt: + case EditSortEnumCreatedAt, EditSortEnumUpdatedAt, EditSortEnumClosedAt: return true } return false diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 8b0af9f55..105fe3d24 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -14,7 +14,6 @@ import ( "strings" "github.com/Yamashou/gqlgenc/client" - "github.com/corona10/goimagehash" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -24,7 +23,6 @@ import ( "github.com/stashapp/stash/pkg/match" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scraper/stashbox/graphql" - "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -78,127 +76,21 @@ func (c Client) QueryStashBoxScene(ctx context.Context, queryStr string) ([]*mod return ret, nil } -func phashMatches(hash, other int64) bool { - // HACK - stash-box match distance is configurable. This needs to be fixed on - // the stash-box end. - const stashBoxDistance = 4 - - imageHash := goimagehash.NewImageHash(uint64(hash), goimagehash.PHash) - otherHash := goimagehash.NewImageHash(uint64(other), goimagehash.PHash) - - distance, _ := imageHash.Distance(otherHash) - return distance <= stashBoxDistance +// FindStashBoxScenesByFingerprints queries stash-box for a scene using the +// scene's MD5/OSHASH checksum, or PHash. +func (c Client) FindStashBoxSceneByFingerprints(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) { + res, err := c.FindStashBoxScenesByFingerprints(ctx, []int{sceneID}) + if len(res) > 0 { + return res[0], err + } + return nil, err } // FindStashBoxScenesByFingerprints queries stash-box for scenes using every // scene's MD5/OSHASH checksum, or PHash, and returns results in the same order // as the input slice. -func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, sceneIDs []string) ([][]*models.ScrapedScene, error) { - ids, err := stringslice.StringSliceToIntSlice(sceneIDs) - if err != nil { - return nil, err - } - - var fingerprints []*graphql.FingerprintQueryInput - // map fingerprints to their scene index - fpToScene := make(map[string][]int) - phashToScene := make(map[int64][]int) - - if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { - qb := r.Scene() - - for index, sceneID := range ids { - scene, err := qb.Find(sceneID) - if err != nil { - return err - } - - if scene == nil { - return fmt.Errorf("scene with id %d not found", sceneID) - } - - if scene.Checksum.Valid { - fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{ - Hash: scene.Checksum.String, - Algorithm: graphql.FingerprintAlgorithmMd5, - }) - fpToScene[scene.Checksum.String] = append(fpToScene[scene.Checksum.String], index) - } - - if scene.OSHash.Valid { - fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{ - Hash: scene.OSHash.String, - Algorithm: graphql.FingerprintAlgorithmOshash, - }) - fpToScene[scene.OSHash.String] = append(fpToScene[scene.OSHash.String], index) - } - - if scene.Phash.Valid { - phashStr := utils.PhashToString(scene.Phash.Int64) - fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{ - Hash: phashStr, - Algorithm: graphql.FingerprintAlgorithmPhash, - }) - fpToScene[phashStr] = append(fpToScene[phashStr], index) - phashToScene[scene.Phash.Int64] = append(phashToScene[scene.Phash.Int64], index) - } - } - - return nil - }); err != nil { - return nil, err - } - - allScenes, err := c.findStashBoxScenesByFingerprints(ctx, fingerprints) - if err != nil { - return nil, err - } - - // set the matched scenes back in their original order - ret := make([][]*models.ScrapedScene, len(sceneIDs)) - for _, s := range allScenes { - var addedTo []int - - addScene := func(sceneIndexes []int) { - for _, index := range sceneIndexes { - if !intslice.IntInclude(addedTo, index) { - addedTo = append(addedTo, index) - ret[index] = append(ret[index], s) - } - } - } - - for _, fp := range s.Fingerprints { - addScene(fpToScene[fp.Hash]) - - // HACK - we really need stash-box to return specific hash-to-result sets - if fp.Algorithm == graphql.FingerprintAlgorithmPhash.String() { - hash, err := utils.StringToPhash(fp.Hash) - if err != nil { - continue - } - - for phash, sceneIndexes := range phashToScene { - if phashMatches(hash, phash) { - addScene(sceneIndexes) - } - } - } - } - } - - return ret, nil -} - -// FindStashBoxScenesByFingerprintsFlat queries stash-box for scenes using every -// scene's MD5/OSHASH checksum, or PHash, and returns results a flat slice. -func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneIDs []string) ([]*models.ScrapedScene, error) { - ids, err := stringslice.StringSliceToIntSlice(sceneIDs) - if err != nil { - return nil, err - } - - var fingerprints []*graphql.FingerprintQueryInput +func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, ids []int) ([][]*models.ScrapedScene, error) { + var fingerprints [][]*graphql.FingerprintQueryInput if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { qb := r.Scene() @@ -213,26 +105,31 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneI return fmt.Errorf("scene with id %d not found", sceneID) } + var sceneFPs []*graphql.FingerprintQueryInput + if scene.Checksum.Valid { - fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{ + sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{ Hash: scene.Checksum.String, Algorithm: graphql.FingerprintAlgorithmMd5, }) } if scene.OSHash.Valid { - fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{ + sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{ Hash: scene.OSHash.String, Algorithm: graphql.FingerprintAlgorithmOshash, }) } if scene.Phash.Valid { - fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{ - Hash: utils.PhashToString(scene.Phash.Int64), + phashStr := utils.PhashToString(scene.Phash.Int64) + sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{ + Hash: phashStr, Algorithm: graphql.FingerprintAlgorithmPhash, }) } + + fingerprints = append(fingerprints, sceneFPs) } return nil @@ -243,27 +140,29 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneI return c.findStashBoxScenesByFingerprints(ctx, fingerprints) } -func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, fingerprints []*graphql.FingerprintQueryInput) ([]*models.ScrapedScene, error) { - var ret []*models.ScrapedScene - for i := 0; i < len(fingerprints); i += 100 { +func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, scenes [][]*graphql.FingerprintQueryInput) ([][]*models.ScrapedScene, error) { + var ret [][]*models.ScrapedScene + for i := 0; i < len(scenes); i += 40 { end := i + 100 - if end > len(fingerprints) { - end = len(fingerprints) + if end > len(scenes) { + end = len(scenes) } - scenes, err := c.client.FindScenesByFullFingerprints(ctx, fingerprints[i:end]) + scenes, err := c.client.FindScenesBySceneFingerprints(ctx, scenes[i:end]) if err != nil { return nil, err } - sceneFragments := scenes.FindScenesByFullFingerprints - - for _, s := range sceneFragments { - ss, err := c.sceneFragmentToScrapedScene(ctx, s) - if err != nil { - return nil, err + for _, sceneFragments := range scenes.FindScenesBySceneFingerprints { + var sceneResults []*models.ScrapedScene + for _, scene := range sceneFragments { + ss, err := c.sceneFragmentToScrapedScene(ctx, scene) + if err != nil { + return nil, err + } + sceneResults = append(sceneResults, ss) } - ret = append(ret, ss) + ret = append(ret, sceneResults) } }