mirror of
https://github.com/stashapp/stash.git
synced 2025-12-07 17:02:38 +01:00
Add Image Scraping (#5562)
Co-authored-by: keenbed <155155956+keenbed@users.noreply.github.com> Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
parent
b6ace42973
commit
e97f647a43
27 changed files with 1063 additions and 11 deletions
|
|
@ -174,6 +174,12 @@ type Query {
|
||||||
input: ScrapeSingleGroupInput!
|
input: ScrapeSingleGroupInput!
|
||||||
): [ScrapedGroup!]!
|
): [ScrapedGroup!]!
|
||||||
|
|
||||||
|
"Scrape for a single image"
|
||||||
|
scrapeSingleImage(
|
||||||
|
source: ScraperSourceInput!
|
||||||
|
input: ScrapeSingleImageInput!
|
||||||
|
): [ScrapedImage!]!
|
||||||
|
|
||||||
"Scrapes content based on a URL"
|
"Scrapes content based on a URL"
|
||||||
scrapeURL(url: String!, ty: ScrapeContentType!): ScrapedContent
|
scrapeURL(url: String!, ty: ScrapeContentType!): ScrapedContent
|
||||||
|
|
||||||
|
|
@ -183,6 +189,8 @@ type Query {
|
||||||
scrapeSceneURL(url: String!): ScrapedScene
|
scrapeSceneURL(url: String!): ScrapedScene
|
||||||
"Scrapes a complete gallery record based on a URL"
|
"Scrapes a complete gallery record based on a URL"
|
||||||
scrapeGalleryURL(url: String!): ScrapedGallery
|
scrapeGalleryURL(url: String!): ScrapedGallery
|
||||||
|
"Scrapes a complete image record based on a URL"
|
||||||
|
scrapeImageURL(url: String!): ScrapedImage
|
||||||
"Scrapes a complete movie record based on a URL"
|
"Scrapes a complete movie record based on a URL"
|
||||||
scrapeMovieURL(url: String!): ScrapedMovie
|
scrapeMovieURL(url: String!): ScrapedMovie
|
||||||
@deprecated(reason: "Use scrapeGroupURL instead")
|
@deprecated(reason: "Use scrapeGroupURL instead")
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ enum ScrapeType {
|
||||||
"Type of the content a scraper generates"
|
"Type of the content a scraper generates"
|
||||||
enum ScrapeContentType {
|
enum ScrapeContentType {
|
||||||
GALLERY
|
GALLERY
|
||||||
|
IMAGE
|
||||||
MOVIE
|
MOVIE
|
||||||
GROUP
|
GROUP
|
||||||
PERFORMER
|
PERFORMER
|
||||||
|
|
@ -22,6 +23,7 @@ union ScrapedContent =
|
||||||
| ScrapedTag
|
| ScrapedTag
|
||||||
| ScrapedScene
|
| ScrapedScene
|
||||||
| ScrapedGallery
|
| ScrapedGallery
|
||||||
|
| ScrapedImage
|
||||||
| ScrapedMovie
|
| ScrapedMovie
|
||||||
| ScrapedGroup
|
| ScrapedGroup
|
||||||
| ScrapedPerformer
|
| ScrapedPerformer
|
||||||
|
|
@ -41,6 +43,8 @@ type Scraper {
|
||||||
scene: ScraperSpec
|
scene: ScraperSpec
|
||||||
"Details for gallery scraper"
|
"Details for gallery scraper"
|
||||||
gallery: ScraperSpec
|
gallery: ScraperSpec
|
||||||
|
"Details for image scraper"
|
||||||
|
image: ScraperSpec
|
||||||
"Details for movie scraper"
|
"Details for movie scraper"
|
||||||
movie: ScraperSpec @deprecated(reason: "use group")
|
movie: ScraperSpec @deprecated(reason: "use group")
|
||||||
"Details for group scraper"
|
"Details for group scraper"
|
||||||
|
|
@ -128,6 +132,26 @@ input ScrapedGalleryInput {
|
||||||
# no studio, tags or performers
|
# no studio, tags or performers
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ScrapedImage {
|
||||||
|
title: String
|
||||||
|
code: String
|
||||||
|
details: String
|
||||||
|
photographer: String
|
||||||
|
urls: [String!]
|
||||||
|
date: String
|
||||||
|
studio: ScrapedStudio
|
||||||
|
tags: [ScrapedTag!]
|
||||||
|
performers: [ScrapedPerformer!]
|
||||||
|
}
|
||||||
|
|
||||||
|
input ScrapedImageInput {
|
||||||
|
title: String
|
||||||
|
code: String
|
||||||
|
details: String
|
||||||
|
urls: [String!]
|
||||||
|
date: String
|
||||||
|
}
|
||||||
|
|
||||||
input ScraperSourceInput {
|
input ScraperSourceInput {
|
||||||
"Index of the configured stash-box instance to use. Should be unset if scraper_id is set"
|
"Index of the configured stash-box instance to use. Should be unset if scraper_id is set"
|
||||||
stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
|
stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
|
||||||
|
|
@ -190,6 +214,15 @@ input ScrapeSingleGalleryInput {
|
||||||
gallery_input: ScrapedGalleryInput
|
gallery_input: ScrapedGalleryInput
|
||||||
}
|
}
|
||||||
|
|
||||||
|
input ScrapeSingleImageInput {
|
||||||
|
"Instructs to query by string"
|
||||||
|
query: String
|
||||||
|
"Instructs to query by image id"
|
||||||
|
image_id: ID
|
||||||
|
"Instructs to query by image fragment"
|
||||||
|
image_input: ScrapedImageInput
|
||||||
|
}
|
||||||
|
|
||||||
input ScrapeSingleMovieInput {
|
input ScrapeSingleMovieInput {
|
||||||
"Instructs to query by string"
|
"Instructs to query by string"
|
||||||
query: String
|
query: String
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,7 @@ func (r *mutationResolver) getImage(ctx context.Context, id int) (ret *models.Im
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInput) (ret *models.Image, err error) {
|
func (r *mutationResolver) ImageUpdate(ctx context.Context, input models.ImageUpdateInput) (ret *models.Image, err error) {
|
||||||
translator := changesetTranslator{
|
translator := changesetTranslator{
|
||||||
inputMap: getUpdateInputMap(ctx),
|
inputMap: getUpdateInputMap(ctx),
|
||||||
}
|
}
|
||||||
|
|
@ -46,7 +46,7 @@ func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInp
|
||||||
return r.getImage(ctx, ret.ID)
|
return r.getImage(ctx, ret.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdateInput) (ret []*models.Image, err error) {
|
func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*models.ImageUpdateInput) (ret []*models.Image, err error) {
|
||||||
inputMaps := getUpdateInputMaps(ctx)
|
inputMaps := getUpdateInputMaps(ctx)
|
||||||
|
|
||||||
// Start the transaction and save the image
|
// Start the transaction and save the image
|
||||||
|
|
@ -89,7 +89,7 @@ func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdat
|
||||||
return newRet, nil
|
return newRet, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
|
func (r *mutationResolver) imageUpdate(ctx context.Context, input models.ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
|
||||||
imageID, err := strconv.Atoi(input.ID)
|
imageID, err := strconv.Atoi(input.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("converting id: %w", err)
|
return nil, fmt.Errorf("converting id: %w", err)
|
||||||
|
|
|
||||||
|
|
@ -197,6 +197,15 @@ func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scra
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *queryResolver) ScrapeImageURL(ctx context.Context, url string) (*scraper.ScrapedImage, error) {
|
||||||
|
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeImage)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return marshalScrapedImage(content)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models.ScrapedMovie, error) {
|
func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models.ScrapedMovie, error) {
|
||||||
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeMovie)
|
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeMovie)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -491,6 +500,39 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *queryResolver) ScrapeSingleImage(ctx context.Context, source scraper.Source, input ScrapeSingleImageInput) ([]*scraper.ScrapedImage, error) {
|
||||||
|
if source.StashBoxIndex != nil {
|
||||||
|
return nil, ErrNotSupported
|
||||||
|
}
|
||||||
|
|
||||||
|
if source.ScraperID == nil {
|
||||||
|
return nil, fmt.Errorf("%w: scraper_id must be set", ErrInput)
|
||||||
|
}
|
||||||
|
|
||||||
|
var c scraper.ScrapedContent
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case input.ImageID != nil:
|
||||||
|
imageID, err := strconv.Atoi(*input.ImageID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: image id is not an integer: '%s'", ErrInput, *input.ImageID)
|
||||||
|
}
|
||||||
|
c, err = r.scraperCache().ScrapeID(ctx, *source.ScraperID, imageID, scraper.ScrapeContentTypeImage)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return marshalScrapedImages([]scraper.ScrapedContent{c})
|
||||||
|
case input.ImageInput != nil:
|
||||||
|
c, err := r.scraperCache().ScrapeFragment(ctx, *source.ScraperID, scraper.Input{Image: input.ImageInput})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return marshalScrapedImages([]scraper.ScrapedContent{c})
|
||||||
|
default:
|
||||||
|
return nil, ErrNotImplemented
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source scraper.Source, input ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) {
|
func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source scraper.Source, input ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) {
|
||||||
return nil, ErrNotSupported
|
return nil, ErrNotSupported
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -76,6 +76,27 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.Scrap
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func marshalScrapedImages(content []scraper.ScrapedContent) ([]*scraper.ScrapedImage, error) {
|
||||||
|
var ret []*scraper.ScrapedImage
|
||||||
|
for _, c := range content {
|
||||||
|
if c == nil {
|
||||||
|
// graphql schema requires images to be non-nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch g := c.(type) {
|
||||||
|
case *scraper.ScrapedImage:
|
||||||
|
ret = append(ret, g)
|
||||||
|
case scraper.ScrapedImage:
|
||||||
|
ret = append(ret, &g)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedImage", models.ErrConversion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret, nil
|
||||||
|
}
|
||||||
|
|
||||||
// marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
|
// marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
|
||||||
// fails, an error is returned.
|
// fails, an error is returned.
|
||||||
func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMovie, error) {
|
func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMovie, error) {
|
||||||
|
|
@ -129,6 +150,16 @@ func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGall
|
||||||
return g[0], nil
|
return g[0], nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// marshalScrapedImage will marshal a single scraped image
|
||||||
|
func marshalScrapedImage(content scraper.ScrapedContent) (*scraper.ScrapedImage, error) {
|
||||||
|
g, err := marshalScrapedImages([]scraper.ScrapedContent{content})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return g[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
// marshalScrapedMovie will marshal a single scraped movie
|
// marshalScrapedMovie will marshal a single scraped movie
|
||||||
func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie, error) {
|
func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie, error) {
|
||||||
m, err := marshalScrapedMovies([]scraper.ScrapedContent{content})
|
m, err := marshalScrapedMovies([]scraper.ScrapedContent{content})
|
||||||
|
|
|
||||||
|
|
@ -63,6 +63,28 @@ type ImageFilterType struct {
|
||||||
UpdatedAt *TimestampCriterionInput `json:"updated_at"`
|
UpdatedAt *TimestampCriterionInput `json:"updated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ImageUpdateInput struct {
|
||||||
|
ClientMutationID *string `json:"clientMutationId"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Title *string `json:"title"`
|
||||||
|
Code *string `json:"code"`
|
||||||
|
Urls []string `json:"urls"`
|
||||||
|
Date *string `json:"date"`
|
||||||
|
Details *string `json:"details"`
|
||||||
|
Photographer *string `json:"photographer"`
|
||||||
|
Rating100 *int `json:"rating100"`
|
||||||
|
Organized *bool `json:"organized"`
|
||||||
|
SceneIds []string `json:"scene_ids"`
|
||||||
|
StudioID *string `json:"studio_id"`
|
||||||
|
TagIds []string `json:"tag_ids"`
|
||||||
|
PerformerIds []string `json:"performer_ids"`
|
||||||
|
GalleryIds []string `json:"gallery_ids"`
|
||||||
|
PrimaryFileID *string `json:"primary_file_id"`
|
||||||
|
|
||||||
|
// deprecated
|
||||||
|
URL *string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
type ImageDestroyInput struct {
|
type ImageDestroyInput struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
DeleteFile *bool `json:"delete_file"`
|
DeleteFile *bool `json:"delete_file"`
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ type scraperActionImpl interface {
|
||||||
|
|
||||||
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
|
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
|
||||||
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
|
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
|
||||||
|
scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {
|
func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {
|
||||||
|
|
|
||||||
|
|
@ -77,11 +77,18 @@ type GalleryFinder interface {
|
||||||
models.URLLoader
|
models.URLLoader
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ImageFinder interface {
|
||||||
|
models.ImageGetter
|
||||||
|
models.FileLoader
|
||||||
|
models.URLLoader
|
||||||
|
}
|
||||||
|
|
||||||
type Repository struct {
|
type Repository struct {
|
||||||
TxnManager models.TxnManager
|
TxnManager models.TxnManager
|
||||||
|
|
||||||
SceneFinder SceneFinder
|
SceneFinder SceneFinder
|
||||||
GalleryFinder GalleryFinder
|
GalleryFinder GalleryFinder
|
||||||
|
ImageFinder ImageFinder
|
||||||
TagFinder TagFinder
|
TagFinder TagFinder
|
||||||
PerformerFinder PerformerFinder
|
PerformerFinder PerformerFinder
|
||||||
GroupFinder match.GroupNamesFinder
|
GroupFinder match.GroupNamesFinder
|
||||||
|
|
@ -93,6 +100,7 @@ func NewRepository(repo models.Repository) Repository {
|
||||||
TxnManager: repo.TxnManager,
|
TxnManager: repo.TxnManager,
|
||||||
SceneFinder: repo.Scene,
|
SceneFinder: repo.Scene,
|
||||||
GalleryFinder: repo.Gallery,
|
GalleryFinder: repo.Gallery,
|
||||||
|
ImageFinder: repo.Image,
|
||||||
TagFinder: repo.Tag,
|
TagFinder: repo.Tag,
|
||||||
PerformerFinder: repo.Performer,
|
PerformerFinder: repo.Performer,
|
||||||
GroupFinder: repo.Group,
|
GroupFinder: repo.Group,
|
||||||
|
|
@ -357,6 +365,28 @@ func (c Cache) ScrapeID(ctx context.Context, scraperID string, id int, ty Scrape
|
||||||
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
|
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if scraped != nil {
|
||||||
|
ret = scraped
|
||||||
|
}
|
||||||
|
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
is, ok := s.(imageScraper)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: cannot use scraper %s as a image scraper", ErrNotSupported, scraperID)
|
||||||
|
}
|
||||||
|
|
||||||
|
scene, err := c.getImage(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scraper %s: unable to load image id %v: %w", scraperID, id, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// don't assign nil concrete pointer to ret interface, otherwise nil
|
||||||
|
// detection is harder
|
||||||
|
scraped, err := is.viaImage(ctx, c.client, scene)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
|
||||||
|
}
|
||||||
|
|
||||||
if scraped != nil {
|
if scraped != nil {
|
||||||
ret = scraped
|
ret = scraped
|
||||||
}
|
}
|
||||||
|
|
@ -426,3 +456,31 @@ func (c Cache) getGallery(ctx context.Context, galleryID int) (*models.Gallery,
|
||||||
}
|
}
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c Cache) getImage(ctx context.Context, imageID int) (*models.Image, error) {
|
||||||
|
var ret *models.Image
|
||||||
|
r := c.repository
|
||||||
|
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||||
|
qb := r.ImageFinder
|
||||||
|
|
||||||
|
var err error
|
||||||
|
ret, err = qb.Find(ctx, imageID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if ret == nil {
|
||||||
|
return fmt.Errorf("image with id %d not found", imageID)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = ret.LoadFiles(ctx, qb)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret.LoadURLs(ctx, qb)
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ret, nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,13 @@ type config struct {
|
||||||
// Configuration for querying a gallery by a URL
|
// Configuration for querying a gallery by a URL
|
||||||
GalleryByURL []*scrapeByURLConfig `yaml:"galleryByURL"`
|
GalleryByURL []*scrapeByURLConfig `yaml:"galleryByURL"`
|
||||||
|
|
||||||
|
// Configuration for querying an image by a URL
|
||||||
|
ImageByURL []*scrapeByURLConfig `yaml:"imageByURL"`
|
||||||
|
|
||||||
|
// Configuration for querying image by an Image fragment
|
||||||
|
ImageByFragment *scraperTypeConfig `yaml:"imageByFragment"`
|
||||||
|
|
||||||
|
// Configuration for querying a movie by a URL
|
||||||
// Configuration for querying a movie by a URL - deprecated, use GroupByURL
|
// Configuration for querying a movie by a URL - deprecated, use GroupByURL
|
||||||
MovieByURL []*scrapeByURLConfig `yaml:"movieByURL"`
|
MovieByURL []*scrapeByURLConfig `yaml:"movieByURL"`
|
||||||
GroupByURL []*scrapeByURLConfig `yaml:"groupByURL"`
|
GroupByURL []*scrapeByURLConfig `yaml:"groupByURL"`
|
||||||
|
|
@ -295,6 +302,21 @@ func (c config) spec() Scraper {
|
||||||
ret.Gallery = &gallery
|
ret.Gallery = &gallery
|
||||||
}
|
}
|
||||||
|
|
||||||
|
image := ScraperSpec{}
|
||||||
|
if c.ImageByFragment != nil {
|
||||||
|
image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeFragment)
|
||||||
|
}
|
||||||
|
if len(c.ImageByURL) > 0 {
|
||||||
|
image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeURL)
|
||||||
|
for _, v := range c.ImageByURL {
|
||||||
|
image.Urls = append(image.Urls, v.URL...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(image.SupportedScrapes) > 0 {
|
||||||
|
ret.Image = &image
|
||||||
|
}
|
||||||
|
|
||||||
group := ScraperSpec{}
|
group := ScraperSpec{}
|
||||||
if len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0 {
|
if len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0 {
|
||||||
group.SupportedScrapes = append(group.SupportedScrapes, ScrapeTypeURL)
|
group.SupportedScrapes = append(group.SupportedScrapes, ScrapeTypeURL)
|
||||||
|
|
@ -319,6 +341,8 @@ func (c config) supports(ty ScrapeContentType) bool {
|
||||||
return (c.SceneByName != nil && c.SceneByQueryFragment != nil) || c.SceneByFragment != nil || len(c.SceneByURL) > 0
|
return (c.SceneByName != nil && c.SceneByQueryFragment != nil) || c.SceneByFragment != nil || len(c.SceneByURL) > 0
|
||||||
case ScrapeContentTypeGallery:
|
case ScrapeContentTypeGallery:
|
||||||
return c.GalleryByFragment != nil || len(c.GalleryByURL) > 0
|
return c.GalleryByFragment != nil || len(c.GalleryByURL) > 0
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
return c.ImageByFragment != nil || len(c.ImageByURL) > 0
|
||||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||||
return len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0
|
return len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0
|
||||||
}
|
}
|
||||||
|
|
@ -346,6 +370,12 @@ func (c config) matchesURL(url string, ty ScrapeContentType) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
for _, scraper := range c.ImageByURL {
|
||||||
|
if scraper.matchesURL(url) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||||
for _, scraper := range c.MovieByURL {
|
for _, scraper := range c.MovieByURL {
|
||||||
if scraper.matchesURL(url) {
|
if scraper.matchesURL(url) {
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,9 @@ func (g group) fragmentScraper(input Input) *scraperTypeConfig {
|
||||||
case input.Gallery != nil:
|
case input.Gallery != nil:
|
||||||
// TODO - this should be galleryByQueryFragment
|
// TODO - this should be galleryByQueryFragment
|
||||||
return g.config.GalleryByFragment
|
return g.config.GalleryByFragment
|
||||||
|
case input.Image != nil:
|
||||||
|
// TODO - this should be imageByImageFragment
|
||||||
|
return g.config.ImageByFragment
|
||||||
case input.Scene != nil:
|
case input.Scene != nil:
|
||||||
return g.config.SceneByQueryFragment
|
return g.config.SceneByQueryFragment
|
||||||
}
|
}
|
||||||
|
|
@ -75,6 +78,15 @@ func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *mod
|
||||||
return s.scrapeGalleryByGallery(ctx, gallery)
|
return s.scrapeGalleryByGallery(ctx, gallery)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*ScrapedImage, error) {
|
||||||
|
if g.config.ImageByFragment == nil {
|
||||||
|
return nil, ErrNotSupported
|
||||||
|
}
|
||||||
|
|
||||||
|
s := g.config.getScraper(*g.config.ImageByFragment, client, g.globalConf)
|
||||||
|
return s.scrapeImageByImage(ctx, gallery)
|
||||||
|
}
|
||||||
|
|
||||||
func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
|
func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
|
||||||
switch ty {
|
switch ty {
|
||||||
case ScrapeContentTypePerformer:
|
case ScrapeContentTypePerformer:
|
||||||
|
|
@ -85,6 +97,8 @@ func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
|
||||||
return append(c.MovieByURL, c.GroupByURL...)
|
return append(c.MovieByURL, c.GroupByURL...)
|
||||||
case ScrapeContentTypeGallery:
|
case ScrapeContentTypeGallery:
|
||||||
return c.GalleryByURL
|
return c.GalleryByURL
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
return c.ImageByURL
|
||||||
}
|
}
|
||||||
|
|
||||||
panic("loadUrlCandidates: unreachable")
|
panic("loadUrlCandidates: unreachable")
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,28 @@ import (
|
||||||
"github.com/stashapp/stash/pkg/utils"
|
"github.com/stashapp/stash/pkg/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type ScrapedImage struct {
|
||||||
|
Title *string `json:"title"`
|
||||||
|
Code *string `json:"code"`
|
||||||
|
Details *string `json:"details"`
|
||||||
|
Photographer *string `json:"photographer"`
|
||||||
|
URLs []string `json:"urls"`
|
||||||
|
Date *string `json:"date"`
|
||||||
|
Studio *models.ScrapedStudio `json:"studio"`
|
||||||
|
Tags []*models.ScrapedTag `json:"tags"`
|
||||||
|
Performers []*models.ScrapedPerformer `json:"performers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ScrapedImage) IsScrapedContent() {}
|
||||||
|
|
||||||
|
type ScrapedImageInput struct {
|
||||||
|
Title *string `json:"title"`
|
||||||
|
Code *string `json:"code"`
|
||||||
|
Details *string `json:"details"`
|
||||||
|
URLs []string `json:"urls"`
|
||||||
|
Date *string `json:"date"`
|
||||||
|
}
|
||||||
|
|
||||||
func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error {
|
func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error {
|
||||||
// backwards compatibility: we fetch the image if it's a URL and set it to the first image
|
// backwards compatibility: we fetch the image if it's a URL and set it to the first image
|
||||||
// Image is deprecated, so only do this if Images is unset
|
// Image is deprecated, so only do this if Images is unset
|
||||||
|
|
|
||||||
|
|
@ -102,6 +102,12 @@ func (s *jsonScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCont
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return ret, nil
|
return ret, nil
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
ret, err := scraper.scrapeImage(ctx, q)
|
||||||
|
if err != nil || ret == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ret, nil
|
||||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||||
ret, err := scraper.scrapeGroup(ctx, q)
|
ret, err := scraper.scrapeGroup(ctx, q)
|
||||||
if err != nil || ret == nil {
|
if err != nil || ret == nil {
|
||||||
|
|
@ -225,6 +231,30 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
|
||||||
return scraper.scrapeScene(ctx, q)
|
return scraper.scrapeScene(ctx, q)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||||
|
// construct the URL
|
||||||
|
queryURL := queryURLParametersFromImage(image)
|
||||||
|
if s.scraper.QueryURLReplacements != nil {
|
||||||
|
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
|
||||||
|
}
|
||||||
|
url := queryURL.constructURL(s.scraper.QueryURL)
|
||||||
|
|
||||||
|
scraper := s.getJsonScraper()
|
||||||
|
|
||||||
|
if scraper == nil {
|
||||||
|
return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := s.loadURL(ctx, url)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
q := s.getJsonQuery(doc)
|
||||||
|
return scraper.scrapeImage(ctx, q)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||||
// construct the URL
|
// construct the URL
|
||||||
queryURL := queryURLParametersFromGallery(gallery)
|
queryURL := queryURLParametersFromGallery(gallery)
|
||||||
|
|
|
||||||
|
|
@ -181,6 +181,7 @@ type mappedGalleryScraperConfig struct {
|
||||||
Performers mappedConfig `yaml:"Performers"`
|
Performers mappedConfig `yaml:"Performers"`
|
||||||
Studio mappedConfig `yaml:"Studio"`
|
Studio mappedConfig `yaml:"Studio"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type _mappedGalleryScraperConfig mappedGalleryScraperConfig
|
type _mappedGalleryScraperConfig mappedGalleryScraperConfig
|
||||||
|
|
||||||
func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
|
@ -228,6 +229,60 @@ func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) e
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type mappedImageScraperConfig struct {
|
||||||
|
mappedConfig
|
||||||
|
|
||||||
|
Tags mappedConfig `yaml:"Tags"`
|
||||||
|
Performers mappedConfig `yaml:"Performers"`
|
||||||
|
Studio mappedConfig `yaml:"Studio"`
|
||||||
|
}
|
||||||
|
type _mappedImageScraperConfig mappedImageScraperConfig
|
||||||
|
|
||||||
|
func (s *mappedImageScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
// HACK - unmarshal to map first, then remove known scene sub-fields, then
|
||||||
|
// remarshal to yaml and pass that down to the base map
|
||||||
|
parentMap := make(map[string]interface{})
|
||||||
|
if err := unmarshal(parentMap); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// move the known sub-fields to a separate map
|
||||||
|
thisMap := make(map[string]interface{})
|
||||||
|
|
||||||
|
thisMap[mappedScraperConfigSceneTags] = parentMap[mappedScraperConfigSceneTags]
|
||||||
|
thisMap[mappedScraperConfigScenePerformers] = parentMap[mappedScraperConfigScenePerformers]
|
||||||
|
thisMap[mappedScraperConfigSceneStudio] = parentMap[mappedScraperConfigSceneStudio]
|
||||||
|
|
||||||
|
delete(parentMap, mappedScraperConfigSceneTags)
|
||||||
|
delete(parentMap, mappedScraperConfigScenePerformers)
|
||||||
|
delete(parentMap, mappedScraperConfigSceneStudio)
|
||||||
|
|
||||||
|
// re-unmarshal the sub-fields
|
||||||
|
yml, err := yaml.Marshal(thisMap)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// needs to be a different type to prevent infinite recursion
|
||||||
|
c := _mappedImageScraperConfig{}
|
||||||
|
if err := yaml.Unmarshal(yml, &c); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*s = mappedImageScraperConfig(c)
|
||||||
|
|
||||||
|
yml, err = yaml.Marshal(parentMap)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := yaml.Unmarshal(yml, &s.mappedConfig); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type mappedPerformerScraperConfig struct {
|
type mappedPerformerScraperConfig struct {
|
||||||
mappedConfig
|
mappedConfig
|
||||||
|
|
||||||
|
|
@ -785,6 +840,7 @@ type mappedScraper struct {
|
||||||
Common commonMappedConfig `yaml:"common"`
|
Common commonMappedConfig `yaml:"common"`
|
||||||
Scene *mappedSceneScraperConfig `yaml:"scene"`
|
Scene *mappedSceneScraperConfig `yaml:"scene"`
|
||||||
Gallery *mappedGalleryScraperConfig `yaml:"gallery"`
|
Gallery *mappedGalleryScraperConfig `yaml:"gallery"`
|
||||||
|
Image *mappedImageScraperConfig `yaml:"image"`
|
||||||
Performer *mappedPerformerScraperConfig `yaml:"performer"`
|
Performer *mappedPerformerScraperConfig `yaml:"performer"`
|
||||||
Movie *mappedMovieScraperConfig `yaml:"movie"`
|
Movie *mappedMovieScraperConfig `yaml:"movie"`
|
||||||
}
|
}
|
||||||
|
|
@ -1016,6 +1072,57 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*ScrapedImage, error) {
|
||||||
|
var ret ScrapedImage
|
||||||
|
|
||||||
|
imageScraperConfig := s.Image
|
||||||
|
if imageScraperConfig == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
imageMap := imageScraperConfig.mappedConfig
|
||||||
|
|
||||||
|
imagePerformersMap := imageScraperConfig.Performers
|
||||||
|
imageTagsMap := imageScraperConfig.Tags
|
||||||
|
imageStudioMap := imageScraperConfig.Studio
|
||||||
|
|
||||||
|
logger.Debug(`Processing image:`)
|
||||||
|
results := imageMap.process(ctx, q, s.Common)
|
||||||
|
|
||||||
|
// now apply the performers and tags
|
||||||
|
if imagePerformersMap != nil {
|
||||||
|
logger.Debug(`Processing image performers:`)
|
||||||
|
ret.Performers = processRelationships[models.ScrapedPerformer](ctx, s, imagePerformersMap, q)
|
||||||
|
}
|
||||||
|
|
||||||
|
if imageTagsMap != nil {
|
||||||
|
logger.Debug(`Processing image tags:`)
|
||||||
|
ret.Tags = processRelationships[models.ScrapedTag](ctx, s, imageTagsMap, q)
|
||||||
|
}
|
||||||
|
|
||||||
|
if imageStudioMap != nil {
|
||||||
|
logger.Debug(`Processing image studio:`)
|
||||||
|
studioResults := imageStudioMap.process(ctx, q, s.Common)
|
||||||
|
|
||||||
|
if len(studioResults) > 0 {
|
||||||
|
studio := &models.ScrapedStudio{}
|
||||||
|
studioResults[0].apply(studio)
|
||||||
|
ret.Studio = studio
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// if no basic fields are populated, and no relationships, then return nil
|
||||||
|
if len(results) == 0 && len(ret.Performers) == 0 && len(ret.Tags) == 0 && ret.Studio == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(results) > 0 {
|
||||||
|
results[0].apply(&ret)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ret, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
|
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
|
||||||
var ret ScrapedGallery
|
var ret ScrapedGallery
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,12 @@ func (c Cache) postScrape(ctx context.Context, content ScrapedContent) (ScrapedC
|
||||||
}
|
}
|
||||||
case ScrapedGallery:
|
case ScrapedGallery:
|
||||||
return c.postScrapeGallery(ctx, v)
|
return c.postScrapeGallery(ctx, v)
|
||||||
|
case *ScrapedImage:
|
||||||
|
if v != nil {
|
||||||
|
return c.postScrapeImage(ctx, *v)
|
||||||
|
}
|
||||||
|
case ScrapedImage:
|
||||||
|
return c.postScrapeImage(ctx, v)
|
||||||
case *models.ScrapedMovie:
|
case *models.ScrapedMovie:
|
||||||
if v != nil {
|
if v != nil {
|
||||||
return c.postScrapeMovie(ctx, *v)
|
return c.postScrapeMovie(ctx, *v)
|
||||||
|
|
@ -315,6 +321,40 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (Scraped
|
||||||
return g, nil
|
return g, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage) (ScrapedContent, error) {
|
||||||
|
r := c.repository
|
||||||
|
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||||
|
pqb := r.PerformerFinder
|
||||||
|
tqb := r.TagFinder
|
||||||
|
sqb := r.StudioFinder
|
||||||
|
|
||||||
|
for _, p := range image.Performers {
|
||||||
|
if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tags, err := postProcessTags(ctx, tqb, image.Tags)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
image.Tags = tags
|
||||||
|
|
||||||
|
if image.Studio != nil {
|
||||||
|
err := match.ScrapedStudio(ctx, sqb, image.Studio, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return image, nil
|
||||||
|
}
|
||||||
|
|
||||||
func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
|
func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
|
||||||
var ret []*models.ScrapedTag
|
var ret []*models.ScrapedTag
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,24 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters {
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func queryURLParametersFromImage(image *models.Image) queryURLParameters {
|
||||||
|
ret := make(queryURLParameters)
|
||||||
|
ret["checksum"] = image.Checksum
|
||||||
|
|
||||||
|
if image.Path != "" {
|
||||||
|
ret["filename"] = filepath.Base(image.Path)
|
||||||
|
}
|
||||||
|
if image.Title != "" {
|
||||||
|
ret["title"] = image.Title
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(image.URLs.List()) > 0 {
|
||||||
|
ret["url"] = image.URLs.List()[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret
|
||||||
|
}
|
||||||
|
|
||||||
func (p queryURLParameters) applyReplacements(r queryURLReplacements) {
|
func (p queryURLParameters) applyReplacements(r queryURLReplacements) {
|
||||||
for k, v := range p {
|
for k, v := range p {
|
||||||
rpl, found := r[k]
|
rpl, found := r[k]
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,7 @@ const (
|
||||||
ScrapeContentTypeGroup ScrapeContentType = "GROUP"
|
ScrapeContentTypeGroup ScrapeContentType = "GROUP"
|
||||||
ScrapeContentTypePerformer ScrapeContentType = "PERFORMER"
|
ScrapeContentTypePerformer ScrapeContentType = "PERFORMER"
|
||||||
ScrapeContentTypeScene ScrapeContentType = "SCENE"
|
ScrapeContentTypeScene ScrapeContentType = "SCENE"
|
||||||
|
ScrapeContentTypeImage ScrapeContentType = "IMAGE"
|
||||||
)
|
)
|
||||||
|
|
||||||
var AllScrapeContentType = []ScrapeContentType{
|
var AllScrapeContentType = []ScrapeContentType{
|
||||||
|
|
@ -44,11 +45,12 @@ var AllScrapeContentType = []ScrapeContentType{
|
||||||
ScrapeContentTypeGroup,
|
ScrapeContentTypeGroup,
|
||||||
ScrapeContentTypePerformer,
|
ScrapeContentTypePerformer,
|
||||||
ScrapeContentTypeScene,
|
ScrapeContentTypeScene,
|
||||||
|
ScrapeContentTypeImage,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e ScrapeContentType) IsValid() bool {
|
func (e ScrapeContentType) IsValid() bool {
|
||||||
switch e {
|
switch e {
|
||||||
case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene:
|
case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene, ScrapeContentTypeImage:
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
|
|
@ -84,6 +86,8 @@ type Scraper struct {
|
||||||
Scene *ScraperSpec `json:"scene"`
|
Scene *ScraperSpec `json:"scene"`
|
||||||
// Details for gallery scraper
|
// Details for gallery scraper
|
||||||
Gallery *ScraperSpec `json:"gallery"`
|
Gallery *ScraperSpec `json:"gallery"`
|
||||||
|
// Details for image scraper
|
||||||
|
Image *ScraperSpec `json:"image"`
|
||||||
// Details for movie scraper
|
// Details for movie scraper
|
||||||
Group *ScraperSpec `json:"group"`
|
Group *ScraperSpec `json:"group"`
|
||||||
// Details for movie scraper
|
// Details for movie scraper
|
||||||
|
|
@ -161,6 +165,7 @@ type Input struct {
|
||||||
Performer *ScrapedPerformerInput
|
Performer *ScrapedPerformerInput
|
||||||
Scene *ScrapedSceneInput
|
Scene *ScrapedSceneInput
|
||||||
Gallery *ScrapedGalleryInput
|
Gallery *ScrapedGalleryInput
|
||||||
|
Image *ScrapedImageInput
|
||||||
}
|
}
|
||||||
|
|
||||||
// populateURL populates the URL field of the input based on the
|
// populateURL populates the URL field of the input based on the
|
||||||
|
|
@ -225,6 +230,14 @@ type sceneScraper interface {
|
||||||
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
|
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// imageScraper is a scraper which supports image scrapes with
|
||||||
|
// image data as the input.
|
||||||
|
type imageScraper interface {
|
||||||
|
scraper
|
||||||
|
|
||||||
|
viaImage(ctx context.Context, client *http.Client, image *models.Image) (*ScrapedImage, error)
|
||||||
|
}
|
||||||
|
|
||||||
// galleryScraper is a scraper which supports gallery scrapes with
|
// galleryScraper is a scraper which supports gallery scrapes with
|
||||||
// gallery data as the input.
|
// gallery data as the input.
|
||||||
type galleryScraper interface {
|
type galleryScraper interface {
|
||||||
|
|
|
||||||
|
|
@ -388,6 +388,10 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
|
||||||
var movie *models.ScrapedMovie
|
var movie *models.ScrapedMovie
|
||||||
err := s.runScraperScript(ctx, input, &movie)
|
err := s.runScraperScript(ctx, input, &movie)
|
||||||
return movie, err
|
return movie, err
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
var image *ScrapedImage
|
||||||
|
err := s.runScraperScript(ctx, input, &image)
|
||||||
|
return image, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, ErrNotSupported
|
return nil, ErrNotSupported
|
||||||
|
|
@ -421,6 +425,20 @@ func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mod
|
||||||
return ret, err
|
return ret, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||||
|
inString, err := json.Marshal(imageToUpdateInput(image))
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var ret *ScrapedImage
|
||||||
|
|
||||||
|
err = s.runScraperScript(ctx, string(inString), &ret)
|
||||||
|
|
||||||
|
return ret, err
|
||||||
|
}
|
||||||
|
|
||||||
func handleScraperStderr(name string, scraperOutputReader io.ReadCloser) {
|
func handleScraperStderr(name string, scraperOutputReader io.ReadCloser) {
|
||||||
const scraperPrefix = "[Scrape / %s] "
|
const scraperPrefix = "[Scrape / %s] "
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -388,6 +388,33 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
||||||
return &ret, nil
|
return &ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||||
|
return nil, ErrNotSupported
|
||||||
|
}
|
||||||
|
|
||||||
func (s *stashScraper) scrapeByURL(_ context.Context, _ string, _ ScrapeContentType) (ScrapedContent, error) {
|
func (s *stashScraper) scrapeByURL(_ context.Context, _ string, _ ScrapeContentType) (ScrapedContent, error) {
|
||||||
return nil, ErrNotSupported
|
return nil, ErrNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func imageToUpdateInput(gallery *models.Image) models.ImageUpdateInput {
|
||||||
|
dateToStringPtr := func(s *models.Date) *string {
|
||||||
|
if s != nil {
|
||||||
|
v := s.String()
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to file basename if title is empty
|
||||||
|
title := gallery.GetTitle()
|
||||||
|
urls := gallery.URLs.List()
|
||||||
|
|
||||||
|
return models.ImageUpdateInput{
|
||||||
|
ID: strconv.Itoa(gallery.ID),
|
||||||
|
Title: &title,
|
||||||
|
Details: &gallery.Details,
|
||||||
|
Urls: urls,
|
||||||
|
Date: dateToStringPtr(gallery.Date),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -83,6 +83,12 @@ func (s *xpathScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCon
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return ret, nil
|
return ret, nil
|
||||||
|
case ScrapeContentTypeImage:
|
||||||
|
ret, err := scraper.scrapeImage(ctx, q)
|
||||||
|
if err != nil || ret == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ret, nil
|
||||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||||
ret, err := scraper.scrapeGroup(ctx, q)
|
ret, err := scraper.scrapeGroup(ctx, q)
|
||||||
if err != nil || ret == nil {
|
if err != nil || ret == nil {
|
||||||
|
|
@ -228,6 +234,30 @@ func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
||||||
return scraper.scrapeGallery(ctx, q)
|
return scraper.scrapeGallery(ctx, q)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||||
|
// construct the URL
|
||||||
|
queryURL := queryURLParametersFromImage(image)
|
||||||
|
if s.scraper.QueryURLReplacements != nil {
|
||||||
|
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
|
||||||
|
}
|
||||||
|
url := queryURL.constructURL(s.scraper.QueryURL)
|
||||||
|
|
||||||
|
scraper := s.getXpathScraper()
|
||||||
|
|
||||||
|
if scraper == nil {
|
||||||
|
return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config")
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := s.loadURL(ctx, url)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
q := s.getXPathQuery(doc)
|
||||||
|
return scraper.scrapeImage(ctx, q)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *xpathScraper) loadURL(ctx context.Context, url string) (*html.Node, error) {
|
func (s *xpathScraper) loadURL(ctx context.Context, url string) (*html.Node, error) {
|
||||||
r, err := loadURL(ctx, url, s.client, s.config, s.globalConfig)
|
r, err := loadURL(ctx, url, s.client, s.config, s.globalConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -205,6 +205,27 @@ fragment ScrapedGalleryData on ScrapedGallery {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fragment ScrapedImageData on ScrapedImage {
|
||||||
|
title
|
||||||
|
code
|
||||||
|
details
|
||||||
|
photographer
|
||||||
|
urls
|
||||||
|
date
|
||||||
|
|
||||||
|
studio {
|
||||||
|
...ScrapedSceneStudioData
|
||||||
|
}
|
||||||
|
|
||||||
|
tags {
|
||||||
|
...ScrapedSceneTagData
|
||||||
|
}
|
||||||
|
|
||||||
|
performers {
|
||||||
|
...ScrapedScenePerformerData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fragment ScrapedStashBoxSceneData on ScrapedScene {
|
fragment ScrapedStashBoxSceneData on ScrapedScene {
|
||||||
title
|
title
|
||||||
code
|
code
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,17 @@ query ListGalleryScrapers {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
query ListImageScrapers {
|
||||||
|
listScrapers(types: [IMAGE]) {
|
||||||
|
id
|
||||||
|
name
|
||||||
|
image {
|
||||||
|
urls
|
||||||
|
supported_scrapes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
query ListGroupScrapers {
|
query ListGroupScrapers {
|
||||||
listScrapers(types: [GROUP]) {
|
listScrapers(types: [GROUP]) {
|
||||||
id
|
id
|
||||||
|
|
@ -108,12 +119,27 @@ query ScrapeSingleGallery(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
query ScrapeSingleImage(
|
||||||
|
$source: ScraperSourceInput!
|
||||||
|
$input: ScrapeSingleImageInput!
|
||||||
|
) {
|
||||||
|
scrapeSingleImage(source: $source, input: $input) {
|
||||||
|
...ScrapedImageData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
query ScrapeGalleryURL($url: String!) {
|
query ScrapeGalleryURL($url: String!) {
|
||||||
scrapeGalleryURL(url: $url) {
|
scrapeGalleryURL(url: $url) {
|
||||||
...ScrapedGalleryData
|
...ScrapedGalleryData
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
query ScrapeImageURL($url: String!) {
|
||||||
|
scrapeImageURL(url: $url) {
|
||||||
|
...ScrapedImageData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
query ScrapeGroupURL($url: String!) {
|
query ScrapeGroupURL($url: String!) {
|
||||||
scrapeGroupURL(url: $url) {
|
scrapeGroupURL(url: $url) {
|
||||||
...ScrapedGroupData
|
...ScrapedGroupData
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import React, { useEffect, useState } from "react";
|
import React, { useEffect, useMemo, useState } from "react";
|
||||||
import { Button, Form, Col, Row } from "react-bootstrap";
|
|
||||||
import { FormattedMessage, useIntl } from "react-intl";
|
import { FormattedMessage, useIntl } from "react-intl";
|
||||||
|
import { Button, Form, Col, Row } from "react-bootstrap";
|
||||||
import Mousetrap from "mousetrap";
|
import Mousetrap from "mousetrap";
|
||||||
import * as GQL from "src/core/generated-graphql";
|
import * as GQL from "src/core/generated-graphql";
|
||||||
import * as yup from "yup";
|
import * as yup from "yup";
|
||||||
|
|
@ -19,6 +19,13 @@ import {
|
||||||
PerformerSelect,
|
PerformerSelect,
|
||||||
} from "src/components/Performers/PerformerSelect";
|
} from "src/components/Performers/PerformerSelect";
|
||||||
import { formikUtils } from "src/utils/form";
|
import { formikUtils } from "src/utils/form";
|
||||||
|
import {
|
||||||
|
queryScrapeImage,
|
||||||
|
queryScrapeImageURL,
|
||||||
|
useListImageScrapers,
|
||||||
|
mutateReloadScrapers,
|
||||||
|
} from "../../../core/StashService";
|
||||||
|
import { ImageScrapeDialog } from "./ImageScrapeDialog";
|
||||||
import { Studio, StudioSelect } from "src/components/Studios/StudioSelect";
|
import { Studio, StudioSelect } from "src/components/Studios/StudioSelect";
|
||||||
import { galleryTitle } from "src/core/galleries";
|
import { galleryTitle } from "src/core/galleries";
|
||||||
import {
|
import {
|
||||||
|
|
@ -27,6 +34,7 @@ import {
|
||||||
excludeFileBasedGalleries,
|
excludeFileBasedGalleries,
|
||||||
} from "src/components/Galleries/GallerySelect";
|
} from "src/components/Galleries/GallerySelect";
|
||||||
import { useTagsEdit } from "src/hooks/tagsEdit";
|
import { useTagsEdit } from "src/hooks/tagsEdit";
|
||||||
|
import { ScraperMenu } from "src/components/Shared/ScraperMenu";
|
||||||
|
|
||||||
interface IProps {
|
interface IProps {
|
||||||
image: GQL.ImageDataFragment;
|
image: GQL.ImageDataFragment;
|
||||||
|
|
@ -51,6 +59,8 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
const [performers, setPerformers] = useState<Performer[]>([]);
|
const [performers, setPerformers] = useState<Performer[]>([]);
|
||||||
const [studio, setStudio] = useState<Studio | null>(null);
|
const [studio, setStudio] = useState<Studio | null>(null);
|
||||||
|
|
||||||
|
const isNew = image.id === undefined;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setGalleries(
|
setGalleries(
|
||||||
image.galleries?.map((g) => ({
|
image.galleries?.map((g) => ({
|
||||||
|
|
@ -62,6 +72,9 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
);
|
);
|
||||||
}, [image.galleries]);
|
}, [image.galleries]);
|
||||||
|
|
||||||
|
const scrapers = useListImageScrapers();
|
||||||
|
const [scrapedImage, setScrapedImage] = useState<GQL.ScrapedImage | null>();
|
||||||
|
|
||||||
const schema = yup.object({
|
const schema = yup.object({
|
||||||
title: yup.string().ensure(),
|
title: yup.string().ensure(),
|
||||||
code: yup.string().ensure(),
|
code: yup.string().ensure(),
|
||||||
|
|
@ -97,8 +110,9 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
onSubmit: (values) => onSave(schema.cast(values)),
|
onSubmit: (values) => onSave(schema.cast(values)),
|
||||||
});
|
});
|
||||||
|
|
||||||
const { tagsControl } = useTagsEdit(image.tags, (ids) =>
|
const { tags, updateTagsStateFromScraper, tagsControl } = useTagsEdit(
|
||||||
formik.setFieldValue("tag_ids", ids)
|
image.tags,
|
||||||
|
(ids) => formik.setFieldValue("tag_ids", ids)
|
||||||
);
|
);
|
||||||
|
|
||||||
function onSetGalleries(items: Gallery[]) {
|
function onSetGalleries(items: Gallery[]) {
|
||||||
|
|
@ -148,6 +162,12 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const fragmentScrapers = useMemo(() => {
|
||||||
|
return (scrapers?.data?.listScrapers ?? []).filter((s) =>
|
||||||
|
s.image?.supported_scrapes.includes(GQL.ScrapeType.Fragment)
|
||||||
|
);
|
||||||
|
}, [scrapers]);
|
||||||
|
|
||||||
async function onSave(input: InputValues) {
|
async function onSave(input: InputValues) {
|
||||||
setIsLoading(true);
|
setIsLoading(true);
|
||||||
try {
|
try {
|
||||||
|
|
@ -162,6 +182,122 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
setIsLoading(false);
|
setIsLoading(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onScrapeClicked(s: GQL.ScraperSourceInput) {
|
||||||
|
if (!image || !image.id) return;
|
||||||
|
|
||||||
|
setIsLoading(true);
|
||||||
|
try {
|
||||||
|
const result = await queryScrapeImage(s.scraper_id!, image.id);
|
||||||
|
if (!result.data || !result.data.scrapeSingleImage?.length) {
|
||||||
|
Toast.success("No images found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setScrapedImage(result.data.scrapeSingleImage[0]);
|
||||||
|
} catch (e) {
|
||||||
|
Toast.error(e);
|
||||||
|
} finally {
|
||||||
|
setIsLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function urlScrapable(scrapedUrl: string): boolean {
|
||||||
|
return (scrapers?.data?.listScrapers ?? []).some((s) =>
|
||||||
|
(s?.image?.urls ?? []).some((u) => scrapedUrl.includes(u))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateImageFromScrapedGallery(
|
||||||
|
imageData: GQL.ScrapedImageDataFragment
|
||||||
|
) {
|
||||||
|
if (imageData.title) {
|
||||||
|
formik.setFieldValue("title", imageData.title);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.code) {
|
||||||
|
formik.setFieldValue("code", imageData.code);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.details) {
|
||||||
|
formik.setFieldValue("details", imageData.details);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.photographer) {
|
||||||
|
formik.setFieldValue("photographer", imageData.photographer);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.date) {
|
||||||
|
formik.setFieldValue("date", imageData.date);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.urls) {
|
||||||
|
formik.setFieldValue("urls", imageData.urls);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.studio?.stored_id) {
|
||||||
|
onSetStudio({
|
||||||
|
id: imageData.studio.stored_id,
|
||||||
|
name: imageData.studio.name ?? "",
|
||||||
|
aliases: [],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (imageData.performers?.length) {
|
||||||
|
const idPerfs = imageData.performers.filter((p) => {
|
||||||
|
return p.stored_id !== undefined && p.stored_id !== null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (idPerfs.length > 0) {
|
||||||
|
onSetPerformers(
|
||||||
|
idPerfs.map((p) => {
|
||||||
|
return {
|
||||||
|
id: p.stored_id!,
|
||||||
|
name: p.name ?? "",
|
||||||
|
alias_list: [],
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
updateTagsStateFromScraper(imageData.tags ?? undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onReloadScrapers() {
|
||||||
|
setIsLoading(true);
|
||||||
|
try {
|
||||||
|
await mutateReloadScrapers();
|
||||||
|
} catch (e) {
|
||||||
|
Toast.error(e);
|
||||||
|
} finally {
|
||||||
|
setIsLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onScrapeDialogClosed(data?: GQL.ScrapedImageDataFragment) {
|
||||||
|
if (data) {
|
||||||
|
updateImageFromScrapedGallery(data);
|
||||||
|
}
|
||||||
|
setScrapedImage(undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onScrapeImageURL(url: string) {
|
||||||
|
if (!url) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setIsLoading(true);
|
||||||
|
try {
|
||||||
|
const result = await queryScrapeImageURL(url);
|
||||||
|
if (!result || !result.data || !result.data.scrapeImageURL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setScrapedImage(result.data.scrapeImageURL);
|
||||||
|
} catch (e) {
|
||||||
|
Toast.error(e);
|
||||||
|
} finally {
|
||||||
|
setIsLoading(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (isLoading) return <LoadingIndicator />;
|
if (isLoading) return <LoadingIndicator />;
|
||||||
|
|
||||||
const splitProps = {
|
const splitProps = {
|
||||||
|
|
@ -243,6 +379,30 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
return renderInputField("details", "textarea", "details", props);
|
return renderInputField("details", "textarea", "details", props);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function maybeRenderScrapeDialog() {
|
||||||
|
if (!scrapedImage) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentImage = {
|
||||||
|
id: image.id!,
|
||||||
|
...formik.values,
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ImageScrapeDialog
|
||||||
|
image={currentImage}
|
||||||
|
imageStudio={studio}
|
||||||
|
imageTags={tags}
|
||||||
|
imagePerformers={performers}
|
||||||
|
scraped={scrapedImage}
|
||||||
|
onClose={(data) => {
|
||||||
|
onScrapeDialogClosed(data);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div id="image-edit-details">
|
<div id="image-edit-details">
|
||||||
<Prompt
|
<Prompt
|
||||||
|
|
@ -250,13 +410,16 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
message={intl.formatMessage({ id: "dialogs.unsaved_changes" })}
|
message={intl.formatMessage({ id: "dialogs.unsaved_changes" })}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
{maybeRenderScrapeDialog()}
|
||||||
<Form noValidate onSubmit={formik.handleSubmit}>
|
<Form noValidate onSubmit={formik.handleSubmit}>
|
||||||
<Row className="form-container edit-buttons-container px-3 pt-3">
|
<Row className="form-container edit-buttons-container px-3 pt-3">
|
||||||
<div className="edit-buttons mb-3 pl-0">
|
<div className="edit-buttons mb-3 pl-0">
|
||||||
<Button
|
<Button
|
||||||
className="edit-button"
|
className="edit-button"
|
||||||
variant="primary"
|
variant="primary"
|
||||||
disabled={!formik.dirty || !isEqual(formik.errors, {})}
|
disabled={
|
||||||
|
(!isNew && !formik.dirty) || !isEqual(formik.errors, {})
|
||||||
|
}
|
||||||
onClick={() => formik.submitForm()}
|
onClick={() => formik.submitForm()}
|
||||||
>
|
>
|
||||||
<FormattedMessage id="actions.save" />
|
<FormattedMessage id="actions.save" />
|
||||||
|
|
@ -269,13 +432,23 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
||||||
<FormattedMessage id="actions.delete" />
|
<FormattedMessage id="actions.delete" />
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="ml-auto text-right d-flex">
|
||||||
|
{!isNew && (
|
||||||
|
<ScraperMenu
|
||||||
|
toggle={intl.formatMessage({ id: "actions.scrape_with" })}
|
||||||
|
scrapers={fragmentScrapers}
|
||||||
|
onScraperClicked={onScrapeClicked}
|
||||||
|
onReloadScrapers={onReloadScrapers}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
</Row>
|
</Row>
|
||||||
<Row className="form-container px-3">
|
<Row className="form-container px-3">
|
||||||
<Col lg={7} xl={12}>
|
<Col lg={7} xl={12}>
|
||||||
{renderInputField("title")}
|
{renderInputField("title")}
|
||||||
{renderInputField("code", "text", "scene_code")}
|
{renderInputField("code", "text", "scene_code")}
|
||||||
|
|
||||||
{renderURLListField("urls")}
|
{renderURLListField("urls", onScrapeImageURL, urlScrapable)}
|
||||||
|
|
||||||
{renderDateField("date")}
|
{renderDateField("date")}
|
||||||
{renderInputField("photographer")}
|
{renderInputField("photographer")}
|
||||||
|
|
|
||||||
227
ui/v2.5/src/components/Images/ImageDetails/ImageScrapeDialog.tsx
Normal file
227
ui/v2.5/src/components/Images/ImageDetails/ImageScrapeDialog.tsx
Normal file
|
|
@ -0,0 +1,227 @@
|
||||||
|
import React, { useState } from "react";
|
||||||
|
import { useIntl } from "react-intl";
|
||||||
|
import * as GQL from "src/core/generated-graphql";
|
||||||
|
import {
|
||||||
|
ScrapeDialog,
|
||||||
|
ScrapedInputGroupRow,
|
||||||
|
ScrapedStringListRow,
|
||||||
|
ScrapedTextAreaRow,
|
||||||
|
} from "src/components/Shared/ScrapeDialog/ScrapeDialog";
|
||||||
|
import {
|
||||||
|
ObjectListScrapeResult,
|
||||||
|
ObjectScrapeResult,
|
||||||
|
ScrapeResult,
|
||||||
|
} from "src/components/Shared/ScrapeDialog/scrapeResult";
|
||||||
|
import {
|
||||||
|
ScrapedPerformersRow,
|
||||||
|
ScrapedStudioRow,
|
||||||
|
} from "src/components/Shared/ScrapeDialog/ScrapedObjectsRow";
|
||||||
|
import { sortStoredIdObjects } from "src/utils/data";
|
||||||
|
import { Performer } from "src/components/Performers/PerformerSelect";
|
||||||
|
import {
|
||||||
|
useCreateScrapedPerformer,
|
||||||
|
useCreateScrapedStudio,
|
||||||
|
} from "src/components/Shared/ScrapeDialog/createObjects";
|
||||||
|
import { uniq } from "lodash-es";
|
||||||
|
import { Tag } from "src/components/Tags/TagSelect";
|
||||||
|
import { Studio } from "src/components/Studios/StudioSelect";
|
||||||
|
import { useScrapedTags } from "src/components/Shared/ScrapeDialog/scrapedTags";
|
||||||
|
|
||||||
|
interface IImageScrapeDialogProps {
|
||||||
|
image: Partial<GQL.ImageUpdateInput>;
|
||||||
|
imageStudio: Studio | null;
|
||||||
|
imageTags: Tag[];
|
||||||
|
imagePerformers: Performer[];
|
||||||
|
scraped: GQL.ScrapedImage;
|
||||||
|
|
||||||
|
onClose: (scrapedImage?: GQL.ScrapedImage) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const ImageScrapeDialog: React.FC<IImageScrapeDialogProps> = ({
|
||||||
|
image,
|
||||||
|
imageStudio,
|
||||||
|
imageTags,
|
||||||
|
imagePerformers,
|
||||||
|
scraped,
|
||||||
|
onClose,
|
||||||
|
}) => {
|
||||||
|
const intl = useIntl();
|
||||||
|
const [title, setTitle] = useState<ScrapeResult<string>>(
|
||||||
|
new ScrapeResult<string>(image.title, scraped.title)
|
||||||
|
);
|
||||||
|
const [code, setCode] = useState<ScrapeResult<string>>(
|
||||||
|
new ScrapeResult<string>(image.code, scraped.code)
|
||||||
|
);
|
||||||
|
const [urls, setURLs] = useState<ScrapeResult<string[]>>(
|
||||||
|
new ScrapeResult<string[]>(
|
||||||
|
image.urls,
|
||||||
|
scraped.urls
|
||||||
|
? uniq((image.urls ?? []).concat(scraped.urls ?? []))
|
||||||
|
: undefined
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const [date, setDate] = useState<ScrapeResult<string>>(
|
||||||
|
new ScrapeResult<string>(image.date, scraped.date)
|
||||||
|
);
|
||||||
|
|
||||||
|
const [photographer, setPhotographer] = useState<ScrapeResult<string>>(
|
||||||
|
new ScrapeResult<string>(image.photographer, scraped.photographer)
|
||||||
|
);
|
||||||
|
|
||||||
|
const [studio, setStudio] = useState<ObjectScrapeResult<GQL.ScrapedStudio>>(
|
||||||
|
new ObjectScrapeResult<GQL.ScrapedStudio>(
|
||||||
|
imageStudio
|
||||||
|
? {
|
||||||
|
stored_id: imageStudio.id,
|
||||||
|
name: imageStudio.name,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
scraped.studio?.stored_id ? scraped.studio : undefined
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const [newStudio, setNewStudio] = useState<GQL.ScrapedStudio | undefined>(
|
||||||
|
scraped.studio && !scraped.studio.stored_id ? scraped.studio : undefined
|
||||||
|
);
|
||||||
|
|
||||||
|
const [performers, setPerformers] = useState<
|
||||||
|
ObjectListScrapeResult<GQL.ScrapedPerformer>
|
||||||
|
>(
|
||||||
|
new ObjectListScrapeResult<GQL.ScrapedPerformer>(
|
||||||
|
sortStoredIdObjects(
|
||||||
|
imagePerformers.map((p) => ({
|
||||||
|
stored_id: p.id,
|
||||||
|
name: p.name,
|
||||||
|
}))
|
||||||
|
),
|
||||||
|
sortStoredIdObjects(scraped.performers ?? undefined)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const [newPerformers, setNewPerformers] = useState<GQL.ScrapedPerformer[]>(
|
||||||
|
scraped.performers?.filter((t) => !t.stored_id) ?? []
|
||||||
|
);
|
||||||
|
|
||||||
|
const { tags, newTags, scrapedTagsRow } = useScrapedTags(
|
||||||
|
imageTags,
|
||||||
|
scraped.tags
|
||||||
|
);
|
||||||
|
|
||||||
|
const [details, setDetails] = useState<ScrapeResult<string>>(
|
||||||
|
new ScrapeResult<string>(image.details, scraped.details)
|
||||||
|
);
|
||||||
|
|
||||||
|
const createNewStudio = useCreateScrapedStudio({
|
||||||
|
scrapeResult: studio,
|
||||||
|
setScrapeResult: setStudio,
|
||||||
|
setNewObject: setNewStudio,
|
||||||
|
});
|
||||||
|
|
||||||
|
const createNewPerformer = useCreateScrapedPerformer({
|
||||||
|
scrapeResult: performers,
|
||||||
|
setScrapeResult: setPerformers,
|
||||||
|
newObjects: newPerformers,
|
||||||
|
setNewObjects: setNewPerformers,
|
||||||
|
});
|
||||||
|
|
||||||
|
// don't show the dialog if nothing was scraped
|
||||||
|
if (
|
||||||
|
[
|
||||||
|
title,
|
||||||
|
code,
|
||||||
|
urls,
|
||||||
|
date,
|
||||||
|
photographer,
|
||||||
|
studio,
|
||||||
|
performers,
|
||||||
|
tags,
|
||||||
|
details,
|
||||||
|
].every((r) => !r.scraped) &&
|
||||||
|
!newStudio &&
|
||||||
|
newPerformers.length === 0 &&
|
||||||
|
newTags.length === 0
|
||||||
|
) {
|
||||||
|
onClose();
|
||||||
|
return <></>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeNewScrapedItem(): GQL.ScrapedImageDataFragment {
|
||||||
|
const newStudioValue = studio.getNewValue();
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: title.getNewValue(),
|
||||||
|
code: code.getNewValue(),
|
||||||
|
urls: urls.getNewValue(),
|
||||||
|
date: date.getNewValue(),
|
||||||
|
photographer: photographer.getNewValue(),
|
||||||
|
studio: newStudioValue,
|
||||||
|
performers: performers.getNewValue(),
|
||||||
|
tags: tags.getNewValue(),
|
||||||
|
details: details.getNewValue(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderScrapeRows() {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<ScrapedInputGroupRow
|
||||||
|
title={intl.formatMessage({ id: "title" })}
|
||||||
|
result={title}
|
||||||
|
onChange={(value) => setTitle(value)}
|
||||||
|
/>
|
||||||
|
<ScrapedInputGroupRow
|
||||||
|
title={intl.formatMessage({ id: "scene_code" })}
|
||||||
|
result={code}
|
||||||
|
onChange={(value) => setCode(value)}
|
||||||
|
/>
|
||||||
|
<ScrapedStringListRow
|
||||||
|
title={intl.formatMessage({ id: "urls" })}
|
||||||
|
result={urls}
|
||||||
|
onChange={(value) => setURLs(value)}
|
||||||
|
/>
|
||||||
|
<ScrapedInputGroupRow
|
||||||
|
title={intl.formatMessage({ id: "date" })}
|
||||||
|
placeholder="YYYY-MM-DD"
|
||||||
|
result={date}
|
||||||
|
onChange={(value) => setDate(value)}
|
||||||
|
/>
|
||||||
|
<ScrapedInputGroupRow
|
||||||
|
title={intl.formatMessage({ id: "photographer" })}
|
||||||
|
result={photographer}
|
||||||
|
onChange={(value) => setPhotographer(value)}
|
||||||
|
/>
|
||||||
|
<ScrapedStudioRow
|
||||||
|
title={intl.formatMessage({ id: "studios" })}
|
||||||
|
result={studio}
|
||||||
|
onChange={(value) => setStudio(value)}
|
||||||
|
newStudio={newStudio}
|
||||||
|
onCreateNew={createNewStudio}
|
||||||
|
/>
|
||||||
|
<ScrapedPerformersRow
|
||||||
|
title={intl.formatMessage({ id: "performers" })}
|
||||||
|
result={performers}
|
||||||
|
onChange={(value) => setPerformers(value)}
|
||||||
|
newObjects={newPerformers}
|
||||||
|
onCreateNew={createNewPerformer}
|
||||||
|
/>
|
||||||
|
{scrapedTagsRow}
|
||||||
|
<ScrapedTextAreaRow
|
||||||
|
title={intl.formatMessage({ id: "details" })}
|
||||||
|
result={details}
|
||||||
|
onChange={(value) => setDetails(value)}
|
||||||
|
/>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ScrapeDialog
|
||||||
|
title={intl.formatMessage(
|
||||||
|
{ id: "dialogs.scrape_entity_title" },
|
||||||
|
{ entity_type: intl.formatMessage({ id: "image" }) }
|
||||||
|
)}
|
||||||
|
renderScrapeRows={renderScrapeRows}
|
||||||
|
onClose={(apply) => {
|
||||||
|
onClose(apply ? makeNewScrapedItem() : undefined);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
@ -7,6 +7,7 @@ import {
|
||||||
useListPerformerScrapers,
|
useListPerformerScrapers,
|
||||||
useListSceneScrapers,
|
useListSceneScrapers,
|
||||||
useListGalleryScrapers,
|
useListGalleryScrapers,
|
||||||
|
useListImageScrapers,
|
||||||
} from "src/core/StashService";
|
} from "src/core/StashService";
|
||||||
import { useToast } from "src/hooks/Toast";
|
import { useToast } from "src/hooks/Toast";
|
||||||
import TextUtils from "src/utils/text";
|
import TextUtils from "src/utils/text";
|
||||||
|
|
@ -178,6 +179,8 @@ const ScrapersSection: React.FC = () => {
|
||||||
useListSceneScrapers();
|
useListSceneScrapers();
|
||||||
const { data: galleryScrapers, loading: loadingGalleries } =
|
const { data: galleryScrapers, loading: loadingGalleries } =
|
||||||
useListGalleryScrapers();
|
useListGalleryScrapers();
|
||||||
|
const { data: imageScrapers, loading: loadingImages } =
|
||||||
|
useListImageScrapers();
|
||||||
const { data: groupScrapers, loading: loadingGroups } =
|
const { data: groupScrapers, loading: loadingGroups } =
|
||||||
useListGroupScrapers();
|
useListGroupScrapers();
|
||||||
|
|
||||||
|
|
@ -193,6 +196,9 @@ const ScrapersSection: React.FC = () => {
|
||||||
galleries: galleryScrapers?.listScrapers.filter((s) =>
|
galleries: galleryScrapers?.listScrapers.filter((s) =>
|
||||||
filterFn(s.name, s.gallery?.urls)
|
filterFn(s.name, s.gallery?.urls)
|
||||||
),
|
),
|
||||||
|
images: imageScrapers?.listScrapers.filter((s) =>
|
||||||
|
filterFn(s.name, s.image?.urls)
|
||||||
|
),
|
||||||
groups: groupScrapers?.listScrapers.filter((s) =>
|
groups: groupScrapers?.listScrapers.filter((s) =>
|
||||||
filterFn(s.name, s.group?.urls)
|
filterFn(s.name, s.group?.urls)
|
||||||
),
|
),
|
||||||
|
|
@ -201,6 +207,7 @@ const ScrapersSection: React.FC = () => {
|
||||||
performerScrapers,
|
performerScrapers,
|
||||||
sceneScrapers,
|
sceneScrapers,
|
||||||
galleryScrapers,
|
galleryScrapers,
|
||||||
|
imageScrapers,
|
||||||
groupScrapers,
|
groupScrapers,
|
||||||
filter,
|
filter,
|
||||||
]);
|
]);
|
||||||
|
|
@ -213,7 +220,13 @@ const ScrapersSection: React.FC = () => {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (loadingScenes || loadingGalleries || loadingPerformers || loadingGroups)
|
if (
|
||||||
|
loadingScenes ||
|
||||||
|
loadingGalleries ||
|
||||||
|
loadingPerformers ||
|
||||||
|
loadingGroups ||
|
||||||
|
loadingImages
|
||||||
|
)
|
||||||
return (
|
return (
|
||||||
<SettingSection headingID="config.scraping.scrapers">
|
<SettingSection headingID="config.scraping.scrapers">
|
||||||
<LoadingIndicator />
|
<LoadingIndicator />
|
||||||
|
|
@ -274,6 +287,23 @@ const ScrapersSection: React.FC = () => {
|
||||||
</ScraperTable>
|
</ScraperTable>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{!!filteredScrapers.images?.length && (
|
||||||
|
<ScraperTable
|
||||||
|
entityType="image"
|
||||||
|
count={filteredScrapers.images?.length}
|
||||||
|
>
|
||||||
|
{filteredScrapers.images?.map((scraper) => (
|
||||||
|
<ScraperTableRow
|
||||||
|
key={scraper.id}
|
||||||
|
name={scraper.name}
|
||||||
|
entityType="image"
|
||||||
|
supportedScrapes={scraper.image?.supported_scrapes ?? []}
|
||||||
|
urls={scraper.image?.urls ?? []}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</ScraperTable>
|
||||||
|
)}
|
||||||
|
|
||||||
{!!filteredScrapers.performers?.length && (
|
{!!filteredScrapers.performers?.length && (
|
||||||
<ScraperTable
|
<ScraperTable
|
||||||
entityType="performer"
|
entityType="performer"
|
||||||
|
|
|
||||||
|
|
@ -2291,6 +2291,8 @@ export const queryScrapeGroupURL = (url: string) =>
|
||||||
|
|
||||||
export const useListGalleryScrapers = () => GQL.useListGalleryScrapersQuery();
|
export const useListGalleryScrapers = () => GQL.useListGalleryScrapersQuery();
|
||||||
|
|
||||||
|
export const useListImageScrapers = () => GQL.useListImageScrapersQuery();
|
||||||
|
|
||||||
export const queryScrapeGallery = (scraperId: string, galleryId: string) =>
|
export const queryScrapeGallery = (scraperId: string, galleryId: string) =>
|
||||||
client.query<GQL.ScrapeSingleGalleryQuery>({
|
client.query<GQL.ScrapeSingleGalleryQuery>({
|
||||||
query: GQL.ScrapeSingleGalleryDocument,
|
query: GQL.ScrapeSingleGalleryDocument,
|
||||||
|
|
@ -2312,6 +2314,27 @@ export const queryScrapeGalleryURL = (url: string) =>
|
||||||
fetchPolicy: "network-only",
|
fetchPolicy: "network-only",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const queryScrapeImage = (scraperId: string, imageId: string) =>
|
||||||
|
client.query<GQL.ScrapeSingleImageQuery>({
|
||||||
|
query: GQL.ScrapeSingleImageDocument,
|
||||||
|
variables: {
|
||||||
|
source: {
|
||||||
|
scraper_id: scraperId,
|
||||||
|
},
|
||||||
|
input: {
|
||||||
|
image_id: imageId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fetchPolicy: "network-only",
|
||||||
|
});
|
||||||
|
|
||||||
|
export const queryScrapeImageURL = (url: string) =>
|
||||||
|
client.query<GQL.ScrapeImageUrlQuery>({
|
||||||
|
query: GQL.ScrapeImageUrlDocument,
|
||||||
|
variables: { url },
|
||||||
|
fetchPolicy: "network-only",
|
||||||
|
});
|
||||||
|
|
||||||
export const mutateSubmitStashBoxSceneDraft = (
|
export const mutateSubmitStashBoxSceneDraft = (
|
||||||
input: GQL.StashBoxDraftSubmissionInput
|
input: GQL.StashBoxDraftSubmissionInput
|
||||||
) =>
|
) =>
|
||||||
|
|
@ -2383,6 +2406,7 @@ export const scraperMutationImpactedQueries = [
|
||||||
GQL.ListGroupScrapersDocument,
|
GQL.ListGroupScrapersDocument,
|
||||||
GQL.ListPerformerScrapersDocument,
|
GQL.ListPerformerScrapersDocument,
|
||||||
GQL.ListSceneScrapersDocument,
|
GQL.ListSceneScrapersDocument,
|
||||||
|
GQL.ListImageScrapersDocument,
|
||||||
GQL.InstalledScraperPackagesDocument,
|
GQL.InstalledScraperPackagesDocument,
|
||||||
GQL.InstalledScraperPackagesStatusDocument,
|
GQL.InstalledScraperPackagesStatusDocument,
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,10 @@ galleryByFragment:
|
||||||
<single scraper config>
|
<single scraper config>
|
||||||
galleryByURL:
|
galleryByURL:
|
||||||
<multiple scraper URL configs>
|
<multiple scraper URL configs>
|
||||||
|
imageByFragment:
|
||||||
|
<single scraper config>
|
||||||
|
imageByURL:
|
||||||
|
<multiple scraper URL configs>
|
||||||
<other configurations>
|
<other configurations>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -81,6 +85,8 @@ The script is sent input and expects output based on the scraping type, as detai
|
||||||
| `groupByURL` | `{"url": "<url>"}` | JSON-encoded group fragment |
|
| `groupByURL` | `{"url": "<url>"}` | JSON-encoded group fragment |
|
||||||
| `galleryByFragment` | JSON-encoded gallery fragment | JSON-encoded gallery fragment |
|
| `galleryByFragment` | JSON-encoded gallery fragment | JSON-encoded gallery fragment |
|
||||||
| `galleryByURL` | `{"url": "<url>"}` | JSON-encoded gallery fragment |
|
| `galleryByURL` | `{"url": "<url>"}` | JSON-encoded gallery fragment |
|
||||||
|
| `imageByFragment` | JSON-encoded image fragment | JSON-encoded image fragment |
|
||||||
|
| `imageByURL` | `{"url": "<url>"}` | JSON-encoded image fragment |
|
||||||
|
|
||||||
For `performerByName`, only `name` is required in the returned performer fragments. One entire object is sent back to `performerByFragment` to scrape a specific performer, so the other fields may be included to assist in scraping a performer. For example, the `url` field may be filled in for the specific performer page, then `performerByFragment` can extract by using its value.
|
For `performerByName`, only `name` is required in the returned performer fragments. One entire object is sent back to `performerByFragment` to scrape a specific performer, so the other fields may be included to assist in scraping a performer. For example, the `url` field may be filled in for the specific performer page, then `performerByFragment` can extract by using its value.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ Stash supports scraping of metadata from various external sources.
|
||||||
| | Fragment | Search | URL |
|
| | Fragment | Search | URL |
|
||||||
|---|:---:|:---:|:---:|
|
|---|:---:|:---:|:---:|
|
||||||
| gallery | ✔️ | | ✔️ |
|
| gallery | ✔️ | | ✔️ |
|
||||||
|
| image | ✔️ | | ✔️ |
|
||||||
| group | | | ✔️ |
|
| group | | | ✔️ |
|
||||||
| performer | | ✔️ | ✔️ |
|
| performer | | ✔️ | ✔️ |
|
||||||
| scene | ✔️ | ✔️ | ✔️ |
|
| scene | ✔️ | ✔️ | ✔️ |
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue