mirror of
https://github.com/stashapp/stash.git
synced 2025-12-06 08:26:00 +01:00
Add Image Scraping (#5562)
Co-authored-by: keenbed <155155956+keenbed@users.noreply.github.com> Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
WeedLordVegeta420
GitHub
parent
b6ace42973
commit
e97f647a43
27 changed files with 1063 additions and 11 deletions
|
|
@ -174,6 +174,12 @@ type Query {
|
|||
input: ScrapeSingleGroupInput!
|
||||
): [ScrapedGroup!]!
|
||||
|
||||
"Scrape for a single image"
|
||||
scrapeSingleImage(
|
||||
source: ScraperSourceInput!
|
||||
input: ScrapeSingleImageInput!
|
||||
): [ScrapedImage!]!
|
||||
|
||||
"Scrapes content based on a URL"
|
||||
scrapeURL(url: String!, ty: ScrapeContentType!): ScrapedContent
|
||||
|
||||
|
|
@ -183,6 +189,8 @@ type Query {
|
|||
scrapeSceneURL(url: String!): ScrapedScene
|
||||
"Scrapes a complete gallery record based on a URL"
|
||||
scrapeGalleryURL(url: String!): ScrapedGallery
|
||||
"Scrapes a complete image record based on a URL"
|
||||
scrapeImageURL(url: String!): ScrapedImage
|
||||
"Scrapes a complete movie record based on a URL"
|
||||
scrapeMovieURL(url: String!): ScrapedMovie
|
||||
@deprecated(reason: "Use scrapeGroupURL instead")
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ enum ScrapeType {
|
|||
"Type of the content a scraper generates"
|
||||
enum ScrapeContentType {
|
||||
GALLERY
|
||||
IMAGE
|
||||
MOVIE
|
||||
GROUP
|
||||
PERFORMER
|
||||
|
|
@ -22,6 +23,7 @@ union ScrapedContent =
|
|||
| ScrapedTag
|
||||
| ScrapedScene
|
||||
| ScrapedGallery
|
||||
| ScrapedImage
|
||||
| ScrapedMovie
|
||||
| ScrapedGroup
|
||||
| ScrapedPerformer
|
||||
|
|
@ -41,6 +43,8 @@ type Scraper {
|
|||
scene: ScraperSpec
|
||||
"Details for gallery scraper"
|
||||
gallery: ScraperSpec
|
||||
"Details for image scraper"
|
||||
image: ScraperSpec
|
||||
"Details for movie scraper"
|
||||
movie: ScraperSpec @deprecated(reason: "use group")
|
||||
"Details for group scraper"
|
||||
|
|
@ -128,6 +132,26 @@ input ScrapedGalleryInput {
|
|||
# no studio, tags or performers
|
||||
}
|
||||
|
||||
type ScrapedImage {
|
||||
title: String
|
||||
code: String
|
||||
details: String
|
||||
photographer: String
|
||||
urls: [String!]
|
||||
date: String
|
||||
studio: ScrapedStudio
|
||||
tags: [ScrapedTag!]
|
||||
performers: [ScrapedPerformer!]
|
||||
}
|
||||
|
||||
input ScrapedImageInput {
|
||||
title: String
|
||||
code: String
|
||||
details: String
|
||||
urls: [String!]
|
||||
date: String
|
||||
}
|
||||
|
||||
input ScraperSourceInput {
|
||||
"Index of the configured stash-box instance to use. Should be unset if scraper_id is set"
|
||||
stash_box_index: Int @deprecated(reason: "use stash_box_endpoint")
|
||||
|
|
@ -190,6 +214,15 @@ input ScrapeSingleGalleryInput {
|
|||
gallery_input: ScrapedGalleryInput
|
||||
}
|
||||
|
||||
input ScrapeSingleImageInput {
|
||||
"Instructs to query by string"
|
||||
query: String
|
||||
"Instructs to query by image id"
|
||||
image_id: ID
|
||||
"Instructs to query by image fragment"
|
||||
image_input: ScrapedImageInput
|
||||
}
|
||||
|
||||
input ScrapeSingleMovieInput {
|
||||
"Instructs to query by string"
|
||||
query: String
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ func (r *mutationResolver) getImage(ctx context.Context, id int) (ret *models.Im
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInput) (ret *models.Image, err error) {
|
||||
func (r *mutationResolver) ImageUpdate(ctx context.Context, input models.ImageUpdateInput) (ret *models.Image, err error) {
|
||||
translator := changesetTranslator{
|
||||
inputMap: getUpdateInputMap(ctx),
|
||||
}
|
||||
|
|
@ -46,7 +46,7 @@ func (r *mutationResolver) ImageUpdate(ctx context.Context, input ImageUpdateInp
|
|||
return r.getImage(ctx, ret.ID)
|
||||
}
|
||||
|
||||
func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdateInput) (ret []*models.Image, err error) {
|
||||
func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*models.ImageUpdateInput) (ret []*models.Image, err error) {
|
||||
inputMaps := getUpdateInputMaps(ctx)
|
||||
|
||||
// Start the transaction and save the image
|
||||
|
|
@ -89,7 +89,7 @@ func (r *mutationResolver) ImagesUpdate(ctx context.Context, input []*ImageUpdat
|
|||
return newRet, nil
|
||||
}
|
||||
|
||||
func (r *mutationResolver) imageUpdate(ctx context.Context, input ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
|
||||
func (r *mutationResolver) imageUpdate(ctx context.Context, input models.ImageUpdateInput, translator changesetTranslator) (*models.Image, error) {
|
||||
imageID, err := strconv.Atoi(input.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("converting id: %w", err)
|
||||
|
|
|
|||
|
|
@ -197,6 +197,15 @@ func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*scra
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func (r *queryResolver) ScrapeImageURL(ctx context.Context, url string) (*scraper.ScrapedImage, error) {
|
||||
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeImage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return marshalScrapedImage(content)
|
||||
}
|
||||
|
||||
func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models.ScrapedMovie, error) {
|
||||
content, err := r.scraperCache().ScrapeURL(ctx, url, scraper.ScrapeContentTypeMovie)
|
||||
if err != nil {
|
||||
|
|
@ -491,6 +500,39 @@ func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source scraper.
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func (r *queryResolver) ScrapeSingleImage(ctx context.Context, source scraper.Source, input ScrapeSingleImageInput) ([]*scraper.ScrapedImage, error) {
|
||||
if source.StashBoxIndex != nil {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
if source.ScraperID == nil {
|
||||
return nil, fmt.Errorf("%w: scraper_id must be set", ErrInput)
|
||||
}
|
||||
|
||||
var c scraper.ScrapedContent
|
||||
|
||||
switch {
|
||||
case input.ImageID != nil:
|
||||
imageID, err := strconv.Atoi(*input.ImageID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: image id is not an integer: '%s'", ErrInput, *input.ImageID)
|
||||
}
|
||||
c, err = r.scraperCache().ScrapeID(ctx, *source.ScraperID, imageID, scraper.ScrapeContentTypeImage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalScrapedImages([]scraper.ScrapedContent{c})
|
||||
case input.ImageInput != nil:
|
||||
c, err := r.scraperCache().ScrapeFragment(ctx, *source.ScraperID, scraper.Input{Image: input.ImageInput})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalScrapedImages([]scraper.ScrapedContent{c})
|
||||
default:
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
}
|
||||
|
||||
func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source scraper.Source, input ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,6 +76,27 @@ func marshalScrapedGalleries(content []scraper.ScrapedContent) ([]*scraper.Scrap
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
func marshalScrapedImages(content []scraper.ScrapedContent) ([]*scraper.ScrapedImage, error) {
|
||||
var ret []*scraper.ScrapedImage
|
||||
for _, c := range content {
|
||||
if c == nil {
|
||||
// graphql schema requires images to be non-nil
|
||||
continue
|
||||
}
|
||||
|
||||
switch g := c.(type) {
|
||||
case *scraper.ScrapedImage:
|
||||
ret = append(ret, g)
|
||||
case scraper.ScrapedImage:
|
||||
ret = append(ret, &g)
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: cannot turn ScrapedContent into ScrapedImage", models.ErrConversion)
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// marshalScrapedMovies converts ScrapedContent into ScrapedMovie. If conversion
|
||||
// fails, an error is returned.
|
||||
func marshalScrapedMovies(content []scraper.ScrapedContent) ([]*models.ScrapedMovie, error) {
|
||||
|
|
@ -129,6 +150,16 @@ func marshalScrapedGallery(content scraper.ScrapedContent) (*scraper.ScrapedGall
|
|||
return g[0], nil
|
||||
}
|
||||
|
||||
// marshalScrapedImage will marshal a single scraped image
|
||||
func marshalScrapedImage(content scraper.ScrapedContent) (*scraper.ScrapedImage, error) {
|
||||
g, err := marshalScrapedImages([]scraper.ScrapedContent{content})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return g[0], nil
|
||||
}
|
||||
|
||||
// marshalScrapedMovie will marshal a single scraped movie
|
||||
func marshalScrapedMovie(content scraper.ScrapedContent) (*models.ScrapedMovie, error) {
|
||||
m, err := marshalScrapedMovies([]scraper.ScrapedContent{content})
|
||||
|
|
|
|||
|
|
@ -63,6 +63,28 @@ type ImageFilterType struct {
|
|||
UpdatedAt *TimestampCriterionInput `json:"updated_at"`
|
||||
}
|
||||
|
||||
type ImageUpdateInput struct {
|
||||
ClientMutationID *string `json:"clientMutationId"`
|
||||
ID string `json:"id"`
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Urls []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
Details *string `json:"details"`
|
||||
Photographer *string `json:"photographer"`
|
||||
Rating100 *int `json:"rating100"`
|
||||
Organized *bool `json:"organized"`
|
||||
SceneIds []string `json:"scene_ids"`
|
||||
StudioID *string `json:"studio_id"`
|
||||
TagIds []string `json:"tag_ids"`
|
||||
PerformerIds []string `json:"performer_ids"`
|
||||
GalleryIds []string `json:"gallery_ids"`
|
||||
PrimaryFileID *string `json:"primary_file_id"`
|
||||
|
||||
// deprecated
|
||||
URL *string `json:"url"`
|
||||
}
|
||||
|
||||
type ImageDestroyInput struct {
|
||||
ID string `json:"id"`
|
||||
DeleteFile *bool `json:"delete_file"`
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ type scraperActionImpl interface {
|
|||
|
||||
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
|
||||
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
|
||||
scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error)
|
||||
}
|
||||
|
||||
func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {
|
||||
|
|
|
|||
|
|
@ -77,11 +77,18 @@ type GalleryFinder interface {
|
|||
models.URLLoader
|
||||
}
|
||||
|
||||
type ImageFinder interface {
|
||||
models.ImageGetter
|
||||
models.FileLoader
|
||||
models.URLLoader
|
||||
}
|
||||
|
||||
type Repository struct {
|
||||
TxnManager models.TxnManager
|
||||
|
||||
SceneFinder SceneFinder
|
||||
GalleryFinder GalleryFinder
|
||||
ImageFinder ImageFinder
|
||||
TagFinder TagFinder
|
||||
PerformerFinder PerformerFinder
|
||||
GroupFinder match.GroupNamesFinder
|
||||
|
|
@ -93,6 +100,7 @@ func NewRepository(repo models.Repository) Repository {
|
|||
TxnManager: repo.TxnManager,
|
||||
SceneFinder: repo.Scene,
|
||||
GalleryFinder: repo.Gallery,
|
||||
ImageFinder: repo.Image,
|
||||
TagFinder: repo.Tag,
|
||||
PerformerFinder: repo.Performer,
|
||||
GroupFinder: repo.Group,
|
||||
|
|
@ -357,6 +365,28 @@ func (c Cache) ScrapeID(ctx context.Context, scraperID string, id int, ty Scrape
|
|||
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
|
||||
}
|
||||
|
||||
if scraped != nil {
|
||||
ret = scraped
|
||||
}
|
||||
|
||||
case ScrapeContentTypeImage:
|
||||
is, ok := s.(imageScraper)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: cannot use scraper %s as a image scraper", ErrNotSupported, scraperID)
|
||||
}
|
||||
|
||||
scene, err := c.getImage(ctx, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraper %s: unable to load image id %v: %w", scraperID, id, err)
|
||||
}
|
||||
|
||||
// don't assign nil concrete pointer to ret interface, otherwise nil
|
||||
// detection is harder
|
||||
scraped, err := is.viaImage(ctx, c.client, scene)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraper %s: %w", scraperID, err)
|
||||
}
|
||||
|
||||
if scraped != nil {
|
||||
ret = scraped
|
||||
}
|
||||
|
|
@ -426,3 +456,31 @@ func (c Cache) getGallery(ctx context.Context, galleryID int) (*models.Gallery,
|
|||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (c Cache) getImage(ctx context.Context, imageID int) (*models.Image, error) {
|
||||
var ret *models.Image
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
qb := r.ImageFinder
|
||||
|
||||
var err error
|
||||
ret, err = qb.Find(ctx, imageID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ret == nil {
|
||||
return fmt.Errorf("image with id %d not found", imageID)
|
||||
}
|
||||
|
||||
err = ret.LoadFiles(ctx, qb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ret.LoadURLs(ctx, qb)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,13 @@ type config struct {
|
|||
// Configuration for querying a gallery by a URL
|
||||
GalleryByURL []*scrapeByURLConfig `yaml:"galleryByURL"`
|
||||
|
||||
// Configuration for querying an image by a URL
|
||||
ImageByURL []*scrapeByURLConfig `yaml:"imageByURL"`
|
||||
|
||||
// Configuration for querying image by an Image fragment
|
||||
ImageByFragment *scraperTypeConfig `yaml:"imageByFragment"`
|
||||
|
||||
// Configuration for querying a movie by a URL
|
||||
// Configuration for querying a movie by a URL - deprecated, use GroupByURL
|
||||
MovieByURL []*scrapeByURLConfig `yaml:"movieByURL"`
|
||||
GroupByURL []*scrapeByURLConfig `yaml:"groupByURL"`
|
||||
|
|
@ -295,6 +302,21 @@ func (c config) spec() Scraper {
|
|||
ret.Gallery = &gallery
|
||||
}
|
||||
|
||||
image := ScraperSpec{}
|
||||
if c.ImageByFragment != nil {
|
||||
image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeFragment)
|
||||
}
|
||||
if len(c.ImageByURL) > 0 {
|
||||
image.SupportedScrapes = append(image.SupportedScrapes, ScrapeTypeURL)
|
||||
for _, v := range c.ImageByURL {
|
||||
image.Urls = append(image.Urls, v.URL...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(image.SupportedScrapes) > 0 {
|
||||
ret.Image = &image
|
||||
}
|
||||
|
||||
group := ScraperSpec{}
|
||||
if len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0 {
|
||||
group.SupportedScrapes = append(group.SupportedScrapes, ScrapeTypeURL)
|
||||
|
|
@ -319,6 +341,8 @@ func (c config) supports(ty ScrapeContentType) bool {
|
|||
return (c.SceneByName != nil && c.SceneByQueryFragment != nil) || c.SceneByFragment != nil || len(c.SceneByURL) > 0
|
||||
case ScrapeContentTypeGallery:
|
||||
return c.GalleryByFragment != nil || len(c.GalleryByURL) > 0
|
||||
case ScrapeContentTypeImage:
|
||||
return c.ImageByFragment != nil || len(c.ImageByURL) > 0
|
||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||
return len(c.MovieByURL) > 0 || len(c.GroupByURL) > 0
|
||||
}
|
||||
|
|
@ -346,6 +370,12 @@ func (c config) matchesURL(url string, ty ScrapeContentType) bool {
|
|||
return true
|
||||
}
|
||||
}
|
||||
case ScrapeContentTypeImage:
|
||||
for _, scraper := range c.ImageByURL {
|
||||
if scraper.matchesURL(url) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||
for _, scraper := range c.MovieByURL {
|
||||
if scraper.matchesURL(url) {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ func (g group) fragmentScraper(input Input) *scraperTypeConfig {
|
|||
case input.Gallery != nil:
|
||||
// TODO - this should be galleryByQueryFragment
|
||||
return g.config.GalleryByFragment
|
||||
case input.Image != nil:
|
||||
// TODO - this should be imageByImageFragment
|
||||
return g.config.ImageByFragment
|
||||
case input.Scene != nil:
|
||||
return g.config.SceneByQueryFragment
|
||||
}
|
||||
|
|
@ -75,6 +78,15 @@ func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *mod
|
|||
return s.scrapeGalleryByGallery(ctx, gallery)
|
||||
}
|
||||
|
||||
func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*ScrapedImage, error) {
|
||||
if g.config.ImageByFragment == nil {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
s := g.config.getScraper(*g.config.ImageByFragment, client, g.globalConf)
|
||||
return s.scrapeImageByImage(ctx, gallery)
|
||||
}
|
||||
|
||||
func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
|
||||
switch ty {
|
||||
case ScrapeContentTypePerformer:
|
||||
|
|
@ -85,6 +97,8 @@ func loadUrlCandidates(c config, ty ScrapeContentType) []*scrapeByURLConfig {
|
|||
return append(c.MovieByURL, c.GroupByURL...)
|
||||
case ScrapeContentTypeGallery:
|
||||
return c.GalleryByURL
|
||||
case ScrapeContentTypeImage:
|
||||
return c.ImageByURL
|
||||
}
|
||||
|
||||
panic("loadUrlCandidates: unreachable")
|
||||
|
|
|
|||
|
|
@ -11,6 +11,28 @@ import (
|
|||
"github.com/stashapp/stash/pkg/utils"
|
||||
)
|
||||
|
||||
type ScrapedImage struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
Photographer *string `json:"photographer"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
Studio *models.ScrapedStudio `json:"studio"`
|
||||
Tags []*models.ScrapedTag `json:"tags"`
|
||||
Performers []*models.ScrapedPerformer `json:"performers"`
|
||||
}
|
||||
|
||||
func (ScrapedImage) IsScrapedContent() {}
|
||||
|
||||
type ScrapedImageInput struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
}
|
||||
|
||||
func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error {
|
||||
// backwards compatibility: we fetch the image if it's a URL and set it to the first image
|
||||
// Image is deprecated, so only do this if Images is unset
|
||||
|
|
|
|||
|
|
@ -102,6 +102,12 @@ func (s *jsonScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCont
|
|||
return nil, err
|
||||
}
|
||||
return ret, nil
|
||||
case ScrapeContentTypeImage:
|
||||
ret, err := scraper.scrapeImage(ctx, q)
|
||||
if err != nil || ret == nil {
|
||||
return nil, err
|
||||
}
|
||||
return ret, nil
|
||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||
ret, err := scraper.scrapeGroup(ctx, q)
|
||||
if err != nil || ret == nil {
|
||||
|
|
@ -225,6 +231,30 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
|
|||
return scraper.scrapeScene(ctx, q)
|
||||
}
|
||||
|
||||
func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromImage(image)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
|
||||
}
|
||||
url := queryURL.constructURL(s.scraper.QueryURL)
|
||||
|
||||
scraper := s.getJsonScraper()
|
||||
|
||||
if scraper == nil {
|
||||
return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
|
||||
}
|
||||
|
||||
doc, err := s.loadURL(ctx, url)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
q := s.getJsonQuery(doc)
|
||||
return scraper.scrapeImage(ctx, q)
|
||||
}
|
||||
|
||||
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromGallery(gallery)
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ type mappedGalleryScraperConfig struct {
|
|||
Performers mappedConfig `yaml:"Performers"`
|
||||
Studio mappedConfig `yaml:"Studio"`
|
||||
}
|
||||
|
||||
type _mappedGalleryScraperConfig mappedGalleryScraperConfig
|
||||
|
||||
func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
|
|
@ -228,6 +229,60 @@ func (s *mappedGalleryScraperConfig) UnmarshalYAML(unmarshal func(interface{}) e
|
|||
return nil
|
||||
}
|
||||
|
||||
type mappedImageScraperConfig struct {
|
||||
mappedConfig
|
||||
|
||||
Tags mappedConfig `yaml:"Tags"`
|
||||
Performers mappedConfig `yaml:"Performers"`
|
||||
Studio mappedConfig `yaml:"Studio"`
|
||||
}
|
||||
type _mappedImageScraperConfig mappedImageScraperConfig
|
||||
|
||||
func (s *mappedImageScraperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
// HACK - unmarshal to map first, then remove known scene sub-fields, then
|
||||
// remarshal to yaml and pass that down to the base map
|
||||
parentMap := make(map[string]interface{})
|
||||
if err := unmarshal(parentMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// move the known sub-fields to a separate map
|
||||
thisMap := make(map[string]interface{})
|
||||
|
||||
thisMap[mappedScraperConfigSceneTags] = parentMap[mappedScraperConfigSceneTags]
|
||||
thisMap[mappedScraperConfigScenePerformers] = parentMap[mappedScraperConfigScenePerformers]
|
||||
thisMap[mappedScraperConfigSceneStudio] = parentMap[mappedScraperConfigSceneStudio]
|
||||
|
||||
delete(parentMap, mappedScraperConfigSceneTags)
|
||||
delete(parentMap, mappedScraperConfigScenePerformers)
|
||||
delete(parentMap, mappedScraperConfigSceneStudio)
|
||||
|
||||
// re-unmarshal the sub-fields
|
||||
yml, err := yaml.Marshal(thisMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// needs to be a different type to prevent infinite recursion
|
||||
c := _mappedImageScraperConfig{}
|
||||
if err := yaml.Unmarshal(yml, &c); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*s = mappedImageScraperConfig(c)
|
||||
|
||||
yml, err = yaml.Marshal(parentMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(yml, &s.mappedConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type mappedPerformerScraperConfig struct {
|
||||
mappedConfig
|
||||
|
||||
|
|
@ -785,6 +840,7 @@ type mappedScraper struct {
|
|||
Common commonMappedConfig `yaml:"common"`
|
||||
Scene *mappedSceneScraperConfig `yaml:"scene"`
|
||||
Gallery *mappedGalleryScraperConfig `yaml:"gallery"`
|
||||
Image *mappedImageScraperConfig `yaml:"image"`
|
||||
Performer *mappedPerformerScraperConfig `yaml:"performer"`
|
||||
Movie *mappedMovieScraperConfig `yaml:"movie"`
|
||||
}
|
||||
|
|
@ -1016,6 +1072,57 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
|
|||
return nil, nil
|
||||
}
|
||||
|
||||
func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*ScrapedImage, error) {
|
||||
var ret ScrapedImage
|
||||
|
||||
imageScraperConfig := s.Image
|
||||
if imageScraperConfig == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
imageMap := imageScraperConfig.mappedConfig
|
||||
|
||||
imagePerformersMap := imageScraperConfig.Performers
|
||||
imageTagsMap := imageScraperConfig.Tags
|
||||
imageStudioMap := imageScraperConfig.Studio
|
||||
|
||||
logger.Debug(`Processing image:`)
|
||||
results := imageMap.process(ctx, q, s.Common)
|
||||
|
||||
// now apply the performers and tags
|
||||
if imagePerformersMap != nil {
|
||||
logger.Debug(`Processing image performers:`)
|
||||
ret.Performers = processRelationships[models.ScrapedPerformer](ctx, s, imagePerformersMap, q)
|
||||
}
|
||||
|
||||
if imageTagsMap != nil {
|
||||
logger.Debug(`Processing image tags:`)
|
||||
ret.Tags = processRelationships[models.ScrapedTag](ctx, s, imageTagsMap, q)
|
||||
}
|
||||
|
||||
if imageStudioMap != nil {
|
||||
logger.Debug(`Processing image studio:`)
|
||||
studioResults := imageStudioMap.process(ctx, q, s.Common)
|
||||
|
||||
if len(studioResults) > 0 {
|
||||
studio := &models.ScrapedStudio{}
|
||||
studioResults[0].apply(studio)
|
||||
ret.Studio = studio
|
||||
}
|
||||
}
|
||||
|
||||
// if no basic fields are populated, and no relationships, then return nil
|
||||
if len(results) == 0 && len(ret.Performers) == 0 && len(ret.Tags) == 0 && ret.Studio == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if len(results) > 0 {
|
||||
results[0].apply(&ret)
|
||||
}
|
||||
|
||||
return &ret, nil
|
||||
}
|
||||
|
||||
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
|
||||
var ret ScrapedGallery
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,12 @@ func (c Cache) postScrape(ctx context.Context, content ScrapedContent) (ScrapedC
|
|||
}
|
||||
case ScrapedGallery:
|
||||
return c.postScrapeGallery(ctx, v)
|
||||
case *ScrapedImage:
|
||||
if v != nil {
|
||||
return c.postScrapeImage(ctx, *v)
|
||||
}
|
||||
case ScrapedImage:
|
||||
return c.postScrapeImage(ctx, v)
|
||||
case *models.ScrapedMovie:
|
||||
if v != nil {
|
||||
return c.postScrapeMovie(ctx, *v)
|
||||
|
|
@ -315,6 +321,40 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery) (Scraped
|
|||
return g, nil
|
||||
}
|
||||
|
||||
func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage) (ScrapedContent, error) {
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
pqb := r.PerformerFinder
|
||||
tqb := r.TagFinder
|
||||
sqb := r.StudioFinder
|
||||
|
||||
for _, p := range image.Performers {
|
||||
if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
tags, err := postProcessTags(ctx, tqb, image.Tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
image.Tags = tags
|
||||
|
||||
if image.Studio != nil {
|
||||
err := match.ScrapedStudio(ctx, sqb, image.Studio, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return image, nil
|
||||
}
|
||||
|
||||
func postProcessTags(ctx context.Context, tqb models.TagQueryer, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
|
||||
var ret []*models.ScrapedTag
|
||||
|
||||
|
|
|
|||
|
|
@ -73,6 +73,24 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters {
|
|||
return ret
|
||||
}
|
||||
|
||||
func queryURLParametersFromImage(image *models.Image) queryURLParameters {
|
||||
ret := make(queryURLParameters)
|
||||
ret["checksum"] = image.Checksum
|
||||
|
||||
if image.Path != "" {
|
||||
ret["filename"] = filepath.Base(image.Path)
|
||||
}
|
||||
if image.Title != "" {
|
||||
ret["title"] = image.Title
|
||||
}
|
||||
|
||||
if len(image.URLs.List()) > 0 {
|
||||
ret["url"] = image.URLs.List()[0]
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func (p queryURLParameters) applyReplacements(r queryURLReplacements) {
|
||||
for k, v := range p {
|
||||
rpl, found := r[k]
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ const (
|
|||
ScrapeContentTypeGroup ScrapeContentType = "GROUP"
|
||||
ScrapeContentTypePerformer ScrapeContentType = "PERFORMER"
|
||||
ScrapeContentTypeScene ScrapeContentType = "SCENE"
|
||||
ScrapeContentTypeImage ScrapeContentType = "IMAGE"
|
||||
)
|
||||
|
||||
var AllScrapeContentType = []ScrapeContentType{
|
||||
|
|
@ -44,11 +45,12 @@ var AllScrapeContentType = []ScrapeContentType{
|
|||
ScrapeContentTypeGroup,
|
||||
ScrapeContentTypePerformer,
|
||||
ScrapeContentTypeScene,
|
||||
ScrapeContentTypeImage,
|
||||
}
|
||||
|
||||
func (e ScrapeContentType) IsValid() bool {
|
||||
switch e {
|
||||
case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene:
|
||||
case ScrapeContentTypeGallery, ScrapeContentTypeMovie, ScrapeContentTypeGroup, ScrapeContentTypePerformer, ScrapeContentTypeScene, ScrapeContentTypeImage:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
|
@ -84,6 +86,8 @@ type Scraper struct {
|
|||
Scene *ScraperSpec `json:"scene"`
|
||||
// Details for gallery scraper
|
||||
Gallery *ScraperSpec `json:"gallery"`
|
||||
// Details for image scraper
|
||||
Image *ScraperSpec `json:"image"`
|
||||
// Details for movie scraper
|
||||
Group *ScraperSpec `json:"group"`
|
||||
// Details for movie scraper
|
||||
|
|
@ -161,6 +165,7 @@ type Input struct {
|
|||
Performer *ScrapedPerformerInput
|
||||
Scene *ScrapedSceneInput
|
||||
Gallery *ScrapedGalleryInput
|
||||
Image *ScrapedImageInput
|
||||
}
|
||||
|
||||
// populateURL populates the URL field of the input based on the
|
||||
|
|
@ -225,6 +230,14 @@ type sceneScraper interface {
|
|||
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
|
||||
}
|
||||
|
||||
// imageScraper is a scraper which supports image scrapes with
|
||||
// image data as the input.
|
||||
type imageScraper interface {
|
||||
scraper
|
||||
|
||||
viaImage(ctx context.Context, client *http.Client, image *models.Image) (*ScrapedImage, error)
|
||||
}
|
||||
|
||||
// galleryScraper is a scraper which supports gallery scrapes with
|
||||
// gallery data as the input.
|
||||
type galleryScraper interface {
|
||||
|
|
|
|||
|
|
@ -388,6 +388,10 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
|
|||
var movie *models.ScrapedMovie
|
||||
err := s.runScraperScript(ctx, input, &movie)
|
||||
return movie, err
|
||||
case ScrapeContentTypeImage:
|
||||
var image *ScrapedImage
|
||||
err := s.runScraperScript(ctx, input, &image)
|
||||
return image, err
|
||||
}
|
||||
|
||||
return nil, ErrNotSupported
|
||||
|
|
@ -421,6 +425,20 @@ func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mod
|
|||
return ret, err
|
||||
}
|
||||
|
||||
func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
inString, err := json.Marshal(imageToUpdateInput(image))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret *ScrapedImage
|
||||
|
||||
err = s.runScraperScript(ctx, string(inString), &ret)
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func handleScraperStderr(name string, scraperOutputReader io.ReadCloser) {
|
||||
const scraperPrefix = "[Scrape / %s] "
|
||||
|
||||
|
|
|
|||
|
|
@ -388,6 +388,33 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
|||
return &ret, nil
|
||||
}
|
||||
|
||||
func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *stashScraper) scrapeByURL(_ context.Context, _ string, _ ScrapeContentType) (ScrapedContent, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
func imageToUpdateInput(gallery *models.Image) models.ImageUpdateInput {
|
||||
dateToStringPtr := func(s *models.Date) *string {
|
||||
if s != nil {
|
||||
v := s.String()
|
||||
return &v
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fallback to file basename if title is empty
|
||||
title := gallery.GetTitle()
|
||||
urls := gallery.URLs.List()
|
||||
|
||||
return models.ImageUpdateInput{
|
||||
ID: strconv.Itoa(gallery.ID),
|
||||
Title: &title,
|
||||
Details: &gallery.Details,
|
||||
Urls: urls,
|
||||
Date: dateToStringPtr(gallery.Date),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,6 +83,12 @@ func (s *xpathScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCon
|
|||
return nil, err
|
||||
}
|
||||
return ret, nil
|
||||
case ScrapeContentTypeImage:
|
||||
ret, err := scraper.scrapeImage(ctx, q)
|
||||
if err != nil || ret == nil {
|
||||
return nil, err
|
||||
}
|
||||
return ret, nil
|
||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||
ret, err := scraper.scrapeGroup(ctx, q)
|
||||
if err != nil || ret == nil {
|
||||
|
|
@ -228,6 +234,30 @@ func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
|||
return scraper.scrapeGallery(ctx, q)
|
||||
}
|
||||
|
||||
func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromImage(image)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
|
||||
}
|
||||
url := queryURL.constructURL(s.scraper.QueryURL)
|
||||
|
||||
scraper := s.getXpathScraper()
|
||||
|
||||
if scraper == nil {
|
||||
return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config")
|
||||
}
|
||||
|
||||
doc, err := s.loadURL(ctx, url)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
q := s.getXPathQuery(doc)
|
||||
return scraper.scrapeImage(ctx, q)
|
||||
}
|
||||
|
||||
func (s *xpathScraper) loadURL(ctx context.Context, url string) (*html.Node, error) {
|
||||
r, err := loadURL(ctx, url, s.client, s.config, s.globalConfig)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -205,6 +205,27 @@ fragment ScrapedGalleryData on ScrapedGallery {
|
|||
}
|
||||
}
|
||||
|
||||
fragment ScrapedImageData on ScrapedImage {
|
||||
title
|
||||
code
|
||||
details
|
||||
photographer
|
||||
urls
|
||||
date
|
||||
|
||||
studio {
|
||||
...ScrapedSceneStudioData
|
||||
}
|
||||
|
||||
tags {
|
||||
...ScrapedSceneTagData
|
||||
}
|
||||
|
||||
performers {
|
||||
...ScrapedScenePerformerData
|
||||
}
|
||||
}
|
||||
|
||||
fragment ScrapedStashBoxSceneData on ScrapedScene {
|
||||
title
|
||||
code
|
||||
|
|
|
|||
|
|
@ -31,6 +31,17 @@ query ListGalleryScrapers {
|
|||
}
|
||||
}
|
||||
|
||||
query ListImageScrapers {
|
||||
listScrapers(types: [IMAGE]) {
|
||||
id
|
||||
name
|
||||
image {
|
||||
urls
|
||||
supported_scrapes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
query ListGroupScrapers {
|
||||
listScrapers(types: [GROUP]) {
|
||||
id
|
||||
|
|
@ -108,12 +119,27 @@ query ScrapeSingleGallery(
|
|||
}
|
||||
}
|
||||
|
||||
query ScrapeSingleImage(
|
||||
$source: ScraperSourceInput!
|
||||
$input: ScrapeSingleImageInput!
|
||||
) {
|
||||
scrapeSingleImage(source: $source, input: $input) {
|
||||
...ScrapedImageData
|
||||
}
|
||||
}
|
||||
|
||||
query ScrapeGalleryURL($url: String!) {
|
||||
scrapeGalleryURL(url: $url) {
|
||||
...ScrapedGalleryData
|
||||
}
|
||||
}
|
||||
|
||||
query ScrapeImageURL($url: String!) {
|
||||
scrapeImageURL(url: $url) {
|
||||
...ScrapedImageData
|
||||
}
|
||||
}
|
||||
|
||||
query ScrapeGroupURL($url: String!) {
|
||||
scrapeGroupURL(url: $url) {
|
||||
...ScrapedGroupData
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import React, { useEffect, useState } from "react";
|
||||
import { Button, Form, Col, Row } from "react-bootstrap";
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { FormattedMessage, useIntl } from "react-intl";
|
||||
import { Button, Form, Col, Row } from "react-bootstrap";
|
||||
import Mousetrap from "mousetrap";
|
||||
import * as GQL from "src/core/generated-graphql";
|
||||
import * as yup from "yup";
|
||||
|
|
@ -19,6 +19,13 @@ import {
|
|||
PerformerSelect,
|
||||
} from "src/components/Performers/PerformerSelect";
|
||||
import { formikUtils } from "src/utils/form";
|
||||
import {
|
||||
queryScrapeImage,
|
||||
queryScrapeImageURL,
|
||||
useListImageScrapers,
|
||||
mutateReloadScrapers,
|
||||
} from "../../../core/StashService";
|
||||
import { ImageScrapeDialog } from "./ImageScrapeDialog";
|
||||
import { Studio, StudioSelect } from "src/components/Studios/StudioSelect";
|
||||
import { galleryTitle } from "src/core/galleries";
|
||||
import {
|
||||
|
|
@ -27,6 +34,7 @@ import {
|
|||
excludeFileBasedGalleries,
|
||||
} from "src/components/Galleries/GallerySelect";
|
||||
import { useTagsEdit } from "src/hooks/tagsEdit";
|
||||
import { ScraperMenu } from "src/components/Shared/ScraperMenu";
|
||||
|
||||
interface IProps {
|
||||
image: GQL.ImageDataFragment;
|
||||
|
|
@ -51,6 +59,8 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
const [performers, setPerformers] = useState<Performer[]>([]);
|
||||
const [studio, setStudio] = useState<Studio | null>(null);
|
||||
|
||||
const isNew = image.id === undefined;
|
||||
|
||||
useEffect(() => {
|
||||
setGalleries(
|
||||
image.galleries?.map((g) => ({
|
||||
|
|
@ -62,6 +72,9 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
);
|
||||
}, [image.galleries]);
|
||||
|
||||
const scrapers = useListImageScrapers();
|
||||
const [scrapedImage, setScrapedImage] = useState<GQL.ScrapedImage | null>();
|
||||
|
||||
const schema = yup.object({
|
||||
title: yup.string().ensure(),
|
||||
code: yup.string().ensure(),
|
||||
|
|
@ -97,8 +110,9 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
onSubmit: (values) => onSave(schema.cast(values)),
|
||||
});
|
||||
|
||||
const { tagsControl } = useTagsEdit(image.tags, (ids) =>
|
||||
formik.setFieldValue("tag_ids", ids)
|
||||
const { tags, updateTagsStateFromScraper, tagsControl } = useTagsEdit(
|
||||
image.tags,
|
||||
(ids) => formik.setFieldValue("tag_ids", ids)
|
||||
);
|
||||
|
||||
function onSetGalleries(items: Gallery[]) {
|
||||
|
|
@ -148,6 +162,12 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
}
|
||||
});
|
||||
|
||||
const fragmentScrapers = useMemo(() => {
|
||||
return (scrapers?.data?.listScrapers ?? []).filter((s) =>
|
||||
s.image?.supported_scrapes.includes(GQL.ScrapeType.Fragment)
|
||||
);
|
||||
}, [scrapers]);
|
||||
|
||||
async function onSave(input: InputValues) {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
|
|
@ -162,6 +182,122 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
setIsLoading(false);
|
||||
}
|
||||
|
||||
async function onScrapeClicked(s: GQL.ScraperSourceInput) {
|
||||
if (!image || !image.id) return;
|
||||
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const result = await queryScrapeImage(s.scraper_id!, image.id);
|
||||
if (!result.data || !result.data.scrapeSingleImage?.length) {
|
||||
Toast.success("No images found");
|
||||
return;
|
||||
}
|
||||
setScrapedImage(result.data.scrapeSingleImage[0]);
|
||||
} catch (e) {
|
||||
Toast.error(e);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
function urlScrapable(scrapedUrl: string): boolean {
|
||||
return (scrapers?.data?.listScrapers ?? []).some((s) =>
|
||||
(s?.image?.urls ?? []).some((u) => scrapedUrl.includes(u))
|
||||
);
|
||||
}
|
||||
|
||||
function updateImageFromScrapedGallery(
|
||||
imageData: GQL.ScrapedImageDataFragment
|
||||
) {
|
||||
if (imageData.title) {
|
||||
formik.setFieldValue("title", imageData.title);
|
||||
}
|
||||
|
||||
if (imageData.code) {
|
||||
formik.setFieldValue("code", imageData.code);
|
||||
}
|
||||
|
||||
if (imageData.details) {
|
||||
formik.setFieldValue("details", imageData.details);
|
||||
}
|
||||
|
||||
if (imageData.photographer) {
|
||||
formik.setFieldValue("photographer", imageData.photographer);
|
||||
}
|
||||
|
||||
if (imageData.date) {
|
||||
formik.setFieldValue("date", imageData.date);
|
||||
}
|
||||
|
||||
if (imageData.urls) {
|
||||
formik.setFieldValue("urls", imageData.urls);
|
||||
}
|
||||
|
||||
if (imageData.studio?.stored_id) {
|
||||
onSetStudio({
|
||||
id: imageData.studio.stored_id,
|
||||
name: imageData.studio.name ?? "",
|
||||
aliases: [],
|
||||
});
|
||||
}
|
||||
|
||||
if (imageData.performers?.length) {
|
||||
const idPerfs = imageData.performers.filter((p) => {
|
||||
return p.stored_id !== undefined && p.stored_id !== null;
|
||||
});
|
||||
|
||||
if (idPerfs.length > 0) {
|
||||
onSetPerformers(
|
||||
idPerfs.map((p) => {
|
||||
return {
|
||||
id: p.stored_id!,
|
||||
name: p.name ?? "",
|
||||
alias_list: [],
|
||||
};
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
updateTagsStateFromScraper(imageData.tags ?? undefined);
|
||||
}
|
||||
|
||||
async function onReloadScrapers() {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
await mutateReloadScrapers();
|
||||
} catch (e) {
|
||||
Toast.error(e);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function onScrapeDialogClosed(data?: GQL.ScrapedImageDataFragment) {
|
||||
if (data) {
|
||||
updateImageFromScrapedGallery(data);
|
||||
}
|
||||
setScrapedImage(undefined);
|
||||
}
|
||||
|
||||
async function onScrapeImageURL(url: string) {
|
||||
if (!url) {
|
||||
return;
|
||||
}
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const result = await queryScrapeImageURL(url);
|
||||
if (!result || !result.data || !result.data.scrapeImageURL) {
|
||||
return;
|
||||
}
|
||||
setScrapedImage(result.data.scrapeImageURL);
|
||||
} catch (e) {
|
||||
Toast.error(e);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (isLoading) return <LoadingIndicator />;
|
||||
|
||||
const splitProps = {
|
||||
|
|
@ -243,6 +379,30 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
return renderInputField("details", "textarea", "details", props);
|
||||
}
|
||||
|
||||
function maybeRenderScrapeDialog() {
|
||||
if (!scrapedImage) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentImage = {
|
||||
id: image.id!,
|
||||
...formik.values,
|
||||
};
|
||||
|
||||
return (
|
||||
<ImageScrapeDialog
|
||||
image={currentImage}
|
||||
imageStudio={studio}
|
||||
imageTags={tags}
|
||||
imagePerformers={performers}
|
||||
scraped={scrapedImage}
|
||||
onClose={(data) => {
|
||||
onScrapeDialogClosed(data);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div id="image-edit-details">
|
||||
<Prompt
|
||||
|
|
@ -250,13 +410,16 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
message={intl.formatMessage({ id: "dialogs.unsaved_changes" })}
|
||||
/>
|
||||
|
||||
{maybeRenderScrapeDialog()}
|
||||
<Form noValidate onSubmit={formik.handleSubmit}>
|
||||
<Row className="form-container edit-buttons-container px-3 pt-3">
|
||||
<div className="edit-buttons mb-3 pl-0">
|
||||
<Button
|
||||
className="edit-button"
|
||||
variant="primary"
|
||||
disabled={!formik.dirty || !isEqual(formik.errors, {})}
|
||||
disabled={
|
||||
(!isNew && !formik.dirty) || !isEqual(formik.errors, {})
|
||||
}
|
||||
onClick={() => formik.submitForm()}
|
||||
>
|
||||
<FormattedMessage id="actions.save" />
|
||||
|
|
@ -269,13 +432,23 @@ export const ImageEditPanel: React.FC<IProps> = ({
|
|||
<FormattedMessage id="actions.delete" />
|
||||
</Button>
|
||||
</div>
|
||||
<div className="ml-auto text-right d-flex">
|
||||
{!isNew && (
|
||||
<ScraperMenu
|
||||
toggle={intl.formatMessage({ id: "actions.scrape_with" })}
|
||||
scrapers={fragmentScrapers}
|
||||
onScraperClicked={onScrapeClicked}
|
||||
onReloadScrapers={onReloadScrapers}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</Row>
|
||||
<Row className="form-container px-3">
|
||||
<Col lg={7} xl={12}>
|
||||
{renderInputField("title")}
|
||||
{renderInputField("code", "text", "scene_code")}
|
||||
|
||||
{renderURLListField("urls")}
|
||||
{renderURLListField("urls", onScrapeImageURL, urlScrapable)}
|
||||
|
||||
{renderDateField("date")}
|
||||
{renderInputField("photographer")}
|
||||
|
|
|
|||
227
ui/v2.5/src/components/Images/ImageDetails/ImageScrapeDialog.tsx
Normal file
227
ui/v2.5/src/components/Images/ImageDetails/ImageScrapeDialog.tsx
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import React, { useState } from "react";
|
||||
import { useIntl } from "react-intl";
|
||||
import * as GQL from "src/core/generated-graphql";
|
||||
import {
|
||||
ScrapeDialog,
|
||||
ScrapedInputGroupRow,
|
||||
ScrapedStringListRow,
|
||||
ScrapedTextAreaRow,
|
||||
} from "src/components/Shared/ScrapeDialog/ScrapeDialog";
|
||||
import {
|
||||
ObjectListScrapeResult,
|
||||
ObjectScrapeResult,
|
||||
ScrapeResult,
|
||||
} from "src/components/Shared/ScrapeDialog/scrapeResult";
|
||||
import {
|
||||
ScrapedPerformersRow,
|
||||
ScrapedStudioRow,
|
||||
} from "src/components/Shared/ScrapeDialog/ScrapedObjectsRow";
|
||||
import { sortStoredIdObjects } from "src/utils/data";
|
||||
import { Performer } from "src/components/Performers/PerformerSelect";
|
||||
import {
|
||||
useCreateScrapedPerformer,
|
||||
useCreateScrapedStudio,
|
||||
} from "src/components/Shared/ScrapeDialog/createObjects";
|
||||
import { uniq } from "lodash-es";
|
||||
import { Tag } from "src/components/Tags/TagSelect";
|
||||
import { Studio } from "src/components/Studios/StudioSelect";
|
||||
import { useScrapedTags } from "src/components/Shared/ScrapeDialog/scrapedTags";
|
||||
|
||||
interface IImageScrapeDialogProps {
|
||||
image: Partial<GQL.ImageUpdateInput>;
|
||||
imageStudio: Studio | null;
|
||||
imageTags: Tag[];
|
||||
imagePerformers: Performer[];
|
||||
scraped: GQL.ScrapedImage;
|
||||
|
||||
onClose: (scrapedImage?: GQL.ScrapedImage) => void;
|
||||
}
|
||||
|
||||
export const ImageScrapeDialog: React.FC<IImageScrapeDialogProps> = ({
|
||||
image,
|
||||
imageStudio,
|
||||
imageTags,
|
||||
imagePerformers,
|
||||
scraped,
|
||||
onClose,
|
||||
}) => {
|
||||
const intl = useIntl();
|
||||
const [title, setTitle] = useState<ScrapeResult<string>>(
|
||||
new ScrapeResult<string>(image.title, scraped.title)
|
||||
);
|
||||
const [code, setCode] = useState<ScrapeResult<string>>(
|
||||
new ScrapeResult<string>(image.code, scraped.code)
|
||||
);
|
||||
const [urls, setURLs] = useState<ScrapeResult<string[]>>(
|
||||
new ScrapeResult<string[]>(
|
||||
image.urls,
|
||||
scraped.urls
|
||||
? uniq((image.urls ?? []).concat(scraped.urls ?? []))
|
||||
: undefined
|
||||
)
|
||||
);
|
||||
const [date, setDate] = useState<ScrapeResult<string>>(
|
||||
new ScrapeResult<string>(image.date, scraped.date)
|
||||
);
|
||||
|
||||
const [photographer, setPhotographer] = useState<ScrapeResult<string>>(
|
||||
new ScrapeResult<string>(image.photographer, scraped.photographer)
|
||||
);
|
||||
|
||||
const [studio, setStudio] = useState<ObjectScrapeResult<GQL.ScrapedStudio>>(
|
||||
new ObjectScrapeResult<GQL.ScrapedStudio>(
|
||||
imageStudio
|
||||
? {
|
||||
stored_id: imageStudio.id,
|
||||
name: imageStudio.name,
|
||||
}
|
||||
: undefined,
|
||||
scraped.studio?.stored_id ? scraped.studio : undefined
|
||||
)
|
||||
);
|
||||
const [newStudio, setNewStudio] = useState<GQL.ScrapedStudio | undefined>(
|
||||
scraped.studio && !scraped.studio.stored_id ? scraped.studio : undefined
|
||||
);
|
||||
|
||||
const [performers, setPerformers] = useState<
|
||||
ObjectListScrapeResult<GQL.ScrapedPerformer>
|
||||
>(
|
||||
new ObjectListScrapeResult<GQL.ScrapedPerformer>(
|
||||
sortStoredIdObjects(
|
||||
imagePerformers.map((p) => ({
|
||||
stored_id: p.id,
|
||||
name: p.name,
|
||||
}))
|
||||
),
|
||||
sortStoredIdObjects(scraped.performers ?? undefined)
|
||||
)
|
||||
);
|
||||
const [newPerformers, setNewPerformers] = useState<GQL.ScrapedPerformer[]>(
|
||||
scraped.performers?.filter((t) => !t.stored_id) ?? []
|
||||
);
|
||||
|
||||
const { tags, newTags, scrapedTagsRow } = useScrapedTags(
|
||||
imageTags,
|
||||
scraped.tags
|
||||
);
|
||||
|
||||
const [details, setDetails] = useState<ScrapeResult<string>>(
|
||||
new ScrapeResult<string>(image.details, scraped.details)
|
||||
);
|
||||
|
||||
const createNewStudio = useCreateScrapedStudio({
|
||||
scrapeResult: studio,
|
||||
setScrapeResult: setStudio,
|
||||
setNewObject: setNewStudio,
|
||||
});
|
||||
|
||||
const createNewPerformer = useCreateScrapedPerformer({
|
||||
scrapeResult: performers,
|
||||
setScrapeResult: setPerformers,
|
||||
newObjects: newPerformers,
|
||||
setNewObjects: setNewPerformers,
|
||||
});
|
||||
|
||||
// don't show the dialog if nothing was scraped
|
||||
if (
|
||||
[
|
||||
title,
|
||||
code,
|
||||
urls,
|
||||
date,
|
||||
photographer,
|
||||
studio,
|
||||
performers,
|
||||
tags,
|
||||
details,
|
||||
].every((r) => !r.scraped) &&
|
||||
!newStudio &&
|
||||
newPerformers.length === 0 &&
|
||||
newTags.length === 0
|
||||
) {
|
||||
onClose();
|
||||
return <></>;
|
||||
}
|
||||
|
||||
function makeNewScrapedItem(): GQL.ScrapedImageDataFragment {
|
||||
const newStudioValue = studio.getNewValue();
|
||||
|
||||
return {
|
||||
title: title.getNewValue(),
|
||||
code: code.getNewValue(),
|
||||
urls: urls.getNewValue(),
|
||||
date: date.getNewValue(),
|
||||
photographer: photographer.getNewValue(),
|
||||
studio: newStudioValue,
|
||||
performers: performers.getNewValue(),
|
||||
tags: tags.getNewValue(),
|
||||
details: details.getNewValue(),
|
||||
};
|
||||
}
|
||||
|
||||
function renderScrapeRows() {
|
||||
return (
|
||||
<>
|
||||
<ScrapedInputGroupRow
|
||||
title={intl.formatMessage({ id: "title" })}
|
||||
result={title}
|
||||
onChange={(value) => setTitle(value)}
|
||||
/>
|
||||
<ScrapedInputGroupRow
|
||||
title={intl.formatMessage({ id: "scene_code" })}
|
||||
result={code}
|
||||
onChange={(value) => setCode(value)}
|
||||
/>
|
||||
<ScrapedStringListRow
|
||||
title={intl.formatMessage({ id: "urls" })}
|
||||
result={urls}
|
||||
onChange={(value) => setURLs(value)}
|
||||
/>
|
||||
<ScrapedInputGroupRow
|
||||
title={intl.formatMessage({ id: "date" })}
|
||||
placeholder="YYYY-MM-DD"
|
||||
result={date}
|
||||
onChange={(value) => setDate(value)}
|
||||
/>
|
||||
<ScrapedInputGroupRow
|
||||
title={intl.formatMessage({ id: "photographer" })}
|
||||
result={photographer}
|
||||
onChange={(value) => setPhotographer(value)}
|
||||
/>
|
||||
<ScrapedStudioRow
|
||||
title={intl.formatMessage({ id: "studios" })}
|
||||
result={studio}
|
||||
onChange={(value) => setStudio(value)}
|
||||
newStudio={newStudio}
|
||||
onCreateNew={createNewStudio}
|
||||
/>
|
||||
<ScrapedPerformersRow
|
||||
title={intl.formatMessage({ id: "performers" })}
|
||||
result={performers}
|
||||
onChange={(value) => setPerformers(value)}
|
||||
newObjects={newPerformers}
|
||||
onCreateNew={createNewPerformer}
|
||||
/>
|
||||
{scrapedTagsRow}
|
||||
<ScrapedTextAreaRow
|
||||
title={intl.formatMessage({ id: "details" })}
|
||||
result={details}
|
||||
onChange={(value) => setDetails(value)}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<ScrapeDialog
|
||||
title={intl.formatMessage(
|
||||
{ id: "dialogs.scrape_entity_title" },
|
||||
{ entity_type: intl.formatMessage({ id: "image" }) }
|
||||
)}
|
||||
renderScrapeRows={renderScrapeRows}
|
||||
onClose={(apply) => {
|
||||
onClose(apply ? makeNewScrapedItem() : undefined);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
|
@ -7,6 +7,7 @@ import {
|
|||
useListPerformerScrapers,
|
||||
useListSceneScrapers,
|
||||
useListGalleryScrapers,
|
||||
useListImageScrapers,
|
||||
} from "src/core/StashService";
|
||||
import { useToast } from "src/hooks/Toast";
|
||||
import TextUtils from "src/utils/text";
|
||||
|
|
@ -178,6 +179,8 @@ const ScrapersSection: React.FC = () => {
|
|||
useListSceneScrapers();
|
||||
const { data: galleryScrapers, loading: loadingGalleries } =
|
||||
useListGalleryScrapers();
|
||||
const { data: imageScrapers, loading: loadingImages } =
|
||||
useListImageScrapers();
|
||||
const { data: groupScrapers, loading: loadingGroups } =
|
||||
useListGroupScrapers();
|
||||
|
||||
|
|
@ -193,6 +196,9 @@ const ScrapersSection: React.FC = () => {
|
|||
galleries: galleryScrapers?.listScrapers.filter((s) =>
|
||||
filterFn(s.name, s.gallery?.urls)
|
||||
),
|
||||
images: imageScrapers?.listScrapers.filter((s) =>
|
||||
filterFn(s.name, s.image?.urls)
|
||||
),
|
||||
groups: groupScrapers?.listScrapers.filter((s) =>
|
||||
filterFn(s.name, s.group?.urls)
|
||||
),
|
||||
|
|
@ -201,6 +207,7 @@ const ScrapersSection: React.FC = () => {
|
|||
performerScrapers,
|
||||
sceneScrapers,
|
||||
galleryScrapers,
|
||||
imageScrapers,
|
||||
groupScrapers,
|
||||
filter,
|
||||
]);
|
||||
|
|
@ -213,7 +220,13 @@ const ScrapersSection: React.FC = () => {
|
|||
}
|
||||
}
|
||||
|
||||
if (loadingScenes || loadingGalleries || loadingPerformers || loadingGroups)
|
||||
if (
|
||||
loadingScenes ||
|
||||
loadingGalleries ||
|
||||
loadingPerformers ||
|
||||
loadingGroups ||
|
||||
loadingImages
|
||||
)
|
||||
return (
|
||||
<SettingSection headingID="config.scraping.scrapers">
|
||||
<LoadingIndicator />
|
||||
|
|
@ -274,6 +287,23 @@ const ScrapersSection: React.FC = () => {
|
|||
</ScraperTable>
|
||||
)}
|
||||
|
||||
{!!filteredScrapers.images?.length && (
|
||||
<ScraperTable
|
||||
entityType="image"
|
||||
count={filteredScrapers.images?.length}
|
||||
>
|
||||
{filteredScrapers.images?.map((scraper) => (
|
||||
<ScraperTableRow
|
||||
key={scraper.id}
|
||||
name={scraper.name}
|
||||
entityType="image"
|
||||
supportedScrapes={scraper.image?.supported_scrapes ?? []}
|
||||
urls={scraper.image?.urls ?? []}
|
||||
/>
|
||||
))}
|
||||
</ScraperTable>
|
||||
)}
|
||||
|
||||
{!!filteredScrapers.performers?.length && (
|
||||
<ScraperTable
|
||||
entityType="performer"
|
||||
|
|
|
|||
|
|
@ -2291,6 +2291,8 @@ export const queryScrapeGroupURL = (url: string) =>
|
|||
|
||||
export const useListGalleryScrapers = () => GQL.useListGalleryScrapersQuery();
|
||||
|
||||
export const useListImageScrapers = () => GQL.useListImageScrapersQuery();
|
||||
|
||||
export const queryScrapeGallery = (scraperId: string, galleryId: string) =>
|
||||
client.query<GQL.ScrapeSingleGalleryQuery>({
|
||||
query: GQL.ScrapeSingleGalleryDocument,
|
||||
|
|
@ -2312,6 +2314,27 @@ export const queryScrapeGalleryURL = (url: string) =>
|
|||
fetchPolicy: "network-only",
|
||||
});
|
||||
|
||||
export const queryScrapeImage = (scraperId: string, imageId: string) =>
|
||||
client.query<GQL.ScrapeSingleImageQuery>({
|
||||
query: GQL.ScrapeSingleImageDocument,
|
||||
variables: {
|
||||
source: {
|
||||
scraper_id: scraperId,
|
||||
},
|
||||
input: {
|
||||
image_id: imageId,
|
||||
},
|
||||
},
|
||||
fetchPolicy: "network-only",
|
||||
});
|
||||
|
||||
export const queryScrapeImageURL = (url: string) =>
|
||||
client.query<GQL.ScrapeImageUrlQuery>({
|
||||
query: GQL.ScrapeImageUrlDocument,
|
||||
variables: { url },
|
||||
fetchPolicy: "network-only",
|
||||
});
|
||||
|
||||
export const mutateSubmitStashBoxSceneDraft = (
|
||||
input: GQL.StashBoxDraftSubmissionInput
|
||||
) =>
|
||||
|
|
@ -2383,6 +2406,7 @@ export const scraperMutationImpactedQueries = [
|
|||
GQL.ListGroupScrapersDocument,
|
||||
GQL.ListPerformerScrapersDocument,
|
||||
GQL.ListSceneScrapersDocument,
|
||||
GQL.ListImageScrapersDocument,
|
||||
GQL.InstalledScraperPackagesDocument,
|
||||
GQL.InstalledScraperPackagesStatusDocument,
|
||||
];
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ galleryByFragment:
|
|||
<single scraper config>
|
||||
galleryByURL:
|
||||
<multiple scraper URL configs>
|
||||
imageByFragment:
|
||||
<single scraper config>
|
||||
imageByURL:
|
||||
<multiple scraper URL configs>
|
||||
<other configurations>
|
||||
```
|
||||
|
||||
|
|
@ -81,6 +85,8 @@ The script is sent input and expects output based on the scraping type, as detai
|
|||
| `groupByURL` | `{"url": "<url>"}` | JSON-encoded group fragment |
|
||||
| `galleryByFragment` | JSON-encoded gallery fragment | JSON-encoded gallery fragment |
|
||||
| `galleryByURL` | `{"url": "<url>"}` | JSON-encoded gallery fragment |
|
||||
| `imageByFragment` | JSON-encoded image fragment | JSON-encoded image fragment |
|
||||
| `imageByURL` | `{"url": "<url>"}` | JSON-encoded image fragment |
|
||||
|
||||
For `performerByName`, only `name` is required in the returned performer fragments. One entire object is sent back to `performerByFragment` to scrape a specific performer, so the other fields may be included to assist in scraping a performer. For example, the `url` field may be filled in for the specific performer page, then `performerByFragment` can extract by using its value.
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ Stash supports scraping of metadata from various external sources.
|
|||
| | Fragment | Search | URL |
|
||||
|---|:---:|:---:|:---:|
|
||||
| gallery | ✔️ | | ✔️ |
|
||||
| image | ✔️ | | ✔️ |
|
||||
| group | | | ✔️ |
|
||||
| performer | | ✔️ | ✔️ |
|
||||
| scene | ✔️ | ✔️ | ✔️ |
|
||||
|
|
|
|||
Loading…
Reference in a new issue