mirror of
https://github.com/stashapp/stash.git
synced 2025-12-14 04:13:56 +01:00
e513b6ffa5
* Add Cookies directly to the request Rather than maintaining a cookie jar on a one-shot HTTP client, maintain the jar ourselves: make a new jar, then use it to select the right cookies. The cookies are set on the request rather than on the client. This will retain the current behavior as we are always throwing the client away after each use. This patch enables the lifting of the http client as well over time. * Introduce a cached scraper HTTP client The scraper cache is augmented with an *http.Client. These are safe for concurrent use, so the pointer can safely be passed around. Push this into scraper configurations where applicable, next to the txnManagers. When we issue a loadUrl request, do so on the cached *http.Client, which will reuse existing idle connections in the client if any are present. * Set MaxIdleConnsPerHost. Closes #1850 We allow for up to 8 idle connections to a single host. This should make concurrent operation toward the same host reuse connections, even for sizeable concurrency. The number isn't bumped excessively high. We should probably limit concurrency toward a single site anyway, since we'll be able to overrun a site with queries quite easily if we have many concurrent goroutines issuing requests at the same time. * Reinstate driverOptions / useCDP check Use DeMorgan's laws to invert the logic and exit early. Fixes tests breaking. * Documentation fixup. * Use the scraper http.Client when fetching images Fold image fetchers onto the cached scraper http.Client as well. This makes the scraper have a single http.Client cache for all its operations. Thread the client upwards to the relevant attachment points: either the cache, or a stash_box instance, which is extended to include a pointer to the client. Style roughly follows that of txnManagers. * Use the same http Client as the GraphQL client use Rather than using http.DefaultClient, use the same client as the GraphQL client use in the stash_box subsystem. This localizes the client used in the subsystem into the constructing New.. call. * Hoist HTTP client construction Create a function for initializaing the HTTP Client we use. While here hoist magic numbers into constants. Introduce a proper static redirect error and use it in the client code as well. * Reinstate printCookies This is a debugging function, and it might still come in handy in the future at some point. * Nitpick comment. * Minor tidy Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
136 lines
4.1 KiB
Go
136 lines
4.1 KiB
Go
package scraper
|
|
|
|
import (
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/stashapp/stash/pkg/logger"
|
|
"github.com/stashapp/stash/pkg/models"
|
|
)
|
|
|
|
// FreeonesScraperID is the scraper ID for the built-in Freeones scraper
|
|
const FreeonesScraperID = "builtin_freeones"
|
|
|
|
// 537: stolen from: https://github.com/stashapp/CommunityScrapers/blob/master/scrapers/FreeonesCommunity.yml
|
|
const freeonesScraperConfig = `
|
|
name: Freeones
|
|
performerByName:
|
|
action: scrapeXPath
|
|
queryURL: https://www.freeones.com/babes?q={}&v=teasers&s=relevance&l=96&m%5BcanPreviewFeatures%5D=0
|
|
scraper: performerSearch
|
|
performerByURL:
|
|
- action: scrapeXPath
|
|
url:
|
|
- freeones.xxx
|
|
- freeones.com
|
|
scraper: performerScraper
|
|
|
|
xPathScrapers:
|
|
performerSearch:
|
|
performer:
|
|
Name: //div[@id="search-result"]//p[@data-test="subject-name"]/text()
|
|
URL:
|
|
selector: //div[@id="search-result"]//div[@data-test="teaser-subject"]/a/@href
|
|
postProcess:
|
|
- replace:
|
|
- regex: ^
|
|
with: https://www.freeones.com
|
|
- regex: /feed$
|
|
with: /bio
|
|
|
|
performerScraper:
|
|
performer:
|
|
Name:
|
|
selector: //h1
|
|
postProcess:
|
|
- replace:
|
|
- regex: \sBio\s*$
|
|
with: ""
|
|
URL: //link[@rel="alternate" and @hreflang="x-default"]/@href
|
|
Twitter: //a[not(starts-with(@href,'https://twitter.com/FreeOnes'))][contains(@href,'twitter.com/')]/@href
|
|
Instagram: //a[contains(@href,'instagram.com/')]/@href
|
|
Birthdate:
|
|
selector: //span[contains(text(),'Born On')]
|
|
postProcess:
|
|
- replace:
|
|
- regex: Born On
|
|
with:
|
|
- parseDate: January 2, 2006
|
|
Ethnicity:
|
|
selector: //a[@data-test="link_ethnicity"]/span/text()
|
|
postProcess:
|
|
- map:
|
|
Asian: Asian
|
|
Caucasian: White
|
|
Black: Black
|
|
Latin: Hispanic
|
|
Country: //a[@data-test="link-country"]/span/text()
|
|
EyeColor: //span[text()='Eye Color']/following-sibling::span/a
|
|
Height:
|
|
selector: //span[text()='Height']/following-sibling::span/a
|
|
postProcess:
|
|
- replace:
|
|
- regex: \D+[\s\S]+
|
|
with: ""
|
|
- map:
|
|
Unknown: ""
|
|
Measurements:
|
|
selector: //span[text()='Measurements']/following-sibling::span/span/a
|
|
concat: " - "
|
|
postProcess:
|
|
- map:
|
|
Unknown: ""
|
|
FakeTits:
|
|
selector: //span[text()='Boobs']/following-sibling::span/a
|
|
postProcess:
|
|
- map:
|
|
Unknown: ""
|
|
Fake: "Yes"
|
|
Natural: "No"
|
|
CareerLength:
|
|
selector: //div[contains(@class,'timeline-horizontal')]//p[@class='m-0']
|
|
concat: "-"
|
|
Aliases: //p[@data-test='p_aliases']/text()
|
|
Tattoos:
|
|
selector: //span[text()='Tattoos']/following-sibling::span/span
|
|
postProcess:
|
|
- map:
|
|
Unknown: ""
|
|
Piercings:
|
|
selector: //span[text()='Piercings']/following-sibling::span/span
|
|
postProcess:
|
|
- map:
|
|
Unknown: ""
|
|
Image:
|
|
selector: //div[contains(@class,'image-container')]//a/img/@src
|
|
Gender:
|
|
fixed: "Female"
|
|
Details: //div[@data-test="biography"]
|
|
DeathDate:
|
|
selector: //div[contains(text(),'Passed away on')]
|
|
postProcess:
|
|
- replace:
|
|
- regex: Passed away on (.+) at the age of \d+
|
|
with: $1
|
|
- parseDate: January 2, 2006
|
|
HairColor: //span[text()='Hair Color']/following-sibling::span/a
|
|
Weight:
|
|
selector: //span[text()='Weight']/following-sibling::span/a
|
|
postProcess:
|
|
- replace:
|
|
- regex: \D+[\s\S]+
|
|
with: ""
|
|
|
|
# Last updated April 13, 2021
|
|
`
|
|
|
|
func getFreeonesScraper(client *http.Client, txnManager models.TransactionManager, globalConfig GlobalConfig) scraper {
|
|
yml := freeonesScraperConfig
|
|
|
|
c, err := loadConfigFromYAML(FreeonesScraperID, strings.NewReader(yml))
|
|
if err != nil {
|
|
logger.Fatalf("Error loading builtin freeones scraper: %s", err.Error())
|
|
}
|
|
|
|
return createScraperFromConfig(*c, client, txnManager, globalConfig)
|
|
}
|