From 144cd6e4f2191c2cfbe1e835af4d76b0a8111ee6 Mon Sep 17 00:00:00 2001 From: bnkai <48220860+bnkai@users.noreply.github.com> Date: Mon, 1 Mar 2021 02:47:39 +0200 Subject: [PATCH] Skip insecure certificates check when scraping (#1120) * Ignore insecure certificates when scraping * add ScraperCertCheck to scraper config options --- graphql/documents/data/config.graphql | 1 + graphql/schema/types/config.graphql | 4 ++++ pkg/api/resolver_mutation_configure.go | 2 ++ pkg/api/resolver_query_configuration.go | 1 + pkg/manager/config/config.go | 12 ++++++++++++ pkg/scraper/image.go | 4 ++++ pkg/scraper/url.go | 5 +++++ .../src/components/Changelog/versions/v060.md | 1 + .../Settings/SettingsConfigurationPanel.tsx | 17 +++++++++++++++++ 9 files changed, 47 insertions(+) diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index cd7dc6835..07330d63b 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -31,6 +31,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { excludes imageExcludes scraperUserAgent + scraperCertCheck scraperCDPPath stashBoxes { name diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 5967a2625..f3f7c8142 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -81,6 +81,8 @@ input ConfigGeneralInput { scraperUserAgent: String """Scraper CDP path. Path to chrome executable or remote address""" scraperCDPPath: String + """Whether the scraper should check for invalid certificates""" + scraperCertCheck: Boolean! """Stash-box instances used for tagging""" stashBoxes: [StashBoxInput!]! } @@ -144,6 +146,8 @@ type ConfigGeneralResult { scraperUserAgent: String """Scraper CDP path. Path to chrome executable or remote address""" scraperCDPPath: String + """Whether the scraper should check for invalid certificates""" + scraperCertCheck: Boolean! """Stash-box instances used for tagging""" stashBoxes: [StashBox!]! } diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index a1051e218..34b416094 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -151,6 +151,8 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co refreshScraperCache = true } + config.Set(config.ScraperCertCheck, input.ScraperCertCheck) + if input.StashBoxes != nil { if err := config.ValidateStashBoxes(input.StashBoxes); err != nil { return nil, err diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index c848375ea..c1d45ba8b 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -71,6 +71,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { Excludes: config.GetExcludes(), ImageExcludes: config.GetImageExcludes(), ScraperUserAgent: &scraperUserAgent, + ScraperCertCheck: config.GetScraperCertCheck(), ScraperCDPPath: &scraperCDPPath, StashBoxes: config.GetStashBoxes(), } diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index 6ef58f6b9..cdb136627 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -86,6 +86,7 @@ const SessionStoreKey = "session_store_key" // scraping options const ScrapersPath = "scrapers_path" const ScraperUserAgent = "scraper_user_agent" +const ScraperCertCheck = "scraper_cert_check" const ScraperCDPPath = "scraper_cdp_path" // stash-box options @@ -274,6 +275,17 @@ func GetScraperCDPPath() string { return viper.GetString(ScraperCDPPath) } +// GetScraperCertCheck returns true if the scraper should check for insecure +// certificates when fetching an image or a page. +func GetScraperCertCheck() bool { + ret := true + if viper.IsSet(ScraperCertCheck) { + ret = viper.GetBool(ScraperCertCheck) + } + + return ret +} + func GetStashBoxes() []*models.StashBox { var boxes []*models.StashBox viper.UnmarshalKey(StashBoxes, &boxes) diff --git a/pkg/scraper/image.go b/pkg/scraper/image.go index 1c6059835..6b41b502d 100644 --- a/pkg/scraper/image.go +++ b/pkg/scraper/image.go @@ -1,11 +1,13 @@ package scraper import ( + "crypto/tls" "io/ioutil" "net/http" "strings" "time" + stashConfig "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -83,6 +85,8 @@ func setMovieBackImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error func getImage(url string, globalConfig GlobalConfig) (*string, error) { client := &http.Client{ + Transport: &http.Transport{ // ignore insecure certificates + TLSClientConfig: &tls.Config{InsecureSkipVerify: !stashConfig.GetScraperCertCheck()}}, Timeout: imageGetTimeout, } diff --git a/pkg/scraper/url.go b/pkg/scraper/url.go index 159dc1ccd..60573f76a 100644 --- a/pkg/scraper/url.go +++ b/pkg/scraper/url.go @@ -3,6 +3,7 @@ package scraper import ( "bytes" "context" + "crypto/tls" "errors" "fmt" "io" @@ -22,6 +23,7 @@ import ( "golang.org/x/net/publicsuffix" "github.com/stashapp/stash/pkg/logger" + stashConfig "github.com/stashapp/stash/pkg/manager/config" ) // Timeout for the scrape http request. Includes transfer time. May want to make this @@ -49,6 +51,9 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re printCookies(jar, scraperConfig, "Jar cookies set from scraper") client := &http.Client{ + Transport: &http.Transport{ // ignore insecure certificates + TLSClientConfig: &tls.Config{InsecureSkipVerify: !stashConfig.GetScraperCertCheck()}, + }, Timeout: scrapeGetTimeout, // defaultCheckRedirect code with max changed from 10 to 20 CheckRedirect: func(req *http.Request, via []*http.Request) error { diff --git a/ui/v2.5/src/components/Changelog/versions/v060.md b/ui/v2.5/src/components/Changelog/versions/v060.md index f03ccad4f..bb453c598 100644 --- a/ui/v2.5/src/components/Changelog/versions/v060.md +++ b/ui/v2.5/src/components/Changelog/versions/v060.md @@ -1,4 +1,5 @@ ### 🎨 Improvements +* Add option to skip checking of insecure SSL certificates when scraping. * Auto-play video previews on mobile devices. * Replace hover menu with dropdown menu for O-Counter. * Support random strings for scraper cookie values. diff --git a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx index 92f1067b6..6070765f3 100644 --- a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx @@ -125,6 +125,7 @@ export const SettingsConfigurationPanel: React.FC = () => { const [scraperCDPPath, setScraperCDPPath] = useState( undefined ); + const [scraperCertCheck, setScraperCertCheck] = useState(true); const [stashBoxes, setStashBoxes] = useState([]); const { data, error, loading } = useConfiguration(); @@ -164,6 +165,7 @@ export const SettingsConfigurationPanel: React.FC = () => { imageExcludes, scraperUserAgent, scraperCDPPath, + scraperCertCheck, stashBoxes: stashBoxes.map( (b) => ({ @@ -212,6 +214,7 @@ export const SettingsConfigurationPanel: React.FC = () => { setImageExcludes(conf.general.imageExcludes); setScraperUserAgent(conf.general.scraperUserAgent ?? undefined); setScraperCDPPath(conf.general.scraperCDPPath ?? undefined); + setScraperCertCheck(conf.general.scraperCertCheck); setStashBoxes( conf.general.stashBoxes.map((box, i) => ({ name: box?.name ?? undefined, @@ -717,6 +720,20 @@ export const SettingsConfigurationPanel: React.FC = () => { http://localhost:9222/json/version) to a Chrome instance. + + + setScraperCertCheck(!scraperCertCheck)} + /> + + Some sites use insecure ssl certificates. When unticked the scraper + skips the insecure certificates check and allows scraping of those + sites. If you get a certificate error when scraping untick this. + +