Skip insecure certificates check when scraping (#1120)

* Ignore insecure certificates when scraping
* add ScraperCertCheck to scraper config options

graphql/documents/data/config.graphql

@@ -31,6 +31,7 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   excludes
   imageExcludes
   scraperUserAgent
+  scraperCertCheck
   scraperCDPPath
   stashBoxes {
     name

graphql/schema/types/config.graphql

@@ -81,6 +81,8 @@ input ConfigGeneralInput {
   scraperUserAgent: String
   """Scraper CDP path. Path to chrome executable or remote address"""
   scraperCDPPath: String
+  """Whether the scraper should check for invalid certificates"""
+  scraperCertCheck: Boolean!
   """Stash-box instances used for tagging"""
   stashBoxes: [StashBoxInput!]!
 }
@@ -144,6 +146,8 @@ type ConfigGeneralResult {
   scraperUserAgent: String
   """Scraper CDP path. Path to chrome executable or remote address"""
   scraperCDPPath: String
+  """Whether the scraper should check for invalid certificates"""
+  scraperCertCheck: Boolean!
   """Stash-box instances used for tagging"""
   stashBoxes: [StashBox!]!
 }

pkg/api/resolver_mutation_configure.go

@@ -151,6 +151,8 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		refreshScraperCache = true
 	}
 
+	config.Set(config.ScraperCertCheck, input.ScraperCertCheck)
+
 	if input.StashBoxes != nil {
 		if err := config.ValidateStashBoxes(input.StashBoxes); err != nil {
 			return nil, err

pkg/api/resolver_query_configuration.go

@@ -71,6 +71,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		Excludes:                   config.GetExcludes(),
 		ImageExcludes:              config.GetImageExcludes(),
 		ScraperUserAgent:           &scraperUserAgent,
+		ScraperCertCheck:           config.GetScraperCertCheck(),
 		ScraperCDPPath:             &scraperCDPPath,
 		StashBoxes:                 config.GetStashBoxes(),
 	}

pkg/manager/config/config.go

@@ -86,6 +86,7 @@ const SessionStoreKey = "session_store_key"
 // scraping options
 const ScrapersPath = "scrapers_path"
 const ScraperUserAgent = "scraper_user_agent"
+const ScraperCertCheck = "scraper_cert_check"
 const ScraperCDPPath = "scraper_cdp_path"
 
 // stash-box options
@@ -274,6 +275,17 @@ func GetScraperCDPPath() string {
 	return viper.GetString(ScraperCDPPath)
 }
 
+// GetScraperCertCheck returns true if the scraper should check for insecure
+// certificates when fetching an image or a page.
+func GetScraperCertCheck() bool {
+	ret := true
+	if viper.IsSet(ScraperCertCheck) {
+		ret = viper.GetBool(ScraperCertCheck)
+	}
+
+	return ret
+}
+
 func GetStashBoxes() []*models.StashBox {
 	var boxes []*models.StashBox
 	viper.UnmarshalKey(StashBoxes, &boxes)

pkg/scraper/image.go

@@ -1,11 +1,13 @@
 package scraper
 
 import (
+	"crypto/tls"
 	"io/ioutil"
 	"net/http"
 	"strings"
 	"time"
 
+	stashConfig "github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
 )
@@ -83,6 +85,8 @@ func setMovieBackImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error
 
 func getImage(url string, globalConfig GlobalConfig) (*string, error) {
 	client := &http.Client{
+		Transport: &http.Transport{ // ignore insecure certificates
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: !stashConfig.GetScraperCertCheck()}},
 		Timeout: imageGetTimeout,
 	}
 

pkg/scraper/url.go

@@ -3,6 +3,7 @@ package scraper
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -22,6 +23,7 @@ import (
 	"golang.org/x/net/publicsuffix"
 
 	"github.com/stashapp/stash/pkg/logger"
+	stashConfig "github.com/stashapp/stash/pkg/manager/config"
 )
 
 // Timeout for the scrape http request. Includes transfer time. May want to make this
@@ -49,6 +51,9 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re
 	printCookies(jar, scraperConfig, "Jar cookies set from scraper")
 
 	client := &http.Client{
+		Transport: &http.Transport{ // ignore insecure certificates
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: !stashConfig.GetScraperCertCheck()},
+		},
 		Timeout: scrapeGetTimeout,
 		// defaultCheckRedirect code with max changed from 10 to 20
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

ui/v2.5/src/components/Changelog/versions/v060.md

@@ -1,4 +1,5 @@
 ### 🎨 Improvements
+* Add option to skip checking of insecure SSL certificates when scraping.
 * Auto-play video previews on mobile devices.
 * Replace hover menu with dropdown menu for O-Counter.
 * Support random strings for scraper cookie values.

ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx

@@ -125,6 +125,7 @@ export const SettingsConfigurationPanel: React.FC = () => {
   const [scraperCDPPath, setScraperCDPPath] = useState<string | undefined>(
     undefined
   );
+  const [scraperCertCheck, setScraperCertCheck] = useState<boolean>(true);
   const [stashBoxes, setStashBoxes] = useState<IStashBoxInstance[]>([]);
 
   const { data, error, loading } = useConfiguration();
@@ -164,6 +165,7 @@ export const SettingsConfigurationPanel: React.FC = () => {
     imageExcludes,
     scraperUserAgent,
     scraperCDPPath,
+    scraperCertCheck,
     stashBoxes: stashBoxes.map(
       (b) =>
         ({
@@ -212,6 +214,7 @@ export const SettingsConfigurationPanel: React.FC = () => {
       setImageExcludes(conf.general.imageExcludes);
       setScraperUserAgent(conf.general.scraperUserAgent ?? undefined);
       setScraperCDPPath(conf.general.scraperCDPPath ?? undefined);
+      setScraperCertCheck(conf.general.scraperCertCheck);
       setStashBoxes(
         conf.general.stashBoxes.map((box, i) => ({
           name: box?.name ?? undefined,
@@ -717,6 +720,20 @@ export const SettingsConfigurationPanel: React.FC = () => {
             http://localhost:9222/json/version) to a Chrome instance.
           </Form.Text>
         </Form.Group>
+
+        <Form.Group>
+          <Form.Check
+            id="scaper-cert-check"
+            checked={scraperCertCheck}
+            label="Check for insecure certificates"
+            onChange={() => setScraperCertCheck(!scraperCertCheck)}
+          />
+          <Form.Text className="text-muted">
+            Some sites use insecure ssl certificates. When unticked the scraper
+            skips the insecure certificates check and allows scraping of those
+            sites. If you get a certificate error when scraping untick this.
+          </Form.Text>
+        </Form.Group>
       </Form.Group>
 
       <hr />