Allow use of Proxy (#3284)

* Proxy config
* Disable proxy for localhost & local LAN
* No_proxy is now configurable

internal/api/check_version.go

@@ -109,8 +109,12 @@ type githubTagResponse struct {
 }
 
 func makeGithubRequest(ctx context.Context, url string, output interface{}) error {
+
+	transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
+
 	client := &http.Client{
-		Timeout: 3 * time.Second,
+		Timeout:   3 * time.Second,
+		Transport: transport,
 	}
 
 	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)

internal/manager/config/config.go

@@ -97,6 +97,13 @@ const (
 
 	ExternalHost = "external_host"
 
+	// http proxy url if required
+	Proxy = "proxy"
+
+	// urls or IPs that should not use the proxy
+	NoProxy        = "no_proxy"
+	noProxyDefault = "localhost,127.0.0.1,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12"
+
 	// key used to sign JWT tokens
 	JWTSignKey = "jwt_secret_key"
 
@@ -1365,6 +1372,27 @@ func (i *Instance) GetMaxUploadSize() int64 {
 	return ret << 20
 }
 
+// GetProxy returns the url of a http proxy to be used for all outgoing http calls.
+func (i *Instance) GetProxy() string {
+	// Validate format
+	reg := regexp.MustCompile(`^((?:socks5h?|https?):\/\/)(([\P{Cc}]+):([\P{Cc}]+)@)?(([a-zA-Z0-9][a-zA-Z0-9.-]*)(:[0-9]{1,5})?)`)
+	proxy := i.getString(Proxy)
+	if proxy != "" && reg.MatchString(proxy) {
+		logger.Debug("Proxy is valid, using it")
+		return proxy
+	} else if proxy != "" {
+		logger.Error("Proxy is invalid, please review your configuration")
+		return ""
+	}
+	return ""
+}
+
+// GetProxy returns the url of a http proxy to be used for all outgoing http calls.
+func (i *Instance) GetNoProxy() string {
+	// NoProxy does not require validation, it is validated by the native Go library sufficiently
+	return i.getString(NoProxy)
+}
+
 // ActivatePublicAccessTripwire sets the security_tripwire_accessed_from_public_internet
 // config field to the provided IP address to indicate that stash has been accessed
 // from this public IP without authentication.
@@ -1440,6 +1468,9 @@ func (i *Instance) setDefaultValues(write bool) error {
 	i.main.SetDefault(ScrapersPath, defaultScrapersPath)
 	i.main.SetDefault(PluginsPath, defaultPluginsPath)
 
+	// Set NoProxy default
+	i.main.SetDefault(NoProxy, noProxyDefault)
+
 	if write {
 		return i.main.WriteConfig()
 	}

internal/manager/manager.go

@@ -492,6 +492,14 @@ func (s *Manager) PostInit(ctx context.Context) error {
 		return err
 	}
 
+	// Set the proxy if defined in config
+	if s.Config.GetProxy() != "" {
+		os.Setenv("HTTP_PROXY", s.Config.GetProxy())
+		os.Setenv("HTTPS_PROXY", s.Config.GetProxy())
+		os.Setenv("NO_PROXY", s.Config.GetNoProxy())
+		logger.Info("Using HTTP Proxy")
+	}
+
 	return nil
 }
 

pkg/ffmpeg/downloader.go

@@ -103,7 +103,13 @@ func downloadSingle(ctx context.Context, configDirectory, url string) error {
 		return err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	transport := &http.Transport{Proxy: http.ProxyFromEnvironment}
+
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}

pkg/scraper/cache.go

@@ -41,6 +41,7 @@ type GlobalConfig interface {
 	GetScraperCDPPath() string
 	GetScraperCertCheck() bool
 	GetPythonPath() string
+	GetProxy() string
 }
 
 func isCDPPathHTTP(c GlobalConfig) bool {
@@ -96,6 +97,7 @@ func newClient(gc GlobalConfig) *http.Client {
 		Transport: &http.Transport{ // ignore insecure certificates
 			TLSClientConfig:     &tls.Config{InsecureSkipVerify: !gc.GetScraperCertCheck()},
 			MaxIdleConnsPerHost: maxIdleConnsPerHost,
+			Proxy:               http.ProxyFromEnvironment,
 		},
 		Timeout: scrapeGetTimeout,
 		// defaultCheckRedirect code with max changed from 10 to maxRedirects

pkg/scraper/url.go

@@ -9,10 +9,12 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/fetch"
 	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/chromedp"
 	jsoniter "github.com/json-iterator/go"
@@ -157,6 +159,11 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO
 				chromedp.UserDataDir(dir),
 				chromedp.ExecPath(cdpPath),
 			)
+			if globalConfig.GetProxy() != "" {
+				url, _, _ := splitProxyAuth(globalConfig.GetProxy())
+				opts = append(opts, chromedp.ProxyServer(url))
+			}
+
 			ctx, cancelAct = chromedp.NewExecAllocator(ctx, opts...)
 		}
 
@@ -173,6 +180,39 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO
 	var res string
 	headers := cdpHeaders(driverOptions)
 
+	if proxyUsesAuth(globalConfig.GetProxy()) {
+		_, user, pass := splitProxyAuth(globalConfig.GetProxy())
+
+		// Based on https://github.com/chromedp/examples/blob/master/proxy/main.go
+		lctx, lcancel := context.WithCancel(ctx)
+		chromedp.ListenTarget(lctx, func(ev interface{}) {
+			switch ev := ev.(type) {
+			case *fetch.EventRequestPaused:
+				go func() {
+					_ = chromedp.Run(ctx, fetch.ContinueRequest(ev.RequestID))
+				}()
+			case *fetch.EventAuthRequired:
+				if ev.AuthChallenge.Source == fetch.AuthChallengeSourceProxy {
+					go func() {
+						_ = chromedp.Run(ctx,
+							fetch.ContinueWithAuth(ev.RequestID, &fetch.AuthChallengeResponse{
+								Response: fetch.AuthChallengeResponseResponseProvideCredentials,
+								Username: user,
+								Password: pass,
+							}),
+							// Chrome will remember the credential for the current instance,
+							// so we can disable the fetch domain once credential is provided.
+							// Please file an issue if Chrome does not work in this way.
+							fetch.Disable(),
+						)
+						// and cancel the event handler too.
+						lcancel()
+					}()
+				}
+			}
+		})
+	}
+
 	err := chromedp.Run(ctx,
 		network.Enable(),
 		setCDPCookies(driverOptions),
@@ -260,3 +300,32 @@ func cdpHeaders(driverOptions scraperDriverOptions) map[string]interface{} {
 	}
 	return headers
 }
+
+func proxyUsesAuth(proxyUrl string) bool {
+	if proxyUrl == "" {
+		return false
+	}
+	reg := regexp.MustCompile(`^(https?:\/\/)(([\P{Cc}]+):([\P{Cc}]+)@)?(([a-zA-Z0-9][a-zA-Z0-9.-]*)(:[0-9]{1,5})?)`)
+	matches := reg.FindAllStringSubmatch(proxyUrl, -1)
+	if matches != nil {
+		split := matches[0]
+		return len(split) == 0 || (len(split) > 5 && split[3] != "")
+	}
+
+	return false
+}
+
+func splitProxyAuth(proxyUrl string) (string, string, string) {
+	if proxyUrl == "" {
+		return "", "", ""
+	}
+	reg := regexp.MustCompile(`^(https?:\/\/)(([\P{Cc}]+):([\P{Cc}]+)@)?(([a-zA-Z0-9][a-zA-Z0-9.-]*)(:[0-9]{1,5})?)`)
+	matches := reg.FindAllStringSubmatch(proxyUrl, -1)
+
+	if matches != nil && len(matches[0]) > 5 {
+		split := matches[0]
+		return split[1] + split[5], split[3], split[4]
+	}
+
+	return proxyUrl, "", ""
+}

pkg/scraper/xpath_test.go

@@ -834,6 +834,10 @@ func (mockGlobalConfig) GetPythonPath() string {
 	return ""
 }
 
+func (mockGlobalConfig) GetProxy() string {
+	return ""
+}
+
 func TestSubScrape(t *testing.T) {
 	retHTML := `
 	<div>

ui/v2.5/src/docs/en/Changelog/v0190.md

@@ -2,6 +2,7 @@
 * Performer autotagging does not currently match on performer aliases. This will be addressed when finer control over the matching is implemented.
 
 ### ✨ New Features
+* Added support for specifying the use of a proxy for network requests. ([#3284](https://github.com/stashapp/stash/pull/3284))
 * Added support for injecting arguments into `ffmpeg` during generation and live-transcoding. ([#3216](https://github.com/stashapp/stash/pull/3216))
 * Added URL and Date fields to Images. ([#3015](https://github.com/stashapp/stash/pull/3015))
 * Added support for plugins to add injected CSS and Javascript to the UI. ([#3195](https://github.com/stashapp/stash/pull/3195))

ui/v2.5/src/docs/en/Manual/Configuration.md

@@ -129,6 +129,8 @@ These options are typically not exposed in the UI and must be changed manually i
 | `custom_ui_location` | The file system folder where the UI files will be served from, instead of using the embedded UI. Empty to disable. Stash must be restarted to take effect. |
 | `max_upload_size` | Maximum file upload size for import files. Defaults to 1GB. |
 | `theme_color` | Sets the `theme-color` property in the UI. |
+| `proxy` | The url of a HTTP(S) proxy to be used when stash makes calls to online services Example: https://user:password@my.proxy:8080 |
+| `no_proxy` | A list of domains for which the proxy must not be used. Default is all local LAN: localhost,127.0.0.1,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 |
 
 ### Custom served folders