diff --git a/internal/api/check_version.go b/internal/api/check_version.go index c3014eab4..90e7e5b88 100644 --- a/internal/api/check_version.go +++ b/internal/api/check_version.go @@ -109,8 +109,12 @@ type githubTagResponse struct { } func makeGithubRequest(ctx context.Context, url string, output interface{}) error { + + transport := &http.Transport{Proxy: http.ProxyFromEnvironment} + client := &http.Client{ - Timeout: 3 * time.Second, + Timeout: 3 * time.Second, + Transport: transport, } req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index 07f13b261..668b2d138 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -97,6 +97,13 @@ const ( ExternalHost = "external_host" + // http proxy url if required + Proxy = "proxy" + + // urls or IPs that should not use the proxy + NoProxy = "no_proxy" + noProxyDefault = "localhost,127.0.0.1,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12" + // key used to sign JWT tokens JWTSignKey = "jwt_secret_key" @@ -1365,6 +1372,27 @@ func (i *Instance) GetMaxUploadSize() int64 { return ret << 20 } +// GetProxy returns the url of a http proxy to be used for all outgoing http calls. +func (i *Instance) GetProxy() string { + // Validate format + reg := regexp.MustCompile(`^((?:socks5h?|https?):\/\/)(([\P{Cc}]+):([\P{Cc}]+)@)?(([a-zA-Z0-9][a-zA-Z0-9.-]*)(:[0-9]{1,5})?)`) + proxy := i.getString(Proxy) + if proxy != "" && reg.MatchString(proxy) { + logger.Debug("Proxy is valid, using it") + return proxy + } else if proxy != "" { + logger.Error("Proxy is invalid, please review your configuration") + return "" + } + return "" +} + +// GetProxy returns the url of a http proxy to be used for all outgoing http calls. +func (i *Instance) GetNoProxy() string { + // NoProxy does not require validation, it is validated by the native Go library sufficiently + return i.getString(NoProxy) +} + // ActivatePublicAccessTripwire sets the security_tripwire_accessed_from_public_internet // config field to the provided IP address to indicate that stash has been accessed // from this public IP without authentication. @@ -1440,6 +1468,9 @@ func (i *Instance) setDefaultValues(write bool) error { i.main.SetDefault(ScrapersPath, defaultScrapersPath) i.main.SetDefault(PluginsPath, defaultPluginsPath) + // Set NoProxy default + i.main.SetDefault(NoProxy, noProxyDefault) + if write { return i.main.WriteConfig() } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 8ca32385f..79db2bb30 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -492,6 +492,14 @@ func (s *Manager) PostInit(ctx context.Context) error { return err } + // Set the proxy if defined in config + if s.Config.GetProxy() != "" { + os.Setenv("HTTP_PROXY", s.Config.GetProxy()) + os.Setenv("HTTPS_PROXY", s.Config.GetProxy()) + os.Setenv("NO_PROXY", s.Config.GetNoProxy()) + logger.Info("Using HTTP Proxy") + } + return nil } diff --git a/pkg/ffmpeg/downloader.go b/pkg/ffmpeg/downloader.go index 304797f7a..b98e20f6f 100644 --- a/pkg/ffmpeg/downloader.go +++ b/pkg/ffmpeg/downloader.go @@ -103,7 +103,13 @@ func downloadSingle(ctx context.Context, configDirectory, url string) error { return err } - resp, err := http.DefaultClient.Do(req) + transport := &http.Transport{Proxy: http.ProxyFromEnvironment} + + client := &http.Client{ + Transport: transport, + } + + resp, err := client.Do(req) if err != nil { return err } diff --git a/pkg/scraper/cache.go b/pkg/scraper/cache.go index 894286c3c..3b5391994 100644 --- a/pkg/scraper/cache.go +++ b/pkg/scraper/cache.go @@ -41,6 +41,7 @@ type GlobalConfig interface { GetScraperCDPPath() string GetScraperCertCheck() bool GetPythonPath() string + GetProxy() string } func isCDPPathHTTP(c GlobalConfig) bool { @@ -96,6 +97,7 @@ func newClient(gc GlobalConfig) *http.Client { Transport: &http.Transport{ // ignore insecure certificates TLSClientConfig: &tls.Config{InsecureSkipVerify: !gc.GetScraperCertCheck()}, MaxIdleConnsPerHost: maxIdleConnsPerHost, + Proxy: http.ProxyFromEnvironment, }, Timeout: scrapeGetTimeout, // defaultCheckRedirect code with max changed from 10 to maxRedirects diff --git a/pkg/scraper/url.go b/pkg/scraper/url.go index ddc63a8fb..b53d7b27f 100644 --- a/pkg/scraper/url.go +++ b/pkg/scraper/url.go @@ -9,10 +9,12 @@ import ( "net/http" "net/url" "os" + "regexp" "strings" "time" "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/cdproto/fetch" "github.com/chromedp/cdproto/network" "github.com/chromedp/chromedp" jsoniter "github.com/json-iterator/go" @@ -157,6 +159,11 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO chromedp.UserDataDir(dir), chromedp.ExecPath(cdpPath), ) + if globalConfig.GetProxy() != "" { + url, _, _ := splitProxyAuth(globalConfig.GetProxy()) + opts = append(opts, chromedp.ProxyServer(url)) + } + ctx, cancelAct = chromedp.NewExecAllocator(ctx, opts...) } @@ -173,6 +180,39 @@ func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverO var res string headers := cdpHeaders(driverOptions) + if proxyUsesAuth(globalConfig.GetProxy()) { + _, user, pass := splitProxyAuth(globalConfig.GetProxy()) + + // Based on https://github.com/chromedp/examples/blob/master/proxy/main.go + lctx, lcancel := context.WithCancel(ctx) + chromedp.ListenTarget(lctx, func(ev interface{}) { + switch ev := ev.(type) { + case *fetch.EventRequestPaused: + go func() { + _ = chromedp.Run(ctx, fetch.ContinueRequest(ev.RequestID)) + }() + case *fetch.EventAuthRequired: + if ev.AuthChallenge.Source == fetch.AuthChallengeSourceProxy { + go func() { + _ = chromedp.Run(ctx, + fetch.ContinueWithAuth(ev.RequestID, &fetch.AuthChallengeResponse{ + Response: fetch.AuthChallengeResponseResponseProvideCredentials, + Username: user, + Password: pass, + }), + // Chrome will remember the credential for the current instance, + // so we can disable the fetch domain once credential is provided. + // Please file an issue if Chrome does not work in this way. + fetch.Disable(), + ) + // and cancel the event handler too. + lcancel() + }() + } + } + }) + } + err := chromedp.Run(ctx, network.Enable(), setCDPCookies(driverOptions), @@ -260,3 +300,32 @@ func cdpHeaders(driverOptions scraperDriverOptions) map[string]interface{} { } return headers } + +func proxyUsesAuth(proxyUrl string) bool { + if proxyUrl == "" { + return false + } + reg := regexp.MustCompile(`^(https?:\/\/)(([\P{Cc}]+):([\P{Cc}]+)@)?(([a-zA-Z0-9][a-zA-Z0-9.-]*)(:[0-9]{1,5})?)`) + matches := reg.FindAllStringSubmatch(proxyUrl, -1) + if matches != nil { + split := matches[0] + return len(split) == 0 || (len(split) > 5 && split[3] != "") + } + + return false +} + +func splitProxyAuth(proxyUrl string) (string, string, string) { + if proxyUrl == "" { + return "", "", "" + } + reg := regexp.MustCompile(`^(https?:\/\/)(([\P{Cc}]+):([\P{Cc}]+)@)?(([a-zA-Z0-9][a-zA-Z0-9.-]*)(:[0-9]{1,5})?)`) + matches := reg.FindAllStringSubmatch(proxyUrl, -1) + + if matches != nil && len(matches[0]) > 5 { + split := matches[0] + return split[1] + split[5], split[3], split[4] + } + + return proxyUrl, "", "" +} diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 7120f8574..06b6ad5b6 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -834,6 +834,10 @@ func (mockGlobalConfig) GetPythonPath() string { return "" } +func (mockGlobalConfig) GetProxy() string { + return "" +} + func TestSubScrape(t *testing.T) { retHTML := `
diff --git a/ui/v2.5/src/docs/en/Changelog/v0190.md b/ui/v2.5/src/docs/en/Changelog/v0190.md index 7d708b3b8..6584e2595 100644 --- a/ui/v2.5/src/docs/en/Changelog/v0190.md +++ b/ui/v2.5/src/docs/en/Changelog/v0190.md @@ -2,6 +2,7 @@ * Performer autotagging does not currently match on performer aliases. This will be addressed when finer control over the matching is implemented. ### ✨ New Features +* Added support for specifying the use of a proxy for network requests. ([#3284](https://github.com/stashapp/stash/pull/3284)) * Added support for injecting arguments into `ffmpeg` during generation and live-transcoding. ([#3216](https://github.com/stashapp/stash/pull/3216)) * Added URL and Date fields to Images. ([#3015](https://github.com/stashapp/stash/pull/3015)) * Added support for plugins to add injected CSS and Javascript to the UI. ([#3195](https://github.com/stashapp/stash/pull/3195)) diff --git a/ui/v2.5/src/docs/en/Manual/Configuration.md b/ui/v2.5/src/docs/en/Manual/Configuration.md index 68037fa83..f9206ab0b 100644 --- a/ui/v2.5/src/docs/en/Manual/Configuration.md +++ b/ui/v2.5/src/docs/en/Manual/Configuration.md @@ -129,6 +129,8 @@ These options are typically not exposed in the UI and must be changed manually i | `custom_ui_location` | The file system folder where the UI files will be served from, instead of using the embedded UI. Empty to disable. Stash must be restarted to take effect. | | `max_upload_size` | Maximum file upload size for import files. Defaults to 1GB. | | `theme_color` | Sets the `theme-color` property in the UI. | +| `proxy` | The url of a HTTP(S) proxy to be used when stash makes calls to online services Example: https://user:password@my.proxy:8080 | +| `no_proxy` | A list of domains for which the proxy must not be used. Default is all local LAN: localhost,127.0.0.1,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 | ### Custom served folders