Backup_Repos/Readarr
1
0
Fork 0
mirror of https://github.com/Readarr/Readarr synced 2025-12-26 10:14:51 +01:00
Code Issues Projects Releases Wiki Activity

Fixed: Use Proxy for MediaCovers and Metadata

Browse source 
Signed-off-by: Robin Dadswell <robin@dadswell.email>
This commit is contained in:
Qstick 2020-09-07 00:28:15 -04:00
parent 1d6749ef52
commit 7ea1bf71dd
7 changed files with 95 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/NzbDrone.Common.Test/Http/HttpClientFixture.cs
View file

@ -99,9 +99,9 @@ public void SetUp()
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<TDispatcher>());
// Used for manual testing of socks proxies.
//Mocker.GetMock<IHttpProxySettingsProvider>()
// .Setup(v => v.GetProxySettings(It.IsAny<HttpRequest>()))
// .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false));
// Mocker.GetMock<IHttpProxySettingsProvider>()
// .Setup(v => v.GetProxySettings(It.IsAny<HttpUri>()))
// .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false));
// Roundrobin over the two servers, to reduce the chance of hitting the ratelimiter.
_httpBinHost2 = _httpBinHosts[_httpBinRandom++ % _httpBinHosts.Length];
@ -283,16 +283,31 @@ public void should_download_file()
{
var file = GetTempFilePath();
var url = "https://sonarr.tv/img/slider/seriesdetails.png";
var url = "https://lidarr.audio/img/slider/artistdetails.png";
Subject.DownloadFile(url, file);
File.Exists(file).Should().BeTrue();
File.Exists(file + ".part").Should().BeFalse();
var fileInfo = new FileInfo(file);
fileInfo.Exists.Should().BeTrue();
fileInfo.Length.Should().Be(146122);
}
[Test]
public void should_download_file_with_redirect()
{
var file = GetTempFilePath();
var request = new HttpRequestBuilder($"https://{_httpBinHost}/redirect-to")
.AddQueryParam("url", $"https://lidarr.audio/img/slider/artistdetails.png")
.Build();
Subject.DownloadFile(request.Url.FullUri, file);
ExceptionVerification.ExpectedErrors(0);
var fileInfo = new FileInfo(file);
fileInfo.Length.Should().Be(307054);
fileInfo.Exists.Should().BeTrue();
fileInfo.Length.Should().Be(146122);
}
[Test]
@ -303,9 +318,8 @@ public void should_not_download_file_with_error()
Assert.Throws<WebException>(() => Subject.DownloadFile("https://download.readarr.com/wrongpath", file));
File.Exists(file).Should().BeFalse();
File.Exists(file + ".part").Should().BeFalse();
ExceptionVerification.ExpectedWarns(0);
ExceptionVerification.ExpectedWarns(1);
}
[Test]
@ -333,7 +347,7 @@ public void GivenOldCookie()
var oldRequest = new HttpRequest($"https://{_httpBinHost2}/get");
oldRequest.Cookies["my"] = "cookie";
var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.GetMock<IUserAgentBuilder>().Object, Mocker.Resolve<Logger>());
var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.Resolve<Logger>());
oldClient.Should().NotBeSameAs(Subject);

1
src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs
View file

@ -5,5 +5,6 @@ namespace NzbDrone.Common.Http.Dispatchers
public interface IHttpDispatcher
{
HttpResponse GetResponse(HttpRequest request, CookieContainer cookies);
void DownloadFile(string url, string fileName);
}
}

51
src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs
View file

@ -1,5 +1,7 @@
using System;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Reflection;
using System.Text;
@ -74,7 +76,7 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds);
}
AddProxy(webRequest, request);
webRequest.Proxy = GetProxy(request.Url);
if (request.Headers != null)
{
@ -163,13 +165,54 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
}
protected virtual void AddProxy(HttpWebRequest webRequest, HttpRequest request)
public void DownloadFile(string url, string fileName)
{
var proxySettings = _proxySettingsProvider.GetProxySettings(request);
try
{
var fileInfo = new FileInfo(fileName);
if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
_logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
var stopWatch = Stopwatch.StartNew();
var uri = new HttpUri(url);
using (var webClient = new GZipWebClient())
{
webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
webClient.Proxy = GetProxy(uri);
webClient.DownloadFile(uri.FullUri, fileName);
stopWatch.Stop();
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
}
}
catch (WebException e)
{
_logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
throw;
}
catch (Exception e)
{
_logger.Warn(e, "Failed to get response from: " + url);
throw;
}
}
protected virtual IWebProxy GetProxy(HttpUri uri)
{
IWebProxy proxy = null;
var proxySettings = _proxySettingsProvider.GetProxySettings(uri);
if (proxySettings != null)
{
webRequest.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
}
return proxy;
}
protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers)

47
src/NzbDrone.Common/Http/HttpClient.cs
View file

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using NLog;
@ -33,19 +32,16 @@ public class HttpClient : IHttpClient
private readonly ICached<CookieContainer> _cookieContainerCache;
private readonly List<IHttpRequestInterceptor> _requestInterceptors;
private readonly IHttpDispatcher _httpDispatcher;
private readonly IUserAgentBuilder _userAgentBuilder;
public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors,
ICacheManager cacheManager,
IRateLimitService rateLimitService,
IHttpDispatcher httpDispatcher,
IUserAgentBuilder userAgentBuilder,
Logger logger)
{
_requestInterceptors = requestInterceptors.ToList();
_rateLimitService = rateLimitService;
_httpDispatcher = httpDispatcher;
_userAgentBuilder = userAgentBuilder;
_logger = logger;
ServicePointManager.DefaultConnectionLimit = 12;
@ -231,48 +227,7 @@ private void HandleResponseCookies(HttpResponse response, CookieContainer cookie
public void DownloadFile(string url, string fileName, string userAgent = null)
{
var fileNamePart = fileName + ".part";
try
{
var fileInfo = new FileInfo(fileName);
if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
_logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
var stopWatch = Stopwatch.StartNew();
using (var fileStream = new FileStream(fileNamePart, FileMode.Create, FileAccess.ReadWrite))
{
var request = new HttpRequest(url);
if (userAgent.IsNotNullOrWhiteSpace())
{
request.Headers.Set("User-Agent", userAgent);
}
request.ResponseStream = fileStream;
var response = Get(request);
}
stopWatch.Stop();
if (File.Exists(fileName))
{
File.Delete(fileName);
}
File.Move(fileNamePart, fileName);
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
}
finally
{
if (File.Exists(fileNamePart))
{
File.Delete(fileNamePart);
}
}
_httpDispatcher.DownloadFile(url, fileName);
}
public HttpResponse Get(HttpRequest request)

3
src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs
View file

@ -2,6 +2,7 @@ namespace NzbDrone.Common.Http.Proxy
{
public interface IHttpProxySettingsProvider
{
HttpProxySettings GetProxySettings(HttpRequest request);
HttpProxySettings GetProxySettings(HttpUri uri);
HttpProxySettings GetProxySettings();
}
}

2
src/NzbDrone.Core.Test/Framework/CoreTest.cs
View file

@ -26,7 +26,7 @@ protected void UseRealHttp()
Mocker.SetConstant<IHttpProxySettingsProvider>(new HttpProxySettingsProvider(Mocker.Resolve<ConfigService>()));
Mocker.SetConstant<ICreateManagedWebProxy>(new ManagedWebProxyFactory(Mocker.Resolve<CacheManager>()));
Mocker.SetConstant<IHttpDispatcher>(new ManagedHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<ICreateManagedWebProxy>(), Mocker.Resolve<UserAgentBuilder>(), Mocker.Resolve<IPlatformInfo>(), TestLogger));
Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.Resolve<UserAgentBuilder>(), TestLogger));
Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), TestLogger));
Mocker.SetConstant<IReadarrCloudRequestBuilder>(new ReadarrCloudRequestBuilder());
Mocker.SetConstant<IMetadataRequestBuilder>(Mocker.Resolve<MetadataRequestBuilder>());

27
src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs
View file

@ -15,27 +15,36 @@ public HttpProxySettingsProvider(IConfigService configService)
_configService = configService;
}
public HttpProxySettings GetProxySettings(HttpRequest request)
public HttpProxySettings GetProxySettings(HttpUri uri)
{
var proxySettings = GetProxySettings();
if (proxySettings == null)
{
return null;
}
if (ShouldProxyBeBypassed(proxySettings, uri))
{
return null;
}
return proxySettings;
}
public HttpProxySettings GetProxySettings()
{
if (!_configService.ProxyEnabled)
{
return null;
}
var proxySettings = new HttpProxySettings(_configService.ProxyType,
return new HttpProxySettings(_configService.ProxyType,
_configService.ProxyHostname,
_configService.ProxyPort,
_configService.ProxyBypassFilter,
_configService.ProxyBypassLocalAddresses,
_configService.ProxyUsername,
_configService.ProxyPassword);
if (ShouldProxyBeBypassed(proxySettings, request.Url))
{
return null;
}
return proxySettings;
}
public bool ShouldProxyBeBypassed(HttpProxySettings proxySettings, HttpUri url)