From c51161c3d1ced4aff456ea367263ab443f2a5440 Mon Sep 17 00:00:00 2001 From: Rae Knowler Date: Thu, 5 Jun 2025 18:55:53 +0200 Subject: [PATCH] Include Accept:image/* header when requesting an image url --- fanficfare/fetchers/base_fetcher.py | 16 ++++++++++------ fanficfare/fetchers/cache_basic.py | 6 ++++-- fanficfare/fetchers/cache_browser.py | 7 ++++--- fanficfare/fetchers/decorators.py | 18 ++++++++++++------ fanficfare/fetchers/fetcher_cloudscraper.py | 5 +++-- fanficfare/requestable.py | 7 ++++--- fanficfare/story.py | 8 +++++--- 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/fanficfare/fetchers/base_fetcher.py b/fanficfare/fetchers/base_fetcher.py index 5a2b15fd..7e57b2f9 100644 --- a/fanficfare/fetchers/base_fetcher.py +++ b/fanficfare/fetchers/base_fetcher.py @@ -80,11 +80,13 @@ class Fetcher(object): def set_cookiejar(self,cookiejar): self.cookiejar = cookiejar - def make_headers(self,url,referer=None): + def make_headers(self,url,referer=None,image=False): headers = {} headers['User-Agent']=self.getConfig('user_agent') if referer: headers['Referer']=referer + if image is True: + headers["Accept"] = "image/*" # if "xf2test" in url: # import base64 # base64string = base64.encodestring(b"sbreview2019:Fs2PwuVE9").replace(b'\n', b'') @@ -99,10 +101,11 @@ class Fetcher(object): def do_request(self, method, url, parameters=None, referer=None, - usecache=True): + usecache=True, + image=False): # logger.debug("fetcher do_request") # logger.debug(self.get_cookiejar()) - headers = self.make_headers(url,referer=referer) + headers = self.make_headers(url,referer=referer,image=image) fetchresp = self.request(method,url, headers=headers, parameters=parameters) @@ -129,10 +132,11 @@ class Fetcher(object): def get_request_redirected(self, url, referer=None, - usecache=True): + usecache=True, + image=False): fetchresp = self.do_request('GET', self.condition_url(url), referer=referer, - usecache=usecache) + usecache=usecache, + image=image) return (fetchresp.content,fetchresp.redirecturl) - diff --git a/fanficfare/fetchers/cache_basic.py b/fanficfare/fetchers/cache_basic.py index fdbc44c8..81c99b1a 100644 --- a/fanficfare/fetchers/cache_basic.py +++ b/fanficfare/fetchers/cache_basic.py @@ -103,7 +103,8 @@ class BasicCacheDecorator(FetcherDecorator): url, parameters=None, referer=None, - usecache=True): + usecache=True, + image=False): ''' When should cache be cleared or not used? logins, primarily Note that usecache=False prevents lookup, but cache still saves @@ -124,7 +125,8 @@ class BasicCacheDecorator(FetcherDecorator): url, parameters=parameters, referer=referer, - usecache=usecache) + usecache=usecache, + image=image) data = fetchresp.content diff --git a/fanficfare/fetchers/cache_browser.py b/fanficfare/fetchers/cache_browser.py index 4bf99642..2e1bf4fa 100644 --- a/fanficfare/fetchers/cache_browser.py +++ b/fanficfare/fetchers/cache_browser.py @@ -54,7 +54,8 @@ class BrowserCacheDecorator(FetcherDecorator): url, parameters=None, referer=None, - usecache=True): + usecache=True, + image=False): with self.cache_lock: # logger.debug("BrowserCacheDecorator fetcher_do_request") fromcache=True @@ -121,5 +122,5 @@ class BrowserCacheDecorator(FetcherDecorator): url, parameters=parameters, referer=referer, - usecache=usecache) - + usecache=usecache, + image=image) diff --git a/fanficfare/fetchers/decorators.py b/fanficfare/fetchers/decorators.py index eb661b6b..0b676651 100644 --- a/fanficfare/fetchers/decorators.py +++ b/fanficfare/fetchers/decorators.py @@ -44,14 +44,16 @@ class FetcherDecorator(object): url, parameters=None, referer=None, - usecache=True): + usecache=True, + image=False): ## can use fetcher.getConfig()/getConfigList(). fetchresp = chainfn( method, url, parameters=parameters, referer=referer, - usecache=usecache) + usecache=usecache, + image=image) return fetchresp @@ -63,14 +65,16 @@ class ProgressBarDecorator(FetcherDecorator): url, parameters=None, referer=None, - usecache=True): + usecache=True, + image=False): # logger.debug("ProgressBarDecorator fetcher_do_request") fetchresp = chainfn( method, url, parameters=parameters, referer=referer, - usecache=usecache) + usecache=usecache, + image=image) ## added ages ago for CLI to give a line of dots showing it's ## doing something. sys.stdout.write('.') @@ -97,14 +101,16 @@ class SleepDecorator(FetcherDecorator): url, parameters=None, referer=None, - usecache=True): + usecache=True, + image=False): # logger.debug("SleepDecorator fetcher_do_request") fetchresp = chainfn( method, url, parameters=parameters, referer=referer, - usecache=usecache) + usecache=usecache, + image=image) # don't sleep cached results. Usually MemCache results will # be before sleep, but check fetchresp.fromcache for file:// diff --git a/fanficfare/fetchers/fetcher_cloudscraper.py b/fanficfare/fetchers/fetcher_cloudscraper.py index 76443781..353a7e41 100644 --- a/fanficfare/fetchers/fetcher_cloudscraper.py +++ b/fanficfare/fetchers/fetcher_cloudscraper.py @@ -54,9 +54,10 @@ class CloudScraperFetcher(RequestsFetcher): source_address=session.source_address, max_retries=self.retries)) - def make_headers(self,url,referer=None): + def make_headers(self,url,referer=None,image=False): headers = super(CloudScraperFetcher,self).make_headers(url, - referer=referer) + referer=referer, + image=image) ## let cloudscraper do its thing with UA. if 'User-Agent' in headers: del headers['User-Agent'] diff --git a/fanficfare/requestable.py b/fanficfare/requestable.py index 13c41531..5189836b 100644 --- a/fanficfare/requestable.py +++ b/fanficfare/requestable.py @@ -124,9 +124,10 @@ class Requestable(Configurable): def get_request_raw(self, url, referer=None, - usecache=True): ## referer is used with raw for images. + usecache=True, + image=False): ## referer is used with raw for images. return self.configuration.get_fetcher().get_request_redirected( self.mod_url_request(url), referer=referer, - usecache=usecache)[0] - + usecache=usecache, + image=image)[0] diff --git a/fanficfare/story.py b/fanficfare/story.py index 76b16e27..4c0e8129 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -766,11 +766,13 @@ class Story(Requestable): self.getConfigList) def get_request_raw(url, referer=None, - usecache=True): ## referer is used with raw for images. + usecache=True, + image=False): ## referer is used with raw for images. return fetcher.get_request_redirected( url, referer=referer, - usecache=usecache)[0] + usecache=usecache, + image=image)[0] self.direct_fetcher = get_request_raw def prepare_replacements(self): @@ -1647,7 +1649,7 @@ class Story(Requestable): url) ): refererurl = url logger.debug("Use Referer:%s"%refererurl) - imgdata = fetch(imgurl,referer=refererurl) + imgdata = fetch(imgurl,referer=refererurl,image=True) if self.no_image_processing(imgurl): (data,ext,mime) = no_convert_image(imgurl,