Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2025-12-06 08:52:55 +01:00
Code Issues Projects Releases Wiki Activity

Include Accept:image/* header when requesting an image url

Browse source
This commit is contained in:
Rae Knowler 2025-06-05 18:55:53 +02:00 committed by Jim Miller
parent bd645a97c7
commit c51161c3d1
7 changed files with 42 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
fanficfare/fetchers/base_fetcher.py
View file

@ -80,11 +80,13 @@ class Fetcher(object):
def set_cookiejar(self,cookiejar): def set_cookiejar(self,cookiejar):
self.cookiejar = cookiejar self.cookiejar = cookiejar
def make_headers(self,url,referer=None): def make_headers(self,url,referer=None,image=False):
headers = {} headers = {}
headers['User-Agent']=self.getConfig('user_agent') headers['User-Agent']=self.getConfig('user_agent')
if referer: if referer:
headers['Referer']=referer headers['Referer']=referer
if image is True:
headers["Accept"] = "image/*"
# if "xf2test" in url: # if "xf2test" in url:
# import base64 # import base64
# base64string = base64.encodestring(b"sbreview2019:Fs2PwuVE9").replace(b'\n', b'') # base64string = base64.encodestring(b"sbreview2019:Fs2PwuVE9").replace(b'\n', b'')
@ -99,10 +101,11 @@ class Fetcher(object):
def do_request(self, method, url, def do_request(self, method, url,
parameters=None, parameters=None,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
# logger.debug("fetcher do_request") # logger.debug("fetcher do_request")
# logger.debug(self.get_cookiejar()) # logger.debug(self.get_cookiejar())
headers = self.make_headers(url,referer=referer) headers = self.make_headers(url,referer=referer,image=image)
fetchresp = self.request(method,url, fetchresp = self.request(method,url,
headers=headers, headers=headers,
parameters=parameters) parameters=parameters)
@ -129,10 +132,11 @@ class Fetcher(object):
def get_request_redirected(self, url, def get_request_redirected(self, url,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
fetchresp = self.do_request('GET', fetchresp = self.do_request('GET',
self.condition_url(url), self.condition_url(url),
referer=referer, referer=referer,
usecache=usecache) usecache=usecache,
image=image)
return (fetchresp.content,fetchresp.redirecturl) return (fetchresp.content,fetchresp.redirecturl)

6
fanficfare/fetchers/cache_basic.py
View file

@ -103,7 +103,8 @@ class BasicCacheDecorator(FetcherDecorator):
url, url,
parameters=None, parameters=None,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
''' '''
When should cache be cleared or not used? logins, primarily When should cache be cleared or not used? logins, primarily
Note that usecache=False prevents lookup, but cache still saves Note that usecache=False prevents lookup, but cache still saves
@ -124,7 +125,8 @@ class BasicCacheDecorator(FetcherDecorator):
url, url,
parameters=parameters, parameters=parameters,
referer=referer, referer=referer,
usecache=usecache) usecache=usecache,
image=image)
data = fetchresp.content data = fetchresp.content

7
fanficfare/fetchers/cache_browser.py
View file

@ -54,7 +54,8 @@ class BrowserCacheDecorator(FetcherDecorator):
url, url,
parameters=None, parameters=None,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
with self.cache_lock: with self.cache_lock:
# logger.debug("BrowserCacheDecorator fetcher_do_request") # logger.debug("BrowserCacheDecorator fetcher_do_request")
fromcache=True fromcache=True
@ -121,5 +122,5 @@ class BrowserCacheDecorator(FetcherDecorator):
url, url,
parameters=parameters, parameters=parameters,
referer=referer, referer=referer,
usecache=usecache) usecache=usecache,
image=image)

18
fanficfare/fetchers/decorators.py
View file

@ -44,14 +44,16 @@ class FetcherDecorator(object):
url, url,
parameters=None, parameters=None,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
## can use fetcher.getConfig()/getConfigList(). ## can use fetcher.getConfig()/getConfigList().
fetchresp = chainfn( fetchresp = chainfn(
method, method,
url, url,
parameters=parameters, parameters=parameters,
referer=referer, referer=referer,
usecache=usecache) usecache=usecache,
image=image)
return fetchresp return fetchresp
@ -63,14 +65,16 @@ class ProgressBarDecorator(FetcherDecorator):
url, url,
parameters=None, parameters=None,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
# logger.debug("ProgressBarDecorator fetcher_do_request") # logger.debug("ProgressBarDecorator fetcher_do_request")
fetchresp = chainfn( fetchresp = chainfn(
method, method,
url, url,
parameters=parameters, parameters=parameters,
referer=referer, referer=referer,
usecache=usecache) usecache=usecache,
image=image)
## added ages ago for CLI to give a line of dots showing it's ## added ages ago for CLI to give a line of dots showing it's
## doing something. ## doing something.
sys.stdout.write('.') sys.stdout.write('.')
@ -97,14 +101,16 @@ class SleepDecorator(FetcherDecorator):
url, url,
parameters=None, parameters=None,
referer=None, referer=None,
usecache=True): usecache=True,
image=False):
# logger.debug("SleepDecorator fetcher_do_request") # logger.debug("SleepDecorator fetcher_do_request")
fetchresp = chainfn( fetchresp = chainfn(
method, method,
url, url,
parameters=parameters, parameters=parameters,
referer=referer, referer=referer,
usecache=usecache) usecache=usecache,
image=image)
# don't sleep cached results. Usually MemCache results will # don't sleep cached results. Usually MemCache results will
# be before sleep, but check fetchresp.fromcache for file:// # be before sleep, but check fetchresp.fromcache for file://

5
fanficfare/fetchers/fetcher_cloudscraper.py
View file

@ -54,9 +54,10 @@ class CloudScraperFetcher(RequestsFetcher):
source_address=session.source_address, source_address=session.source_address,
max_retries=self.retries)) max_retries=self.retries))
def make_headers(self,url,referer=None): def make_headers(self,url,referer=None,image=False):
headers = super(CloudScraperFetcher,self).make_headers(url, headers = super(CloudScraperFetcher,self).make_headers(url,
referer=referer) referer=referer,
image=image)
## let cloudscraper do its thing with UA. ## let cloudscraper do its thing with UA.
if 'User-Agent' in headers: if 'User-Agent' in headers:
del headers['User-Agent'] del headers['User-Agent']

7
fanficfare/requestable.py
View file

@ -124,9 +124,10 @@ class Requestable(Configurable):
def get_request_raw(self, url, def get_request_raw(self, url,
referer=None, referer=None,
usecache=True): ## referer is used with raw for images. usecache=True,
image=False): ## referer is used with raw for images.
return self.configuration.get_fetcher().get_request_redirected( return self.configuration.get_fetcher().get_request_redirected(
self.mod_url_request(url), self.mod_url_request(url),
referer=referer, referer=referer,
usecache=usecache)[0] usecache=usecache,
image=image)[0]

8
fanficfare/story.py
View file

@ -766,11 +766,13 @@ class Story(Requestable):
self.getConfigList) self.getConfigList)
def get_request_raw(url, def get_request_raw(url,
referer=None, referer=None,
usecache=True): ## referer is used with raw for images. usecache=True,
image=False): ## referer is used with raw for images.
return fetcher.get_request_redirected( return fetcher.get_request_redirected(
url, url,
referer=referer, referer=referer,
usecache=usecache)[0] usecache=usecache,
image=image)[0]
self.direct_fetcher = get_request_raw self.direct_fetcher = get_request_raw
def prepare_replacements(self): def prepare_replacements(self):
@ -1647,7 +1649,7 @@ class Story(Requestable):
url) ): url) ):
refererurl = url refererurl = url
logger.debug("Use Referer:%s"%refererurl) logger.debug("Use Referer:%s"%refererurl)
imgdata = fetch(imgurl,referer=refererurl) imgdata = fetch(imgurl,referer=refererurl,image=True)
if self.no_image_processing(imgurl): if self.no_image_processing(imgurl):
(data,ext,mime) = no_convert_image(imgurl, (data,ext,mime) = no_convert_image(imgurl,