Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2025-12-06 17:02:43 +01:00
Code Issues Projects Releases Wiki Activity

BrowserCache: Better handle cache file changing/failing while reading.

Browse source
This commit is contained in:
Jim Miller 2025-02-24 21:24:43 -06:00
parent 82825d1b16
commit c379b45cb9
1 changed files with 9 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
fanficfare/fetchers/cache_browser.py
View file

@ -76,11 +76,18 @@ class BrowserCacheDecorator(FetcherDecorator):
# logger.debug("First time for (%s) extra sleep"%parsedUrl.netloc) # logger.debug("First time for (%s) extra sleep"%parsedUrl.netloc)
# time.sleep(10) # time.sleep(10)
fromcache=False fromcache=False
read_try_sleeps = [2, 2, 4, 5, 6] read_try_sleeps = [2, 2, 4, 10, 20]
while not d and read_try_sleeps: while not d and read_try_sleeps:
time.sleep(read_try_sleeps.pop(0)) time.sleep(read_try_sleeps.pop(0))
logger.debug("Checking for cache...") logger.debug("Checking for cache...")
d = self.cache.get_data(url) try:
d = self.cache.get_data(url)
except Exception as e:
## catch exception while retrying, but
## re-raise if out of retries.
logger.debug("Exception reading cache after open_pages_in_browser %s"%e)
if not read_try_sleeps:
raise
# logger.debug(d) # logger.debug(d)
open_tries -= 1 open_tries -= 1
domain_open_tries[parsedUrl.netloc] = domain_open_tries.get(parsedUrl.netloc,0) + 1 domain_open_tries[parsedUrl.netloc] = domain_open_tries.get(parsedUrl.netloc,0) + 1