diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index a32aec82..86797816 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -673,6 +673,15 @@ browser_cache_age_limit:4.0 ## an error message in the ebook for that chapter. continue_on_chapter_error:false +## When continue_on_chapter_error:true, after +## continue_on_chapter_error_try_limit chapters have failed, continue +## processing, but stop trying to download chapters. Mark all such +## chapters with chapter_title_error_mark, but chapter text will +## explain that no download attempt was made because +## continue_on_chapter_error_try_limit was exceeded. Set to -1 for +## infinite chapter errors. +continue_on_chapter_error_try_limit:5 + ## Append this to chapter titles that errored. Only used with ## continue_on_chapter_error:true ## Set empty to not mark failed chapters. diff --git a/fanficfare/adapters/adapter_test1.py b/fanficfare/adapters/adapter_test1.py index 0ea749a4..dd978b4e 100644 --- a/fanficfare/adapters/adapter_test1.py +++ b/fanficfare/adapters/adapter_test1.py @@ -366,7 +366,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
''' - elif self.story.getMetadata('storyId') == '667' and "chapter=2" in url: + elif self.story.getMetadata('storyId') == '667' and ("chapter=2" in url or "chapter=3" in url or "chapter=4" in url): raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url) elif self.getSiteDomain() not in url: ## for chapter_urls setting. diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index c14eb809..6b86f10b 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -230,6 +230,21 @@ class BaseSiteAdapter(Requestable): percent = 0.0 per_step = 1.0/self.story.getChapterCount() # logger.debug("self.story.getChapterCount():%s per_step:%s"%(self.story.getChapterCount(),per_step)) + continue_on_chapter_error_try_limit = 5 + try: + continue_on_chapter_error_try_limit = int(self.getConfig('continue_on_chapter_error_try_limit', + continue_on_chapter_error_try_limit)) + except: + logger.warning('Parsing continue_on_chapter_error_try_limit:%s failed, using %s'%( + self.getConfig('continue_on_chapter_error_try_limit'), + continue_on_chapter_error_try_limit)) + + def do_error_chapter(txt,title): + data = self.make_soup(txt) + title = title+self.getConfig("chapter_title_error_mark","(CHAPTER ERROR)") + url="chapter url removed due to failure" + return data, title, url + for index, chap in enumerate(self.chapterUrls): title = chap['title'] url = chap['url'] @@ -263,9 +278,21 @@ class BaseSiteAdapter(Requestable): try: if not data: - data = self.getChapterTextNum(url,index) - # if had to fetch and has existing chapters - newchap = bool(self.oldchapters or self.oldchaptersmap) + if( self.getConfig('continue_on_chapter_error') and + continue_on_chapter_error_try_limit > 0 and # for -1 == infinite + self.story.chapter_error_count >= continue_on_chapter_error_try_limit ): + data, title, url = do_error_chapter("""
+

Error

+

FanFicFare didn't try to download this chapter, due to earlier chapter errors.

+Because continue_on_chapter_error:true is set, processing continued, but because +continue_on_chapter_error_try_limit(%s) has been exceeded, this chapter did not +try to download.

+

Chapter URL:
%s

+
"""%(continue_on_chapter_error_try_limit,url,url),title) + else: + data = self.getChapterTextNum(url,index) + # if had to fetch and has existing chapters + newchap = bool(self.oldchapters or self.oldchaptersmap) if index == 0 and self.getConfig('always_reload_first_chapter'): data = self.getChapterTextNum(url,index) @@ -275,17 +302,15 @@ class BaseSiteAdapter(Requestable): newchap = False except Exception as e: if self.getConfig('continue_on_chapter_error',False): - data = self.make_soup("""
+ data, title, url = do_error_chapter("""

Error

FanFicFare failed to download this chapter. Because continue_on_chapter_error is set to true, the download continued.

Chapter URL:
%s

Error:

%s

-
"""%(url,url,traceback.format_exc().replace("&","&").replace(">",">").replace("<","<"))) - title = title+self.getConfig("chapter_title_error_mark","(CHAPTER ERROR)") +
"""%(url,url,traceback.format_exc().replace("&","&").replace(">",">").replace("<","<")),title) logger.info("continue_on_chapter_error: (%s) %s"%(url,e)) logger.debug(traceback.format_exc()) - url="chapter url removed due to failure" self.story.chapter_error_count += 1 else: raise diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index 5e71cfd3..5a0ecae0 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -568,6 +568,7 @@ def get_valid_keywords(): 'capitalize_forumtags', 'continue_on_chapter_error', 'chapter_title_error_mark', + 'continue_on_chapter_error_try_limit', 'minimum_threadmarks', 'first_post_title', 'always_include_first_post', diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 17097f9b..a90d679e 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -631,6 +631,15 @@ browser_cache_age_limit:4.0 ## an error message in the ebook for that chapter. continue_on_chapter_error:false +## When continue_on_chapter_error:true, after +## continue_on_chapter_error_try_limit chapters have failed, continue +## processing, but stop trying to download chapters. Mark all such +## chapters with chapter_title_error_mark, but chapter text will +## explain that no download attempt was made because +## continue_on_chapter_error_try_limit was exceeded. Set to -1 for +## infinite chapter errors. +continue_on_chapter_error_try_limit:5 + ## Append this to chapter titles that errored. Only used with ## continue_on_chapter_error:true ## Set empty to not mark failed chapters.