'+ - _("%s requires that you be an adult. Please confirm you are an adult in your locale:")%url, - show_copy_button=False): - adapter.is_adult=True - - # let other exceptions percolate up. - story = adapter.getStoryMetadataOnly(get_cover=False) - book['title'] = story.getMetadata('title') - book['author'] = [story.getMetadata('author')] - book['url'] = story.getMetadata('storyUrl') - + + if not bgmeta: + ## three tries, that's enough if both user/pass & is_adult needed, + ## or a couple tries of one or the other + for x in range(0,2): + try: + adapter.getStoryMetadataOnly(get_cover=False) + except exceptions.FailedToLogin, f: + logger.warn("Login Failed, Need Username/Password.") + userpass = UserPassDialog(self.gui,url,f) + userpass.exec_() # exec_ will make it act modal + if userpass.status: + adapter.username = userpass.user.text() + adapter.password = userpass.passwd.text() + + except exceptions.AdultCheckRequired: + if question_dialog(self.gui, _('Are You an Adult?'), '
'+ + _("%s requires that you be an adult. Please confirm you are an adult in your locale:")%url, + show_copy_button=False): + adapter.is_adult=True + + # let other exceptions percolate up. + story = adapter.getStoryMetadataOnly(get_cover=False) + book['title'] = story.getMetadata('title') + book['author'] = [story.getMetadata('author')] + book['url'] = story.getMetadata('storyUrl') + ## Check reject list. Redundant with below for when story ## URL changes, but also kept here to avoid network hit in ## most common case where given url is story url. if self.reject_url(merge,book): return - series = story.getMetadata('series') - if not merge and series and prefs['checkforseriesurlid']: - # try to find *series anthology* by *seriesUrl* identifier url or uri first. - identicalbooks = self.do_id_search(story.getMetadata('seriesUrl')) - # print("identicalbooks:%s"%identicalbooks) - if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),''' -
%s
-%s
-%s
- '''%( - _('Skip Anthology Story?'), - _('"%s" is in series "%s" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]), - _("Click 'Yes' to Skip."), - _("Click 'No' to download anyway.")), - show_copy_button=False): - book['comment'] = _("Story in Series Anthology(%s).")%series - book['title'] = story.getMetadata('title') - book['author'] = [story.getMetadata('author')] - book['url'] = story.getMetadata('storyUrl') - book['good']=False - book['icon']='rotate-right.png' - book['status'] = _('Skipped') - return - + if not bgmeta: + series = story.getMetadata('series') + if not merge and series and prefs['checkforseriesurlid']: + # try to find *series anthology* by *seriesUrl* identifier url or uri first. + identicalbooks = self.do_id_search(story.getMetadata('seriesUrl')) + # print("identicalbooks:%s"%identicalbooks) + if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),''' +%s
+%s
+%s
+ '''%( + _('Skip Anthology Story?'), + _('"%s" is in series "%s" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]), + _("Click 'Yes' to Skip."), + _("Click 'No' to download anyway.")), + show_copy_button=False): + book['comment'] = _("Story in Series Anthology(%s).")%series + book['title'] = story.getMetadata('title') + book['author'] = [story.getMetadata('author')] + book['url'] = story.getMetadata('storyUrl') + book['good']=False + book['icon']='rotate-right.png' + book['status'] = _('Skipped') + return + ################################################################################################################################################33 - # set PI version instead of default. - if 'version' in options: - story.setMetadata('version',options['version']) - - # all_metadata duplicates some data, but also includes extra_entries, etc. - book['all_metadata'] = story.getAllMetadata(removeallentities=True) - if prefs['savemetacol'] != '': - # get metadata to save in configured column. - book['savemetacol'] = story.dump_html_metadata() - - book['title'] = story.getMetadata("title", removeallentities=True) - book['author_sort'] = book['author'] = story.getList("author", removeallentities=True) - book['publisher'] = story.getMetadata("site") - book['url'] = story.getMetadata("storyUrl") - book['tags'] = story.getSubjectTags(removeallentities=True) - if story.getMetadata("description"): - book['comments'] = sanitize_comments_html(story.getMetadata("description")) - else: - book['comments']='' - book['series'] = story.getMetadata("series", removeallentities=True) - book['is_adult'] = adapter.is_adult book['username'] = adapter.username book['password'] = adapter.password book['icon'] = 'plus.png' book['status'] = _('Add') - if story.getMetadataRaw('datePublished'): - book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz) - if story.getMetadataRaw('dateUpdated'): - book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz) - if story.getMetadataRaw('dateCreated'): - book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz) - else: - book['timestamp'] = None # need *something* there for calibre. + + if not bgmeta: + # set PI version instead of default. + if 'version' in options: + story.setMetadata('version',options['version']) + # all_metadata duplicates some data, but also includes extra_entries, etc. + book['all_metadata'] = story.getAllMetadata(removeallentities=True) + if prefs['savemetacol'] != '': + # get metadata to save in configured column. + book['savemetacol'] = story.dump_html_metadata() + + book['title'] = story.getMetadata("title", removeallentities=True) + book['author_sort'] = book['author'] = story.getList("author", removeallentities=True) + book['publisher'] = story.getMetadata("site") + book['url'] = story.getMetadata("storyUrl") + book['tags'] = story.getSubjectTags(removeallentities=True) + if story.getMetadata("description"): + book['comments'] = sanitize_comments_html(story.getMetadata("description")) + else: + book['comments']='' + book['series'] = story.getMetadata("series", removeallentities=True) + + if story.getMetadataRaw('datePublished'): + book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz) + if story.getMetadataRaw('dateUpdated'): + book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz) + if story.getMetadataRaw('dateCreated'): + book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz) + else: + book['timestamp'] = None # need *something* there for calibre. + if not merge:# skip all the collision code when d/ling for merging. if collision in (CALIBREONLY, CALIBREONLYSAVECOL): book['icon'] = 'metadata.png' @@ -1182,9 +1204,7 @@ class FanFicFarePlugin(InterfaceAction): # print("identicalbooks:%s"%identicalbooks) if len(identicalbooks) < 1 and prefs['matchtitleauth']: # find dups - authlist = story.getList("author", removeallentities=True) - mi = MetaInformation(story.getMetadata("title", removeallentities=True), - authlist) + mi = MetaInformation(book['title'],book['author']) identicalbooks = db.find_identical_books(mi) if len(identicalbooks) > 0: logger.debug("existing found by title/author(s)") @@ -1261,7 +1281,7 @@ class FanFicFarePlugin(InterfaceAction): ## newer/chaptercount checks are the same for both: # Update epub, but only if more chapters. - if collision in (UPDATE,UPDATEALWAYS): # collision == UPDATE + if not bgmeta and collision in (UPDATE,UPDATEALWAYS): # collision == UPDATE # 'book' can exist without epub. If there's no existing epub, # let it go and it will download it. if db.has_format(book_id,fileform,index_is_id=True): @@ -1279,17 +1299,17 @@ class FanFicFarePlugin(InterfaceAction): if collision == OVERWRITE and \ db.has_format(book_id,formmapping[fileform],index_is_id=True): - # check make sure incoming is newer. - lastupdated=story.getMetadataRaw('dateUpdated') fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8]) - - # updated doesn't have time (or is midnight), use dates only. - # updated does have time, use full timestamps. - if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \ - (lastupdated.time() != time.min and fileupdated > lastupdated): - raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png') - - + book['fileupdated']=fileupdated + if not bgmeta: + # check make sure incoming is newer. + lastupdated=story.getMetadataRaw('dateUpdated') + + # updated doesn't have time (or is midnight), use dates only. + # updated does have time, use full timestamps. + if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \ + (lastupdated.time() != time.min and fileupdated > lastupdated): + raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png') # For update, provide a tmp file copy of the existing epub so # it can't change underneath us. Now also overwrite for logpage preserve. @@ -1360,7 +1380,11 @@ class FanFicFarePlugin(InterfaceAction): # For HTML format users, make the filename inside the zip something reasonable. # For crazy long titles/authors, limit it to 200chars. # For weird/OS-unsafe characters, use file safe only. - tmp = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100], + try: + prefix = story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100] + except NameError: + prefix = "bgmeta-" + tmp = PersistentTemporaryFile(prefix=prefix, suffix='.'+options['fileform'], dir=options['tdir']) logger.debug("title:"+book['title']) @@ -1373,6 +1397,7 @@ class FanFicFarePlugin(InterfaceAction): options={'fileform':'epub', 'collision':ADDNEW, 'updatemeta':True, + 'bgmeta':False, 'updateepubcover':True}, merge=False): ''' @@ -1438,7 +1463,7 @@ class FanFicFarePlugin(InterfaceAction): func = 'arbitrary_n' cpus = self.gui.job_manager.server.pool_size args = ['calibre_plugins.fanficfare_plugin.jobs', 'do_download_worker', - (book_list, options, cpus)] + (book_list, options, cpus, merge)] desc = _('Download FanFiction Book') job = self.gui.job_manager.run_job( self.Dispatcher(partial(self.download_list_completed,options=options,merge=merge)), @@ -1452,6 +1477,7 @@ class FanFicFarePlugin(InterfaceAction): options={'fileform':'epub', 'collision':ADDNEW, 'updatemeta':True, + 'bgmeta':False, 'updateepubcover':True}): custom_columns = self.gui.library_view.model().custom_columns if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns: @@ -2155,6 +2181,7 @@ class FanFicFarePlugin(InterfaceAction): book['comment'] = '' # note this is a comment on the d/l or update. book['url'] = '' book['site'] = '' + book['series'] = '' book['added'] = False book['pubdate'] = None book['publisher'] = None diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index de8e017e..54639757 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -10,12 +10,15 @@ __docformat__ = 'restructuredtext en' import logging logger = logging.getLogger(__name__) -import time, traceback +import traceback +from datetime import time from StringIO import StringIO from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob from calibre.constants import numeric_version as calibre_version +from calibre.utils.date import local_tz +from calibre.library.comments import sanitize_comments_html # ------------------------------------------------------------------------------ # @@ -23,8 +26,11 @@ from calibre.constants import numeric_version as calibre_version # # ------------------------------------------------------------------------------ -def do_download_worker(book_list, options, - cpus, notification=lambda x,y:x): +def do_download_worker(book_list, + options, + cpus, + merge=False, + notification=lambda x,y:x): ''' Master job, to launch child jobs to extract ISBN for a set of books This is run as a worker job in the background to keep the UI more @@ -44,7 +50,7 @@ def do_download_worker(book_list, options, total += 1 args = ['calibre_plugins.fanficfare_plugin.jobs', 'do_download_for_worker', - (book,options)] + (book,options,merge)] job = ParallelJob('arbitrary_n', "url:(%s) id:(%s)"%(book['url'],book['calibre_id']), done=None, @@ -90,7 +96,7 @@ def do_download_worker(book_list, options, # return the book list as the job result return book_list -def do_download_for_worker(book,options,notification=lambda x,y:x): +def do_download_for_worker(book,options,merge,notification=lambda x,y:x): ''' Child job, to download story when run as a worker job ''' @@ -147,6 +153,26 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): if 'version' in options: story.setMetadata('version',options['version']) + book['title'] = story.getMetadata("title", removeallentities=True) + book['author_sort'] = book['author'] = story.getList("author", removeallentities=True) + book['publisher'] = story.getMetadata("site") + book['url'] = story.getMetadata("storyUrl") + book['tags'] = story.getSubjectTags(removeallentities=True) + if story.getMetadata("description"): + book['comments'] = sanitize_comments_html(story.getMetadata("description")) + else: + book['comments']='' + book['series'] = story.getMetadata("series", removeallentities=True) + + if story.getMetadataRaw('datePublished'): + book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz) + if story.getMetadataRaw('dateUpdated'): + book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz) + if story.getMetadataRaw('dateCreated'): + book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz) + else: + book['timestamp'] = None # need *something* there for calibre. + writer = writers.getWriter(options['fileform'],configuration,adapter) outfile = book['outfile'] @@ -165,12 +191,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): # preserve logfile even on overwrite. if 'epub_for_update' in book: - adapter.logfile = get_update_data(book['epub_for_update'])[6] # change the existing entries id to notid so # write_epub writes a whole new set to indicate overwrite. if adapter.logfile: adapter.logfile = adapter.logfile.replace("span id","span notid") + + if options['collision'] == OVERWRITE: + lastupdated=story.getMetadataRaw('dateUpdated') + fileupdated=book['fileupdated'] + + # updated doesn't have time (or is midnight), use dates only. + # updated does have time, use full timestamps. + if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \ + (lastupdated.time() != time.min and fileupdated > lastupdated): + raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png') + logger.info("write to %s"%outfile) inject_cal_cols(book,story,configuration) @@ -199,17 +235,20 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): # dup handling from fff_plugin needed for anthology updates. if options['collision'] == UPDATE: if chaptercount == urlchaptercount: - book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount - book['all_metadata'] = story.getAllMetadata(removeallentities=True) - if options['savemetacol'] != '': - book['savemetacol'] = story.dump_html_metadata() - book['outfile'] = book['epub_for_update'] # for anthology merge ops. - return book - - # dup handling from fff_plugin needed for anthology updates. - if chaptercount > urlchaptercount: - raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png') - + if merge: + book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount + book['all_metadata'] = story.getAllMetadata(removeallentities=True) + if options['savemetacol'] != '': + book['savemetacol'] = story.dump_html_metadata() + book['outfile'] = book['epub_for_update'] # for anthology merge ops. + return book + else: # not merge, + raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png') + elif chaptercount > urlchaptercount: + raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png') + elif chaptercount == 0: + raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png') + if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \ and adapter.getConfig("do_update_hook"): chaptercount = adapter.hookForUpdates(chaptercount) diff --git a/calibre-plugin/prefs.py b/calibre-plugin/prefs.py index a8eaabcf..df4b6ac3 100644 --- a/calibre-plugin/prefs.py +++ b/calibre-plugin/prefs.py @@ -59,6 +59,7 @@ Dup from another site''' default_prefs['reject_always'] = False default_prefs['updatemeta'] = True +default_prefs['bgmeta'] = False default_prefs['updateepubcover'] = False default_prefs['keeptags'] = False default_prefs['suppressauthorsort'] = False