mirror of
https://github.com/JimmXinu/FanFicFare.git
synced 2025-12-06 08:52:55 +01:00
Add Background Metadata feature for updates.
This commit is contained in:
Jim Miller
parent
37a084a699
commit
be04eed180
5 changed files with 224 additions and 126 deletions
|
|
@ -243,6 +243,7 @@ class ConfigWidget(QWidget):
|
|||
prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
|
||||
prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
|
||||
prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
|
||||
prefs['bgmeta'] = self.basic_tab.bgmeta.isChecked()
|
||||
prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
|
||||
prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
|
||||
prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
|
||||
|
|
@ -430,6 +431,12 @@ class BasicTab(QWidget):
|
|||
self.updateepubcover.setToolTip(_("On each download, FanFicFare offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
|
||||
self.updateepubcover.setChecked(prefs['updateepubcover'])
|
||||
horz.addWidget(self.updateepubcover)
|
||||
|
||||
self.bgmeta = QCheckBox(_('Default Background Metadata?'),self)
|
||||
self.bgmeta.setToolTip(_("On each download, FanFicFare offers an option to Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail.<br />Only available for Update/Overwrite of existing books in case URL given isn't canonical or matches to existing book by Title/Author."))
|
||||
self.bgmeta.setChecked(prefs['bgmeta'])
|
||||
horz.addWidget(self.bgmeta)
|
||||
|
||||
self.l.addLayout(horz)
|
||||
|
||||
cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
|
||||
|
|
@ -457,7 +464,7 @@ class BasicTab(QWidget):
|
|||
self.l.addWidget(self.suppresstitlesort)
|
||||
|
||||
self.checkforseriesurlid = QCheckBox(_("Check for existing Series Anthology books?"),self)
|
||||
self.checkforseriesurlid.setToolTip(_("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found."))
|
||||
self.checkforseriesurlid.setToolTip(_("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.\nDoesn't work when Collect Metadata in Background is selected."))
|
||||
self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
|
||||
self.l.addWidget(self.checkforseriesurlid)
|
||||
|
||||
|
|
|
|||
|
|
@ -335,6 +335,18 @@ class AddNewDialog(SizePersistedDialog):
|
|||
|
||||
self.gbl.addLayout(horz)
|
||||
|
||||
## bgmeta not used with Add New because of stories that change
|
||||
## story URL and for title/author collision matching.
|
||||
# horz = QHBoxLayout()
|
||||
# self.bgmeta = QCheckBox(_('Background Metadata?'),self)
|
||||
# self.bgmeta.setToolTip(_("Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail."))
|
||||
# self.bgmeta.setChecked(self.prefs['bgmeta'])
|
||||
# horz.addWidget(self.bgmeta)
|
||||
# self.mergehide.append(self.bgmeta)
|
||||
# self.mergeupdateshow.append(self.bgmeta)
|
||||
|
||||
# self.gbl.addLayout(horz)
|
||||
|
||||
self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
|
||||
self.button_box.accepted.connect(self.ok_clicked)
|
||||
self.button_box.rejected.connect(self.reject)
|
||||
|
|
@ -417,6 +429,7 @@ class AddNewDialog(SizePersistedDialog):
|
|||
self.collision.setCurrentIndex(i)
|
||||
|
||||
self.updatemeta.setChecked(self.prefs['updatemeta'])
|
||||
# self.bgmeta.setChecked(self.prefs['bgmeta'])
|
||||
|
||||
if not self.merge:
|
||||
self.updateepubcover.setChecked(self.prefs['updateepubcover'])
|
||||
|
|
@ -457,6 +470,7 @@ class AddNewDialog(SizePersistedDialog):
|
|||
'fileform': unicode(self.fileform.currentText()),
|
||||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': False, # self.bgmeta.isChecked(),
|
||||
'updateepubcover': self.updateepubcover.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation']
|
||||
}
|
||||
|
|
@ -787,15 +801,18 @@ class UpdateExistingDialog(SizePersistedDialog):
|
|||
gbl = QVBoxLayout()
|
||||
gbl.addWidget(gbf)
|
||||
groupbox.setLayout(gbl)
|
||||
gbl = QHBoxLayout()
|
||||
gbl = QVBoxLayout()
|
||||
gbf.setLayout(gbl)
|
||||
options_layout.addWidget(groupbox)
|
||||
|
||||
gbf.setVisible(False)
|
||||
groupbox.toggled.connect(gbf.setVisible)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
gbl.addLayout(horz)
|
||||
|
||||
label = QLabel(_('Output &Format:'))
|
||||
gbl.addWidget(label)
|
||||
horz.addWidget(label)
|
||||
self.fileform = QComboBox(self)
|
||||
self.fileform.addItem('epub')
|
||||
self.fileform.addItem('mobi')
|
||||
|
|
@ -805,10 +822,10 @@ class UpdateExistingDialog(SizePersistedDialog):
|
|||
self.fileform.setToolTip(_('Choose output format to create. May set default from plugin configuration.'))
|
||||
self.fileform.activated.connect(self.set_collisions)
|
||||
label.setBuddy(self.fileform)
|
||||
gbl.addWidget(self.fileform)
|
||||
horz.addWidget(self.fileform)
|
||||
|
||||
label = QLabel(_('Update Mode:'))
|
||||
gbl.addWidget(label)
|
||||
horz.addWidget(label)
|
||||
self.collision = QComboBox(self)
|
||||
self.collision.setToolTip(_("What sort of update to perform. May set default from plugin configuration."))
|
||||
# add collision options
|
||||
|
|
@ -817,19 +834,25 @@ class UpdateExistingDialog(SizePersistedDialog):
|
|||
if i > -1:
|
||||
self.collision.setCurrentIndex(i)
|
||||
label.setBuddy(self.collision)
|
||||
gbl.addWidget(self.collision)
|
||||
horz.addWidget(self.collision)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
gbl.addLayout(horz)
|
||||
|
||||
self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
|
||||
self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
|
||||
self.updatemeta.setChecked(self.prefs['updatemeta'])
|
||||
gbl.addWidget(self.updatemeta)
|
||||
horz.addWidget(self.updatemeta)
|
||||
|
||||
self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
|
||||
self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
|
||||
self.updateepubcover.setChecked(self.prefs['updateepubcover'])
|
||||
gbl.addWidget(self.updateepubcover)
|
||||
|
||||
horz.addWidget(self.updateepubcover)
|
||||
|
||||
self.bgmeta = QCheckBox(_('Background Metadata?'),self)
|
||||
self.bgmeta.setToolTip(_("Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail."))
|
||||
self.bgmeta.setChecked(self.prefs['bgmeta'])
|
||||
horz.addWidget(self.bgmeta)
|
||||
|
||||
button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
|
||||
button_box.accepted.connect(self.accept)
|
||||
|
|
@ -872,6 +895,7 @@ class UpdateExistingDialog(SizePersistedDialog):
|
|||
'fileform': unicode(self.fileform.currentText()),
|
||||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': self.bgmeta.isChecked(),
|
||||
'updateepubcover': self.updateepubcover.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation']
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en'
|
|||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import time, os, copy, threading, re, platform, sys
|
||||
import os, copy, threading, re, platform, sys
|
||||
from StringIO import StringIO
|
||||
from functools import partial
|
||||
from datetime import datetime, time, date
|
||||
|
|
@ -923,11 +923,25 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
options['tdir']=tdir
|
||||
|
||||
if 0 < len(filter(lambda x : x['good'], books)):
|
||||
self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.')%len(books), 3000)
|
||||
if options['bgmeta']:
|
||||
status_bar=_('Start queuing downloading for %s stories.')%len(books)
|
||||
init_label=_("Queuing download for stories...")
|
||||
win_title=_("Queuing download for stories")
|
||||
status_prefix=_("Queued download for")
|
||||
else:
|
||||
status_bar=_('Started fetching metadata for %s stories.')%len(books)
|
||||
init_label=_("Fetching metadata for stories...")
|
||||
win_title=_("Downloading metadata for stories")
|
||||
status_prefix=_("Fetched metadata for")
|
||||
|
||||
self.gui.status_bar.show_message(status_bar, 3000)
|
||||
LoopProgressDialog(self.gui,
|
||||
books,
|
||||
partial(self.prep_download_loop, options = options, merge=merge),
|
||||
partial(self.start_download_job, options = options, merge=merge))
|
||||
partial(self.start_download_job, options = options, merge=merge),
|
||||
init_label=init_label,
|
||||
win_title=win_title,
|
||||
status_prefix=status_prefix)
|
||||
else:
|
||||
self.gui.status_bar.show_message(_('No valid story URLs entered.'), 3000)
|
||||
# LoopProgressDialog calls prep_download_loop for each 'good' story,
|
||||
|
|
@ -976,6 +990,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
options={'fileform':'epub',
|
||||
'collision':ADDNEW,
|
||||
'updatemeta':True,
|
||||
'bgmeta':False,
|
||||
'updateepubcover':True},
|
||||
merge=False):
|
||||
'''
|
||||
|
|
@ -1003,6 +1018,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
fileform = options['fileform']
|
||||
collision = options['collision']
|
||||
updatemeta= options['updatemeta']
|
||||
bgmeta= options['bgmeta']
|
||||
updateepubcover= options['updateepubcover']
|
||||
|
||||
# Dialogs should prevent this case now.
|
||||
|
|
@ -1048,9 +1064,11 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
if savedmetadata:
|
||||
# sets flag inside story so getStoryMetadataOnly won't hit server.
|
||||
adapter.setStoryMetadata(savedmetadata)
|
||||
|
||||
|
||||
# let other exceptions percolate up.
|
||||
# bgmeta doesn't work with CALIBREONLY.
|
||||
story = adapter.getStoryMetadataOnly(get_cover=False)
|
||||
bgmeta = False
|
||||
else:
|
||||
# reduce foreground sleep time for ffnet when few books.
|
||||
if 'ffnetcount' in options and \
|
||||
|
|
@ -1064,101 +1082,105 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
slp = min(maxslp,m*float(options['ffnetcount'])+b)
|
||||
#print("m:%s b:%s = %s"%(m,b,slp))
|
||||
adapter.set_sleep(slp)
|
||||
|
||||
## three tries, that's enough if both user/pass & is_adult needed,
|
||||
## or a couple tries of one or the other
|
||||
for x in range(0,2):
|
||||
try:
|
||||
adapter.getStoryMetadataOnly(get_cover=False)
|
||||
except exceptions.FailedToLogin, f:
|
||||
logger.warn("Login Failed, Need Username/Password.")
|
||||
userpass = UserPassDialog(self.gui,url,f)
|
||||
userpass.exec_() # exec_ will make it act modal
|
||||
if userpass.status:
|
||||
adapter.username = userpass.user.text()
|
||||
adapter.password = userpass.passwd.text()
|
||||
|
||||
except exceptions.AdultCheckRequired:
|
||||
if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
|
||||
_("%s requires that you be an adult. Please confirm you are an adult in your locale:")%url,
|
||||
show_copy_button=False):
|
||||
adapter.is_adult=True
|
||||
|
||||
# let other exceptions percolate up.
|
||||
story = adapter.getStoryMetadataOnly(get_cover=False)
|
||||
book['title'] = story.getMetadata('title')
|
||||
book['author'] = [story.getMetadata('author')]
|
||||
book['url'] = story.getMetadata('storyUrl')
|
||||
|
||||
|
||||
if not bgmeta:
|
||||
## three tries, that's enough if both user/pass & is_adult needed,
|
||||
## or a couple tries of one or the other
|
||||
for x in range(0,2):
|
||||
try:
|
||||
adapter.getStoryMetadataOnly(get_cover=False)
|
||||
except exceptions.FailedToLogin, f:
|
||||
logger.warn("Login Failed, Need Username/Password.")
|
||||
userpass = UserPassDialog(self.gui,url,f)
|
||||
userpass.exec_() # exec_ will make it act modal
|
||||
if userpass.status:
|
||||
adapter.username = userpass.user.text()
|
||||
adapter.password = userpass.passwd.text()
|
||||
|
||||
except exceptions.AdultCheckRequired:
|
||||
if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
|
||||
_("%s requires that you be an adult. Please confirm you are an adult in your locale:")%url,
|
||||
show_copy_button=False):
|
||||
adapter.is_adult=True
|
||||
|
||||
# let other exceptions percolate up.
|
||||
story = adapter.getStoryMetadataOnly(get_cover=False)
|
||||
book['title'] = story.getMetadata('title')
|
||||
book['author'] = [story.getMetadata('author')]
|
||||
book['url'] = story.getMetadata('storyUrl')
|
||||
|
||||
## Check reject list. Redundant with below for when story
|
||||
## URL changes, but also kept here to avoid network hit in
|
||||
## most common case where given url is story url.
|
||||
if self.reject_url(merge,book):
|
||||
return
|
||||
|
||||
series = story.getMetadata('series')
|
||||
if not merge and series and prefs['checkforseriesurlid']:
|
||||
# try to find *series anthology* by *seriesUrl* identifier url or uri first.
|
||||
identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
|
||||
# print("identicalbooks:%s"%identicalbooks)
|
||||
if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
|
||||
<h3>%s</h3>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
'''%(
|
||||
_('Skip Anthology Story?'),
|
||||
_('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
|
||||
_("Click '<b>Yes</b>' to Skip."),
|
||||
_("Click '<b>No</b>' to download anyway.")),
|
||||
show_copy_button=False):
|
||||
book['comment'] = _("Story in Series Anthology(%s).")%series
|
||||
book['title'] = story.getMetadata('title')
|
||||
book['author'] = [story.getMetadata('author')]
|
||||
book['url'] = story.getMetadata('storyUrl')
|
||||
book['good']=False
|
||||
book['icon']='rotate-right.png'
|
||||
book['status'] = _('Skipped')
|
||||
return
|
||||
|
||||
if not bgmeta:
|
||||
series = story.getMetadata('series')
|
||||
if not merge and series and prefs['checkforseriesurlid']:
|
||||
# try to find *series anthology* by *seriesUrl* identifier url or uri first.
|
||||
identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
|
||||
# print("identicalbooks:%s"%identicalbooks)
|
||||
if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
|
||||
<h3>%s</h3>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
'''%(
|
||||
_('Skip Anthology Story?'),
|
||||
_('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
|
||||
_("Click '<b>Yes</b>' to Skip."),
|
||||
_("Click '<b>No</b>' to download anyway.")),
|
||||
show_copy_button=False):
|
||||
book['comment'] = _("Story in Series Anthology(%s).")%series
|
||||
book['title'] = story.getMetadata('title')
|
||||
book['author'] = [story.getMetadata('author')]
|
||||
book['url'] = story.getMetadata('storyUrl')
|
||||
book['good']=False
|
||||
book['icon']='rotate-right.png'
|
||||
book['status'] = _('Skipped')
|
||||
return
|
||||
|
||||
################################################################################################################################################33
|
||||
|
||||
# set PI version instead of default.
|
||||
if 'version' in options:
|
||||
story.setMetadata('version',options['version'])
|
||||
|
||||
# all_metadata duplicates some data, but also includes extra_entries, etc.
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
if prefs['savemetacol'] != '':
|
||||
# get metadata to save in configured column.
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
book['title'] = story.getMetadata("title", removeallentities=True)
|
||||
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
|
||||
book['publisher'] = story.getMetadata("site")
|
||||
book['url'] = story.getMetadata("storyUrl")
|
||||
book['tags'] = story.getSubjectTags(removeallentities=True)
|
||||
if story.getMetadata("description"):
|
||||
book['comments'] = sanitize_comments_html(story.getMetadata("description"))
|
||||
else:
|
||||
book['comments']=''
|
||||
book['series'] = story.getMetadata("series", removeallentities=True)
|
||||
|
||||
book['is_adult'] = adapter.is_adult
|
||||
book['username'] = adapter.username
|
||||
book['password'] = adapter.password
|
||||
|
||||
book['icon'] = 'plus.png'
|
||||
book['status'] = _('Add')
|
||||
if story.getMetadataRaw('datePublished'):
|
||||
book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
|
||||
if story.getMetadataRaw('dateUpdated'):
|
||||
book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz)
|
||||
if story.getMetadataRaw('dateCreated'):
|
||||
book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
|
||||
else:
|
||||
book['timestamp'] = None # need *something* there for calibre.
|
||||
|
||||
if not bgmeta:
|
||||
# set PI version instead of default.
|
||||
if 'version' in options:
|
||||
story.setMetadata('version',options['version'])
|
||||
|
||||
# all_metadata duplicates some data, but also includes extra_entries, etc.
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
if prefs['savemetacol'] != '':
|
||||
# get metadata to save in configured column.
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
book['title'] = story.getMetadata("title", removeallentities=True)
|
||||
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
|
||||
book['publisher'] = story.getMetadata("site")
|
||||
book['url'] = story.getMetadata("storyUrl")
|
||||
book['tags'] = story.getSubjectTags(removeallentities=True)
|
||||
if story.getMetadata("description"):
|
||||
book['comments'] = sanitize_comments_html(story.getMetadata("description"))
|
||||
else:
|
||||
book['comments']=''
|
||||
book['series'] = story.getMetadata("series", removeallentities=True)
|
||||
|
||||
if story.getMetadataRaw('datePublished'):
|
||||
book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
|
||||
if story.getMetadataRaw('dateUpdated'):
|
||||
book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz)
|
||||
if story.getMetadataRaw('dateCreated'):
|
||||
book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
|
||||
else:
|
||||
book['timestamp'] = None # need *something* there for calibre.
|
||||
|
||||
if not merge:# skip all the collision code when d/ling for merging.
|
||||
if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
|
||||
book['icon'] = 'metadata.png'
|
||||
|
|
@ -1182,9 +1204,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# print("identicalbooks:%s"%identicalbooks)
|
||||
if len(identicalbooks) < 1 and prefs['matchtitleauth']:
|
||||
# find dups
|
||||
authlist = story.getList("author", removeallentities=True)
|
||||
mi = MetaInformation(story.getMetadata("title", removeallentities=True),
|
||||
authlist)
|
||||
mi = MetaInformation(book['title'],book['author'])
|
||||
identicalbooks = db.find_identical_books(mi)
|
||||
if len(identicalbooks) > 0:
|
||||
logger.debug("existing found by title/author(s)")
|
||||
|
|
@ -1261,7 +1281,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
## newer/chaptercount checks are the same for both:
|
||||
# Update epub, but only if more chapters.
|
||||
if collision in (UPDATE,UPDATEALWAYS): # collision == UPDATE
|
||||
if not bgmeta and collision in (UPDATE,UPDATEALWAYS): # collision == UPDATE
|
||||
# 'book' can exist without epub. If there's no existing epub,
|
||||
# let it go and it will download it.
|
||||
if db.has_format(book_id,fileform,index_is_id=True):
|
||||
|
|
@ -1279,17 +1299,17 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
if collision == OVERWRITE and \
|
||||
db.has_format(book_id,formmapping[fileform],index_is_id=True):
|
||||
# check make sure incoming is newer.
|
||||
lastupdated=story.getMetadataRaw('dateUpdated')
|
||||
fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8])
|
||||
|
||||
# updated doesn't have time (or is midnight), use dates only.
|
||||
# updated does have time, use full timestamps.
|
||||
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
|
||||
(lastupdated.time() != time.min and fileupdated > lastupdated):
|
||||
raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
|
||||
|
||||
|
||||
book['fileupdated']=fileupdated
|
||||
if not bgmeta:
|
||||
# check make sure incoming is newer.
|
||||
lastupdated=story.getMetadataRaw('dateUpdated')
|
||||
|
||||
# updated doesn't have time (or is midnight), use dates only.
|
||||
# updated does have time, use full timestamps.
|
||||
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
|
||||
(lastupdated.time() != time.min and fileupdated > lastupdated):
|
||||
raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
|
||||
|
||||
# For update, provide a tmp file copy of the existing epub so
|
||||
# it can't change underneath us. Now also overwrite for logpage preserve.
|
||||
|
|
@ -1360,7 +1380,11 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# For HTML format users, make the filename inside the zip something reasonable.
|
||||
# For crazy long titles/authors, limit it to 200chars.
|
||||
# For weird/OS-unsafe characters, use file safe only.
|
||||
tmp = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
|
||||
try:
|
||||
prefix = story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100]
|
||||
except NameError:
|
||||
prefix = "bgmeta-"
|
||||
tmp = PersistentTemporaryFile(prefix=prefix,
|
||||
suffix='.'+options['fileform'],
|
||||
dir=options['tdir'])
|
||||
logger.debug("title:"+book['title'])
|
||||
|
|
@ -1373,6 +1397,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
options={'fileform':'epub',
|
||||
'collision':ADDNEW,
|
||||
'updatemeta':True,
|
||||
'bgmeta':False,
|
||||
'updateepubcover':True},
|
||||
merge=False):
|
||||
'''
|
||||
|
|
@ -1438,7 +1463,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
func = 'arbitrary_n'
|
||||
cpus = self.gui.job_manager.server.pool_size
|
||||
args = ['calibre_plugins.fanficfare_plugin.jobs', 'do_download_worker',
|
||||
(book_list, options, cpus)]
|
||||
(book_list, options, cpus, merge)]
|
||||
desc = _('Download FanFiction Book')
|
||||
job = self.gui.job_manager.run_job(
|
||||
self.Dispatcher(partial(self.download_list_completed,options=options,merge=merge)),
|
||||
|
|
@ -1452,6 +1477,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
options={'fileform':'epub',
|
||||
'collision':ADDNEW,
|
||||
'updatemeta':True,
|
||||
'bgmeta':False,
|
||||
'updateepubcover':True}):
|
||||
custom_columns = self.gui.library_view.model().custom_columns
|
||||
if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
|
||||
|
|
@ -2155,6 +2181,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
book['comment'] = '' # note this is a comment on the d/l or update.
|
||||
book['url'] = ''
|
||||
book['site'] = ''
|
||||
book['series'] = ''
|
||||
book['added'] = False
|
||||
book['pubdate'] = None
|
||||
book['publisher'] = None
|
||||
|
|
|
|||
|
|
@ -10,12 +10,15 @@ __docformat__ = 'restructuredtext en'
|
|||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
import time, traceback
|
||||
import traceback
|
||||
from datetime import time
|
||||
from StringIO import StringIO
|
||||
|
||||
from calibre.utils.ipc.server import Server
|
||||
from calibre.utils.ipc.job import ParallelJob
|
||||
from calibre.constants import numeric_version as calibre_version
|
||||
from calibre.utils.date import local_tz
|
||||
from calibre.library.comments import sanitize_comments_html
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
#
|
||||
|
|
@ -23,8 +26,11 @@ from calibre.constants import numeric_version as calibre_version
|
|||
#
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
def do_download_worker(book_list, options,
|
||||
cpus, notification=lambda x,y:x):
|
||||
def do_download_worker(book_list,
|
||||
options,
|
||||
cpus,
|
||||
merge=False,
|
||||
notification=lambda x,y:x):
|
||||
'''
|
||||
Master job, to launch child jobs to extract ISBN for a set of books
|
||||
This is run as a worker job in the background to keep the UI more
|
||||
|
|
@ -44,7 +50,7 @@ def do_download_worker(book_list, options,
|
|||
total += 1
|
||||
args = ['calibre_plugins.fanficfare_plugin.jobs',
|
||||
'do_download_for_worker',
|
||||
(book,options)]
|
||||
(book,options,merge)]
|
||||
job = ParallelJob('arbitrary_n',
|
||||
"url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
|
||||
done=None,
|
||||
|
|
@ -90,7 +96,7 @@ def do_download_worker(book_list, options,
|
|||
# return the book list as the job result
|
||||
return book_list
|
||||
|
||||
def do_download_for_worker(book,options,notification=lambda x,y:x):
|
||||
def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
||||
'''
|
||||
Child job, to download story when run as a worker job
|
||||
'''
|
||||
|
|
@ -147,6 +153,26 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
|||
if 'version' in options:
|
||||
story.setMetadata('version',options['version'])
|
||||
|
||||
book['title'] = story.getMetadata("title", removeallentities=True)
|
||||
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
|
||||
book['publisher'] = story.getMetadata("site")
|
||||
book['url'] = story.getMetadata("storyUrl")
|
||||
book['tags'] = story.getSubjectTags(removeallentities=True)
|
||||
if story.getMetadata("description"):
|
||||
book['comments'] = sanitize_comments_html(story.getMetadata("description"))
|
||||
else:
|
||||
book['comments']=''
|
||||
book['series'] = story.getMetadata("series", removeallentities=True)
|
||||
|
||||
if story.getMetadataRaw('datePublished'):
|
||||
book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
|
||||
if story.getMetadataRaw('dateUpdated'):
|
||||
book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz)
|
||||
if story.getMetadataRaw('dateCreated'):
|
||||
book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
|
||||
else:
|
||||
book['timestamp'] = None # need *something* there for calibre.
|
||||
|
||||
writer = writers.getWriter(options['fileform'],configuration,adapter)
|
||||
|
||||
outfile = book['outfile']
|
||||
|
|
@ -165,12 +191,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
|||
|
||||
# preserve logfile even on overwrite.
|
||||
if 'epub_for_update' in book:
|
||||
|
||||
adapter.logfile = get_update_data(book['epub_for_update'])[6]
|
||||
# change the existing entries id to notid so
|
||||
# write_epub writes a whole new set to indicate overwrite.
|
||||
if adapter.logfile:
|
||||
adapter.logfile = adapter.logfile.replace("span id","span notid")
|
||||
|
||||
if options['collision'] == OVERWRITE:
|
||||
lastupdated=story.getMetadataRaw('dateUpdated')
|
||||
fileupdated=book['fileupdated']
|
||||
|
||||
# updated doesn't have time (or is midnight), use dates only.
|
||||
# updated does have time, use full timestamps.
|
||||
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
|
||||
(lastupdated.time() != time.min and fileupdated > lastupdated):
|
||||
raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
|
||||
|
||||
|
||||
logger.info("write to %s"%outfile)
|
||||
inject_cal_cols(book,story,configuration)
|
||||
|
|
@ -199,17 +235,20 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
|||
# dup handling from fff_plugin needed for anthology updates.
|
||||
if options['collision'] == UPDATE:
|
||||
if chaptercount == urlchaptercount:
|
||||
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
|
||||
return book
|
||||
|
||||
# dup handling from fff_plugin needed for anthology updates.
|
||||
if chaptercount > urlchaptercount:
|
||||
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
|
||||
|
||||
if merge:
|
||||
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
|
||||
return book
|
||||
else: # not merge,
|
||||
raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
|
||||
elif chaptercount > urlchaptercount:
|
||||
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
|
||||
elif chaptercount == 0:
|
||||
raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
|
||||
|
||||
if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
|
||||
and adapter.getConfig("do_update_hook"):
|
||||
chaptercount = adapter.hookForUpdates(chaptercount)
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ Dup from another site'''
|
|||
default_prefs['reject_always'] = False
|
||||
|
||||
default_prefs['updatemeta'] = True
|
||||
default_prefs['bgmeta'] = False
|
||||
default_prefs['updateepubcover'] = False
|
||||
default_prefs['keeptags'] = False
|
||||
default_prefs['suppressauthorsort'] = False
|
||||
|
|
|
|||
Loading…
Reference in a new issue