Plugin-Update custom columns from metadata, allow skip update confirm, progress

bar for updating books after download.
This commit is contained in:
Jim Miller 2012-01-26 23:12:50 -06:00
parent 9b17e0c396
commit d1680a74dc
5 changed files with 328 additions and 87 deletions

View file

@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
description = 'UI plugin to download FanFiction stories from various sites.'
supported_platforms = ['windows', 'osx', 'linux']
author = 'Jim Miller'
version = (1, 2, 3)
version = (1, 3, 0)
minimum_calibre_version = (0, 8, 30)
#: This field defines the GUI plugin class that contains all the code

View file

@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en'
import traceback, copy
from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget)
QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
from calibre.gui2 import dynamic, info_dialog
from calibre.utils.config import JSONConfig
@ -47,6 +47,7 @@ all_prefs.defaults['read_lists'] = ''
all_prefs.defaults['addtolists'] = False
all_prefs.defaults['addtoreadlists'] = False
all_prefs.defaults['addtolistsonread'] = False
all_prefs.defaults['custom_cols'] = {}
# The list of settings to copy from all_prefs or the previous library
# when config is called for the first time on a library.
@ -132,10 +133,14 @@ class ConfigWidget(QWidget):
tab_widget.addTab(self.list_tab, 'Reading Lists')
if 'Reading List' not in plugin_action.gui.iactions:
self.list_tab.setEnabled(False)
self.columns_tab = ColumnsTab(self, plugin_action)
tab_widget.addTab(self.columns_tab, 'Custom Columns')
self.other_tab = OtherTab(self, plugin_action)
tab_widget.addTab(self.other_tab, 'Other')
def save_settings(self):
# basic
@ -164,6 +169,15 @@ class ConfigWidget(QWidget):
else:
# if they've removed everything, reset to default.
prefs['personal.ini'] = get_resources('plugin-example.ini')
# Custom Columns tab
colsmap = {}
for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
val = unicode(combo.itemData(combo.currentIndex()).toString())
if val != 'none':
colsmap[col] = val
print("colsmap[%s]:%s"%(col,colsmap[col]))
prefs['custom_cols'] = colsmap
def edit_shortcuts(self):
self.save_settings()
@ -401,3 +415,108 @@ class OtherTab(QWidget):
info_dialog(self, _('Done'),
_('Confirmation dialogs have all been reset'), show=True)
permitted_values = {
'int' : ['numWords','numChapters'],
'float' : ['numWords','numChapters'],
'bool' : ['status-C','status-I'],
'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
'enumeration' : ['category',
'genre',
'characters',
'status',
'datePublished',
'dateUpdated',
'dateCreated',
'rating',
'warnings',
'numChapters',
'numWords',
'site',
'storyId',
'authorId',
'extratags',
'title',
'storyUrl',
'description',
'author',
'authorUrl',
'formatname',
'formatext',
'siteabbrev',
'version']
}
# no point copying the whole list.
permitted_values['text'] = permitted_values['enumeration']
permitted_values['comments'] = permitted_values['enumeration']
titleLabels = {
'category':'Category',
'genre':'Genre',
'status':'Status',
'status-C':'Status:Completed',
'status-I':'Status:In-Progress',
'characters':'Characters',
'datePublished':'Published',
'dateUpdated':'Updated',
'dateCreated':'Packaged',
'rating':'Rating',
'warnings':'Warnings',
'numChapters':'Chapters',
'numWords':'Words',
'site':'Site',
'storyId':'Story ID',
'authorId':'Author ID',
'extratags':'Extra Tags',
'title':'Title',
'storyUrl':'Story URL',
'description':'Summary',
'author':'Author',
'authorUrl':'Author URL',
'formatname':'File Format',
'formatext':'File Extension',
'siteabbrev':'Site Abbrev',
'version':'FFD Version'
}
class ColumnsTab(QWidget):
def __init__(self, parent_dialog, plugin_action):
self.parent_dialog = parent_dialog
self.plugin_action = plugin_action
QWidget.__init__(self)
self.l = QVBoxLayout()
self.setLayout(self.l)
self.custcol_dropdowns = {}
custom_columns = self.plugin_action.gui.library_view.model().custom_columns
for key, column in custom_columns.iteritems():
if column['datatype'] in permitted_values:
# print("\n============== %s ===========\n"%key)
# for (k,v) in column.iteritems():
# print("column['%s'] => %s"%(k,v))
horz = QHBoxLayout()
label = QLabel('%s(%s)'%(column['name'],key))
label.setToolTip("Update this %s column with..."%column['datatype'])
horz.addWidget(label)
dropdown = QComboBox(self)
dropdown.addItem('',QVariant('none'))
for md in permitted_values[column['datatype']]:
dropdown.addItem(titleLabels[md],QVariant(md))
self.custcol_dropdowns[key] = dropdown
if key in prefs['custom_cols']:
dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
if column['datatype'] == 'enumeration':
dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
else:
dropdown.setToolTip("Metadata values valid for this type of column.")
horz.addWidget(dropdown)
self.l.addLayout(horz)
self.l.insertStretch(-1)
#print("prefs['custom_cols'] %s"%prefs['custom_cols'])

View file

@ -18,6 +18,9 @@ from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayo
from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
from calibre.gui2.dialogs.confirm_delete import confirm
from calibre import confirm_config_name
from calibre.gui2 import dynamic
from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
from calibre_plugins.fanfictiondownloader_plugin.common_utils \
import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
@ -197,25 +200,27 @@ class UserPassDialog(QDialog):
self.status=False
self.hide()
class MetadataProgressDialog(QProgressDialog):
class LoopProgressDialog(QProgressDialog):
'''
ProgressDialog displayed while fetching metadata for each story.
'''
def __init__(self, gui,
book_list,
options,
metadata_function,
startdownload_function):
foreach_function,
finish_function,
init_label="Fetching metadata for stories...",
win_title="Downloading metadata for stories",
status_prefix="Fetched metadata for"):
QProgressDialog.__init__(self,
"Fetching metadata for stories...",
init_label,
QString(), 0, len(book_list), gui)
self.setWindowTitle("Downloading metadata for stories")
self.setWindowTitle(win_title)
self.setMinimumWidth(500)
self.gui = gui
self.book_list = book_list
self.options = options
self.metadata_function = metadata_function
self.startdownload_function = startdownload_function
self.foreach_function = foreach_function
self.finish_function = finish_function
self.status_prefix = status_prefix
self.i = 0
## self.do_loop does QTimer.singleShot on self.do_loop also.
@ -224,7 +229,7 @@ class MetadataProgressDialog(QProgressDialog):
self.exec_()
def updateStatus(self):
self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list)))
self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
self.setValue(self.i+1)
print(self.labelText())
@ -237,7 +242,7 @@ class MetadataProgressDialog(QProgressDialog):
try:
## collision spec passed into getadapter by partial from ffdl_plugin
## no retval only if it exists, but collision is SKIP
self.metadata_function(book)
self.foreach_function(book)
except NotGoingToDownload as d:
book['good']=False
@ -262,7 +267,7 @@ class MetadataProgressDialog(QProgressDialog):
self.hide()
self.gui = None
# Queues a job to process these books in the background.
self.startdownload_function(self.book_list)
self.finish_function(self.book_list)
class AboutDialog(QDialog):
@ -307,7 +312,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
class UpdateExistingDialog(SizePersistedDialog):
def __init__(self, gui, header, prefs, icon, books,
save_size_name='FanFictionDownLoader plugin:update list dialog'):
save_size_name='fanfictiondownloader_plugin:update list dialog'):
SizePersistedDialog.__init__(self, gui, save_size_name)
self.gui = gui
@ -415,14 +420,35 @@ class UpdateExistingDialog(SizePersistedDialog):
'updatemeta': unicode(self.updatemeta.isChecked()),
}
def display_story_list(gui, header, prefs, icon, books,
label_text='',
save_size_name='fanfictiondownloader_plugin:display list dialog',
offer_skip=False):
all_good = True
for b in books:
if not b['good']:
all_good=False
break
##
if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
return True
pass
## fake accept?
d = DisplayStoryListDialog(gui, header, prefs, icon, books,
label_text,
save_size_name,
offer_skip and all_good)
d.exec_()
return d.result() == d.Accepted
class DisplayStoryListDialog(SizePersistedDialog):
def __init__(self, gui, header, prefs, icon, books,
label_text='',
save_size_name='FanFictionDownLoader plugin:display list dialog'):
save_size_name='fanfictiondownloader_plugin:display list dialog',
offer_skip=False):
SizePersistedDialog.__init__(self, gui, save_size_name)
# UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books,
# save_size_name='FanFictionDownLoader plugin:display list dialog')
self.name = save_size_name
self.gui = gui
self.setWindowTitle(header)
@ -443,6 +469,15 @@ class DisplayStoryListDialog(SizePersistedDialog):
#self.label.setWordWrap(True)
options_layout.addWidget(self.label)
if offer_skip:
spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
options_layout.addItem(spacerItem1)
self.again = QCheckBox('Show this again?',self)
self.again.setChecked(True)
self.again.stateChanged.connect(self.toggle)
self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
options_layout.addWidget(self.again)
button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
button_box.accepted.connect(self.accept)
button_box.rejected.connect(self.reject)
@ -454,10 +489,15 @@ class DisplayStoryListDialog(SizePersistedDialog):
# Cause our dialog size to be restored from prefs or created on first usage
self.resize_dialog()
self.books_table.populate_table(books)
def get_books(self):
return self.books_table.get_books()
def toggle(self, *args):
dynamic[confirm_config_name(self.name)] = self.again.isChecked()
class StoryListTableWidget(QTableWidget):
def __init__(self, parent):

View file

@ -33,10 +33,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapter
from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
from calibre_plugins.fanfictiondownloader_plugin.dcsource import get_dcsource
from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog,
MetadataProgressDialog, UserPassDialog, AboutDialog,
AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
LoopProgressDialog, UserPassDialog, AboutDialog,
OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
NotGoingToDownload )
@ -337,14 +337,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
MetadataProgressDialog(self.gui,
books,
options,
partial(self.get_metadata_for_book, options = options),
partial(self.start_download_list, options = options))
# MetadataProgressDialog calls get_metadata_for_book for each 'good' story,
LoopProgressDialog(self.gui,
books,
partial(self.get_metadata_for_book, options = options),
partial(self.start_download_list, options = options))
# LoopProgressDialog calls get_metadata_for_book for each 'good' story,
# get_metadata_for_book updates book for each,
# MetadataProgressDialog calls start_download_list at the end which goes
# LoopProgressDialog calls start_download_list at the end which goes
# into the BG, or shows list if no 'good' books.
def get_metadata_for_book(self,book,
@ -353,7 +352,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
'updatemeta':True}):
'''
Update passed in book dict with metadata from website and
necessary data. To be called from MetadataProgressDialog
necessary data. To be called from LoopProgressDialog
'loop'. Also pops dialogs for is adult, user/pass.
'''
@ -407,6 +406,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
story = adapter.getStoryMetadataOnly()
writer = writers.getWriter(options['fileform'],adapter.config,adapter)
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['title'] = story.getMetadata("title", removeallentities=True)
book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
book['publisher'] = story.getMetadata("site")
@ -542,7 +542,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
'collision':ADDNEW,
'updatemeta':True}):
'''
Called by MetadataProgressDialog to start story downloads BG processing.
Called by LoopProgressDialog to start story downloads BG processing.
adapter_list is a list of tuples of (url,adapter)
'''
#print("start_download_list:book_list:%s"%book_list)
@ -585,75 +585,125 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000)
def _update_book(self,book,db=None,
options={'fileform':'epub',
'collision':ADDNEW,
'updatemeta':True}):
print("add/update %s %s"%(book['title'],book['url']))
mi = self._make_mi_from_book(book)
if options['collision'] != CALIBREONLY:
self._add_or_update_book(book,options,prefs,mi)
if options['collision'] == CALIBREONLY or \
(options['updatemeta'] and book['good']) :
self._update_metadata(db, book['calibre_id'], book, mi)
def _update_books_completed(self, book_list, options={}):
add_list = filter(lambda x : x['good'] and x['added'], book_list)
update_list = filter(lambda x : x['good'] and not x['added'], book_list)
update_ids = [ x['calibre_id'] for x in update_list ]
if len(add_list):
## even shows up added to searchs. Nice.
self.gui.library_view.model().books_added(len(add_list))
if update_ids:
self.gui.library_view.model().refresh_ids(update_ids)
current = self.gui.library_view.currentIndex()
self.gui.library_view.model().current_changed(current, self.previous)
self.gui.tags_view.recount()
self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
if len(update_list) + len(add_list) != len(book_list):
d = DisplayStoryListDialog(self.gui,
'Updates completed, final status',
prefs,
self.qaction.icon(),
book_list,
label_text='Stories have be added or updated in Calibre, some had additional problems.'
)
d.exec_()
print("all done, remove temp dir.")
remove_dir(options['tdir'])
def download_list_completed(self, job, options={}):
if job.failed:
self.gui.job_exception(job, dialog_title='Failed to Download Stories')
return
previous = self.gui.library_view.currentIndex()
self.previous = self.gui.library_view.currentIndex()
db = self.gui.current_db
# XXX Switch this to a calibre standard confirm that has a
# 'don't show this anymore' checkbox. (But only if all good?)
d = DisplayStoryListDialog(self.gui,
'Downloads finished, confirm to update Calibre',
prefs,
self.qaction.icon(),
job.result,
label_text='Stories will not be added or updated in Calibre without confirmation.'
)
d.exec_()
if d.result() == d.Accepted:
if display_story_list(self.gui,
'Downloads finished, confirm to update Calibre',
prefs,
self.qaction.icon(),
job.result,
label_text='Stories will not be added or updated in Calibre without confirmation.',
offer_skip=True):
## in case the user removed any from the list.
book_list = d.get_books()
book_list = job.result
good_list = filter(lambda x : x['good'], book_list)
total_good = len(good_list)
self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000)
self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
for book in good_list:
print("add/update %s %s"%(book['title'],book['url']))
mi = self._make_mi_from_book(book)
LoopProgressDialog(self.gui,
good_list,
partial(self._update_book, options=options, db=self.gui.current_db),
partial(self._update_books_completed, options=options),
init_label="Updating calibre for stories...",
win_title="Update calibre for stories",
status_prefix="Updated")
# for book in good_list:
# print("add/update %s %s"%(book['title'],book['url']))
# mi = self._make_mi_from_book(book)
if options['collision'] != CALIBREONLY:
self._add_or_update_book(book,options,prefs,mi)
# if options['collision'] != CALIBREONLY:
# self._add_or_update_book(book,options,prefs,mi)
if options['collision'] == CALIBREONLY or \
(options['updatemeta'] and book['good']) :
self._update_metadata(db, book['calibre_id'], book, mi)
add_list = filter(lambda x : x['good'] and x['added'], book_list)
update_list = filter(lambda x : x['good'] and not x['added'], book_list)
update_ids = [ x['calibre_id'] for x in update_list ]
if len(add_list):
## even shows up added to searchs. Nice.
self.gui.library_view.model().books_added(len(add_list))
# if options['collision'] == CALIBREONLY or \
# (options['updatemeta'] and book['good']) :
# self._update_metadata(db, book['calibre_id'], book, mi)
if update_ids:
self.gui.library_view.model().refresh_ids(update_ids)
current = self.gui.library_view.currentIndex()
self.gui.library_view.model().current_changed(current, previous)
self.gui.tags_view.recount()
##### split here.
self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
# add_list = filter(lambda x : x['good'] and x['added'], book_list)
# update_list = filter(lambda x : x['good'] and not x['added'], book_list)
# update_ids = [ x['calibre_id'] for x in update_list ]
# if len(add_list):
# ## even shows up added to searchs. Nice.
# self.gui.library_view.model().books_added(len(add_list))
# if update_ids:
# self.gui.library_view.model().refresh_ids(update_ids)
# current = self.gui.library_view.currentIndex()
# self.gui.library_view.model().current_changed(current, previous)
# self.gui.tags_view.recount()
if len(update_list) + len(add_list) != total_good:
d = DisplayStoryListDialog(self.gui,
'Updates completed, final status',
prefs,
self.qaction.icon(),
book_list,
label_text='Stories have be added or updated in Calibre, some had additional problems.'
)
d.exec_()
# self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
# if len(update_list) + len(add_list) != total_good:
# d = DisplayStoryListDialog(self.gui,
# 'Updates completed, final status',
# prefs,
# self.qaction.icon(),
# book_list,
# label_text='Stories have be added or updated in Calibre, some had additional problems.'
# )
# d.exec_()
print("all done, remove temp dir.")
remove_dir(options['tdir'])
# print("all done, remove temp dir.")
# remove_dir(options['tdir'])
def _add_or_update_book(self,book,options,prefs,mi=None):
db = self.gui.current_db
@ -706,6 +756,38 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
mi.languages=['eng']
db.set_metadata(book_id,mi)
# do configured column updates here.
#print("all_metadata: %s"%book['all_metadata'])
custom_columns = self.gui.library_view.model().custom_columns
#print("prefs['custom_cols'] %s"%prefs['custom_cols'])
for col, meta in prefs['custom_cols'].iteritems():
#print("setting %s to %s"%(col,meta))
if col not in custom_columns:
print("%s not an existing column, skipping."%col)
continue
coldef = custom_columns[col]
if not meta.startswith('status-') and meta not in book['all_metadata']:
print("No value for %s, skipping."%meta)
continue
if meta not in permitted_values[coldef['datatype']]:
print("%s not a valid column type for %s, skipping."%(col,meta))
continue
label = coldef['label']
if coldef['datatype'] in ('enumeration','text','comments','datetime'):
db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
elif coldef['datatype'] in ('int','float'):
num = unicode(book['all_metadata'][meta]).replace(",","")
db.set_custom(book_id, num, label=label, commit=False)
elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
if meta == 'status-C':
val = book['all_metadata']['status'] == 'Completed'
if meta == 'status-I':
val = book['all_metadata']['status'] == 'In-Progress'
db.set_custom(book_id, val, label=label, commit=False)
db.commit()
def _get_clean_reading_lists(self,lists):
if lists == None or lists.strip() == "" :
return []

View file

@ -56,15 +56,15 @@ class Story:
else:
return value
def getAllMetadata(self):
def getAllMetadata(self, removeallentities=False):
'''
All single value *and* list value metadata as strings.
'''
allmetadata = {}
for k in self.metadata.keys():
allmetadata[k] = self.getMetadata(k)
allmetadata[k] = self.getMetadata(k, removeallentities)
for l in self.listables.keys():
allmetadata[l] = self.getMetadata(l)
allmetadata[l] = self.getMetadata(l, removeallentities)
return allmetadata