diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index d242ee29..675d4728 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 2, 3) + version = (1, 3, 0) minimum_calibre_version = (0, 8, 30) #: This field defines the GUI plugin class that contains all the code diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index eb53d6bb..bd5a5388 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en' import traceback, copy from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, - QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget) + QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant) from calibre.gui2 import dynamic, info_dialog from calibre.utils.config import JSONConfig @@ -47,6 +47,7 @@ all_prefs.defaults['read_lists'] = '' all_prefs.defaults['addtolists'] = False all_prefs.defaults['addtoreadlists'] = False all_prefs.defaults['addtolistsonread'] = False +all_prefs.defaults['custom_cols'] = {} # The list of settings to copy from all_prefs or the previous library # when config is called for the first time on a library. @@ -132,10 +133,14 @@ class ConfigWidget(QWidget): tab_widget.addTab(self.list_tab, 'Reading Lists') if 'Reading List' not in plugin_action.gui.iactions: self.list_tab.setEnabled(False) - + + self.columns_tab = ColumnsTab(self, plugin_action) + tab_widget.addTab(self.columns_tab, 'Custom Columns') + self.other_tab = OtherTab(self, plugin_action) tab_widget.addTab(self.other_tab, 'Other') + def save_settings(self): # basic @@ -164,6 +169,15 @@ class ConfigWidget(QWidget): else: # if they've removed everything, reset to default. prefs['personal.ini'] = get_resources('plugin-example.ini') + + # Custom Columns tab + colsmap = {} + for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems(): + val = unicode(combo.itemData(combo.currentIndex()).toString()) + if val != 'none': + colsmap[col] = val + print("colsmap[%s]:%s"%(col,colsmap[col])) + prefs['custom_cols'] = colsmap def edit_shortcuts(self): self.save_settings() @@ -401,3 +415,108 @@ class OtherTab(QWidget): info_dialog(self, _('Done'), _('Confirmation dialogs have all been reset'), show=True) +permitted_values = { + 'int' : ['numWords','numChapters'], + 'float' : ['numWords','numChapters'], + 'bool' : ['status-C','status-I'], + 'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'], + 'enumeration' : ['category', + 'genre', + 'characters', + 'status', + 'datePublished', + 'dateUpdated', + 'dateCreated', + 'rating', + 'warnings', + 'numChapters', + 'numWords', + 'site', + 'storyId', + 'authorId', + 'extratags', + 'title', + 'storyUrl', + 'description', + 'author', + 'authorUrl', + 'formatname', + 'formatext', + 'siteabbrev', + 'version'] + } +# no point copying the whole list. +permitted_values['text'] = permitted_values['enumeration'] +permitted_values['comments'] = permitted_values['enumeration'] + +titleLabels = { + 'category':'Category', + 'genre':'Genre', + 'status':'Status', + 'status-C':'Status:Completed', + 'status-I':'Status:In-Progress', + 'characters':'Characters', + 'datePublished':'Published', + 'dateUpdated':'Updated', + 'dateCreated':'Packaged', + 'rating':'Rating', + 'warnings':'Warnings', + 'numChapters':'Chapters', + 'numWords':'Words', + 'site':'Site', + 'storyId':'Story ID', + 'authorId':'Author ID', + 'extratags':'Extra Tags', + 'title':'Title', + 'storyUrl':'Story URL', + 'description':'Summary', + 'author':'Author', + 'authorUrl':'Author URL', + 'formatname':'File Format', + 'formatext':'File Extension', + 'siteabbrev':'Site Abbrev', + 'version':'FFD Version' + } + +class ColumnsTab(QWidget): + + def __init__(self, parent_dialog, plugin_action): + self.parent_dialog = parent_dialog + self.plugin_action = plugin_action + QWidget.__init__(self) + + self.l = QVBoxLayout() + self.setLayout(self.l) + + self.custcol_dropdowns = {} + + custom_columns = self.plugin_action.gui.library_view.model().custom_columns + + for key, column in custom_columns.iteritems(): + + if column['datatype'] in permitted_values: + # print("\n============== %s ===========\n"%key) + # for (k,v) in column.iteritems(): + # print("column['%s'] => %s"%(k,v)) + horz = QHBoxLayout() + label = QLabel('%s(%s)'%(column['name'],key)) + label.setToolTip("Update this %s column with..."%column['datatype']) + horz.addWidget(label) + dropdown = QComboBox(self) + dropdown.addItem('',QVariant('none')) + for md in permitted_values[column['datatype']]: + dropdown.addItem(titleLabels[md],QVariant(md)) + self.custcol_dropdowns[key] = dropdown + if key in prefs['custom_cols']: + dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key]))) + if column['datatype'] == 'enumeration': + dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.") + else: + dropdown.setToolTip("Metadata values valid for this type of column.") + + horz.addWidget(dropdown) + self.l.addLayout(horz) + + self.l.insertStretch(-1) + + #print("prefs['custom_cols'] %s"%prefs['custom_cols']) diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 1a388f47..43fe702b 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -18,6 +18,9 @@ from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayo from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog from calibre.gui2.dialogs.confirm_delete import confirm +from calibre import confirm_config_name +from calibre.gui2 import dynamic + from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions from calibre_plugins.fanfictiondownloader_plugin.common_utils \ import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog, @@ -197,25 +200,27 @@ class UserPassDialog(QDialog): self.status=False self.hide() -class MetadataProgressDialog(QProgressDialog): +class LoopProgressDialog(QProgressDialog): ''' ProgressDialog displayed while fetching metadata for each story. ''' def __init__(self, gui, book_list, - options, - metadata_function, - startdownload_function): + foreach_function, + finish_function, + init_label="Fetching metadata for stories...", + win_title="Downloading metadata for stories", + status_prefix="Fetched metadata for"): QProgressDialog.__init__(self, - "Fetching metadata for stories...", + init_label, QString(), 0, len(book_list), gui) - self.setWindowTitle("Downloading metadata for stories") + self.setWindowTitle(win_title) self.setMinimumWidth(500) self.gui = gui self.book_list = book_list - self.options = options - self.metadata_function = metadata_function - self.startdownload_function = startdownload_function + self.foreach_function = foreach_function + self.finish_function = finish_function + self.status_prefix = status_prefix self.i = 0 ## self.do_loop does QTimer.singleShot on self.do_loop also. @@ -224,7 +229,7 @@ class MetadataProgressDialog(QProgressDialog): self.exec_() def updateStatus(self): - self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list))) + self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list))) self.setValue(self.i+1) print(self.labelText()) @@ -237,7 +242,7 @@ class MetadataProgressDialog(QProgressDialog): try: ## collision spec passed into getadapter by partial from ffdl_plugin ## no retval only if it exists, but collision is SKIP - self.metadata_function(book) + self.foreach_function(book) except NotGoingToDownload as d: book['good']=False @@ -262,7 +267,7 @@ class MetadataProgressDialog(QProgressDialog): self.hide() self.gui = None # Queues a job to process these books in the background. - self.startdownload_function(self.book_list) + self.finish_function(self.book_list) class AboutDialog(QDialog): @@ -307,7 +312,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem): class UpdateExistingDialog(SizePersistedDialog): def __init__(self, gui, header, prefs, icon, books, - save_size_name='FanFictionDownLoader plugin:update list dialog'): + save_size_name='fanfictiondownloader_plugin:update list dialog'): SizePersistedDialog.__init__(self, gui, save_size_name) self.gui = gui @@ -415,14 +420,35 @@ class UpdateExistingDialog(SizePersistedDialog): 'updatemeta': unicode(self.updatemeta.isChecked()), } +def display_story_list(gui, header, prefs, icon, books, + label_text='', + save_size_name='fanfictiondownloader_plugin:display list dialog', + offer_skip=False): + all_good = True + for b in books: + if not b['good']: + all_good=False + break + + ## + if all_good and not dynamic.get(confirm_config_name(save_size_name), True): + return True + pass + ## fake accept? + d = DisplayStoryListDialog(gui, header, prefs, icon, books, + label_text, + save_size_name, + offer_skip and all_good) + d.exec_() + return d.result() == d.Accepted + class DisplayStoryListDialog(SizePersistedDialog): def __init__(self, gui, header, prefs, icon, books, label_text='', - save_size_name='FanFictionDownLoader plugin:display list dialog'): + save_size_name='fanfictiondownloader_plugin:display list dialog', + offer_skip=False): SizePersistedDialog.__init__(self, gui, save_size_name) - # UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books, - # save_size_name='FanFictionDownLoader plugin:display list dialog') - + self.name = save_size_name self.gui = gui self.setWindowTitle(header) @@ -443,6 +469,15 @@ class DisplayStoryListDialog(SizePersistedDialog): #self.label.setWordWrap(True) options_layout.addWidget(self.label) + if offer_skip: + spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum) + options_layout.addItem(spacerItem1) + self.again = QCheckBox('Show this again?',self) + self.again.setChecked(True) + self.again.stateChanged.connect(self.toggle) + self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.') + options_layout.addWidget(self.again) + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) button_box.accepted.connect(self.accept) button_box.rejected.connect(self.reject) @@ -454,10 +489,15 @@ class DisplayStoryListDialog(SizePersistedDialog): # Cause our dialog size to be restored from prefs or created on first usage self.resize_dialog() self.books_table.populate_table(books) - + def get_books(self): return self.books_table.get_books() + def toggle(self, *args): + dynamic[confirm_config_name(self.name)] = self.again.isChecked() + + + class StoryListTableWidget(QTableWidget): def __init__(self, parent): diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index 3f84e6dd..f5b14249 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -33,10 +33,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapter from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge from calibre_plugins.fanfictiondownloader_plugin.dcsource import get_dcsource -from calibre_plugins.fanfictiondownloader_plugin.config import (prefs) +from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values) from calibre_plugins.fanfictiondownloader_plugin.dialogs import ( - AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog, - MetadataProgressDialog, UserPassDialog, AboutDialog, + AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog, + LoopProgressDialog, UserPassDialog, AboutDialog, OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, NotGoingToDownload ) @@ -337,14 +337,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000) - MetadataProgressDialog(self.gui, - books, - options, - partial(self.get_metadata_for_book, options = options), - partial(self.start_download_list, options = options)) - # MetadataProgressDialog calls get_metadata_for_book for each 'good' story, + LoopProgressDialog(self.gui, + books, + partial(self.get_metadata_for_book, options = options), + partial(self.start_download_list, options = options)) + # LoopProgressDialog calls get_metadata_for_book for each 'good' story, # get_metadata_for_book updates book for each, - # MetadataProgressDialog calls start_download_list at the end which goes + # LoopProgressDialog calls start_download_list at the end which goes # into the BG, or shows list if no 'good' books. def get_metadata_for_book(self,book, @@ -353,7 +352,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): 'updatemeta':True}): ''' Update passed in book dict with metadata from website and - necessary data. To be called from MetadataProgressDialog + necessary data. To be called from LoopProgressDialog 'loop'. Also pops dialogs for is adult, user/pass. ''' @@ -407,6 +406,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): story = adapter.getStoryMetadataOnly() writer = writers.getWriter(options['fileform'],adapter.config,adapter) + book['all_metadata'] = story.getAllMetadata(removeallentities=True) book['title'] = story.getMetadata("title", removeallentities=True) book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True) book['publisher'] = story.getMetadata("site") @@ -542,7 +542,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): 'collision':ADDNEW, 'updatemeta':True}): ''' - Called by MetadataProgressDialog to start story downloads BG processing. + Called by LoopProgressDialog to start story downloads BG processing. adapter_list is a list of tuples of (url,adapter) ''' #print("start_download_list:book_list:%s"%book_list) @@ -585,75 +585,125 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000) + def _update_book(self,book,db=None, + options={'fileform':'epub', + 'collision':ADDNEW, + 'updatemeta':True}): + print("add/update %s %s"%(book['title'],book['url'])) + mi = self._make_mi_from_book(book) + + if options['collision'] != CALIBREONLY: + self._add_or_update_book(book,options,prefs,mi) + + if options['collision'] == CALIBREONLY or \ + (options['updatemeta'] and book['good']) : + self._update_metadata(db, book['calibre_id'], book, mi) + + + def _update_books_completed(self, book_list, options={}): + + add_list = filter(lambda x : x['good'] and x['added'], book_list) + update_list = filter(lambda x : x['good'] and not x['added'], book_list) + update_ids = [ x['calibre_id'] for x in update_list ] + + if len(add_list): + ## even shows up added to searchs. Nice. + self.gui.library_view.model().books_added(len(add_list)) + + if update_ids: + self.gui.library_view.model().refresh_ids(update_ids) + + current = self.gui.library_view.currentIndex() + self.gui.library_view.model().current_changed(current, self.previous) + self.gui.tags_view.recount() + + self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000) + + if len(update_list) + len(add_list) != len(book_list): + d = DisplayStoryListDialog(self.gui, + 'Updates completed, final status', + prefs, + self.qaction.icon(), + book_list, + label_text='Stories have be added or updated in Calibre, some had additional problems.' + ) + d.exec_() + + print("all done, remove temp dir.") + remove_dir(options['tdir']) + def download_list_completed(self, job, options={}): if job.failed: self.gui.job_exception(job, dialog_title='Failed to Download Stories') return - previous = self.gui.library_view.currentIndex() + self.previous = self.gui.library_view.currentIndex() db = self.gui.current_db - # XXX Switch this to a calibre standard confirm that has a - # 'don't show this anymore' checkbox. (But only if all good?) - d = DisplayStoryListDialog(self.gui, - 'Downloads finished, confirm to update Calibre', - prefs, - self.qaction.icon(), - job.result, - label_text='Stories will not be added or updated in Calibre without confirmation.' - ) - d.exec_() - if d.result() == d.Accepted: + if display_story_list(self.gui, + 'Downloads finished, confirm to update Calibre', + prefs, + self.qaction.icon(), + job.result, + label_text='Stories will not be added or updated in Calibre without confirmation.', + offer_skip=True): - ## in case the user removed any from the list. - book_list = d.get_books() - + book_list = job.result good_list = filter(lambda x : x['good'], book_list) - total_good = len(good_list) - self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000) + self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good)) - for book in good_list: - print("add/update %s %s"%(book['title'],book['url'])) - mi = self._make_mi_from_book(book) + LoopProgressDialog(self.gui, + good_list, + partial(self._update_book, options=options, db=self.gui.current_db), + partial(self._update_books_completed, options=options), + init_label="Updating calibre for stories...", + win_title="Update calibre for stories", + status_prefix="Updated") + + # for book in good_list: + # print("add/update %s %s"%(book['title'],book['url'])) + # mi = self._make_mi_from_book(book) - if options['collision'] != CALIBREONLY: - self._add_or_update_book(book,options,prefs,mi) + # if options['collision'] != CALIBREONLY: + # self._add_or_update_book(book,options,prefs,mi) - if options['collision'] == CALIBREONLY or \ - (options['updatemeta'] and book['good']) : - self._update_metadata(db, book['calibre_id'], book, mi) - - add_list = filter(lambda x : x['good'] and x['added'], book_list) - update_list = filter(lambda x : x['good'] and not x['added'], book_list) - update_ids = [ x['calibre_id'] for x in update_list ] - - if len(add_list): - ## even shows up added to searchs. Nice. - self.gui.library_view.model().books_added(len(add_list)) + # if options['collision'] == CALIBREONLY or \ + # (options['updatemeta'] and book['good']) : + # self._update_metadata(db, book['calibre_id'], book, mi) - if update_ids: - self.gui.library_view.model().refresh_ids(update_ids) - - current = self.gui.library_view.currentIndex() - self.gui.library_view.model().current_changed(current, previous) - self.gui.tags_view.recount() + ##### split here. - self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000) + # add_list = filter(lambda x : x['good'] and x['added'], book_list) + # update_list = filter(lambda x : x['good'] and not x['added'], book_list) + # update_ids = [ x['calibre_id'] for x in update_list ] + + # if len(add_list): + # ## even shows up added to searchs. Nice. + # self.gui.library_view.model().books_added(len(add_list)) + + # if update_ids: + # self.gui.library_view.model().refresh_ids(update_ids) + + # current = self.gui.library_view.currentIndex() + # self.gui.library_view.model().current_changed(current, previous) + # self.gui.tags_view.recount() - if len(update_list) + len(add_list) != total_good: - d = DisplayStoryListDialog(self.gui, - 'Updates completed, final status', - prefs, - self.qaction.icon(), - book_list, - label_text='Stories have be added or updated in Calibre, some had additional problems.' - ) - d.exec_() + # self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000) + + # if len(update_list) + len(add_list) != total_good: + # d = DisplayStoryListDialog(self.gui, + # 'Updates completed, final status', + # prefs, + # self.qaction.icon(), + # book_list, + # label_text='Stories have be added or updated in Calibre, some had additional problems.' + # ) + # d.exec_() - print("all done, remove temp dir.") - remove_dir(options['tdir']) + # print("all done, remove temp dir.") + # remove_dir(options['tdir']) def _add_or_update_book(self,book,options,prefs,mi=None): db = self.gui.current_db @@ -706,6 +756,38 @@ class FanFictionDownLoaderPlugin(InterfaceAction): mi.languages=['eng'] db.set_metadata(book_id,mi) + # do configured column updates here. + #print("all_metadata: %s"%book['all_metadata']) + custom_columns = self.gui.library_view.model().custom_columns + + #print("prefs['custom_cols'] %s"%prefs['custom_cols']) + for col, meta in prefs['custom_cols'].iteritems(): + #print("setting %s to %s"%(col,meta)) + if col not in custom_columns: + print("%s not an existing column, skipping."%col) + continue + coldef = custom_columns[col] + if not meta.startswith('status-') and meta not in book['all_metadata']: + print("No value for %s, skipping."%meta) + continue + if meta not in permitted_values[coldef['datatype']]: + print("%s not a valid column type for %s, skipping."%(col,meta)) + continue + label = coldef['label'] + if coldef['datatype'] in ('enumeration','text','comments','datetime'): + db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False) + elif coldef['datatype'] in ('int','float'): + num = unicode(book['all_metadata'][meta]).replace(",","") + db.set_custom(book_id, num, label=label, commit=False) + elif coldef['datatype'] == 'bool' and meta.startswith('status-'): + if meta == 'status-C': + val = book['all_metadata']['status'] == 'Completed' + if meta == 'status-I': + val = book['all_metadata']['status'] == 'In-Progress' + db.set_custom(book_id, val, label=label, commit=False) + + db.commit() + def _get_clean_reading_lists(self,lists): if lists == None or lists.strip() == "" : return [] diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index 53f2d66a..8a4ed78e 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -56,15 +56,15 @@ class Story: else: return value - def getAllMetadata(self): + def getAllMetadata(self, removeallentities=False): ''' All single value *and* list value metadata as strings. ''' allmetadata = {} for k in self.metadata.keys(): - allmetadata[k] = self.getMetadata(k) + allmetadata[k] = self.getMetadata(k, removeallentities) for l in self.listables.keys(): - allmetadata[l] = self.getMetadata(l) + allmetadata[l] = self.getMetadata(l, removeallentities) return allmetadata