diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py
index 5ae0bdea..e6f68158 100644
--- a/calibre-plugin/__init__.py
+++ b/calibre-plugin/__init__.py
@@ -28,6 +28,8 @@ class InterfacePluginDemo(InterfaceActionBase):
version = (1, 0, 0)
minimum_calibre_version = (0, 7, 53)
+# action_menu_clone_qaction = True
+
#: This field defines the GUI plugin class that contains all the code
#: that actually does something. Its format is module_path:class_name
#: The specified class must be defined in the specified module.
diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py
index aae3a56e..c1e2e740 100644
--- a/calibre-plugin/config.py
+++ b/calibre-plugin/config.py
@@ -7,10 +7,13 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Jim Miller'
__docformat__ = 'restructuredtext en'
-from PyQt4.Qt import QWidget, QVBoxLayout, QLabel, QLineEdit, QTextEdit
+from PyQt4.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QTextEdit,
+ QComboBox, QCheckBox)
from calibre.utils.config import JSONConfig
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (OVERWRITE, ADDNEW, SKIP)
+
# This is where all preferences for this plugin will be stored
# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
# in a global namespace, so make it as unique as possible.
@@ -20,6 +23,9 @@ prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
# Set defaults
prefs.defaults['personal.ini'] = get_resources('example.ini')
+prefs.defaults['updatemeta'] = True
+prefs.defaults['fileform'] = 'epub'
+prefs.defaults['collision'] = OVERWRITE
class ConfigWidget(QWidget):
@@ -28,6 +34,43 @@ class ConfigWidget(QWidget):
self.l = QVBoxLayout()
self.setLayout(self.l)
+ horz = QHBoxLayout()
+ label = QLabel('Default Output &Format:')
+ horz.addWidget(label)
+ self.fileform = QComboBox(self)
+ self.fileform.addItem('epub')
+ self.fileform.addItem('mobi')
+ self.fileform.addItem('html')
+ self.fileform.addItem('txt')
+ self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+ self.fileform.setToolTip('Choose output format to create. May set default from plugin configuration.')
+ label.setBuddy(self.fileform)
+ horz.addWidget(self.fileform)
+ self.l.addLayout(horz)
+
+ horz = QHBoxLayout()
+ label = QLabel('Default On &Collision?')
+ label.setToolTip("What to do if there's already an existing story with the same title and author.")
+ horz.addWidget(label)
+ self.collision = QComboBox(self)
+ self.collision.addItem(OVERWRITE)
+ self.collision.addItem(ADDNEW)
+ self.collision.addItem(SKIP)
+ self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
+ self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.')
+ label.setBuddy(self.collision)
+ horz.addWidget(self.collision)
+ self.l.addLayout(horz)
+
+ horz = QHBoxLayout()
+ horz.addStretch(1)
+
+ self.updatemeta = QCheckBox('Default Update &Metadata?',self)
+ self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+ horz.addWidget(self.updatemeta)
+ self.updatemeta.setChecked(prefs['updatemeta'])
+ self.l.addLayout(horz)
+
self.label = QLabel('personal.ini:')
self.l.addWidget(self.label)
@@ -37,6 +80,10 @@ class ConfigWidget(QWidget):
self.l.addWidget(self.ini)
def save_settings(self):
+ prefs['fileform'] = unicode(self.fileform.currentText())
+ prefs['collision'] = unicode(self.collision.currentText())
+ prefs['updatemeta'] = self.updatemeta.isChecked()
+
ini = unicode(self.ini.toPlainText())
if ini:
prefs['personal.ini'] = ini
@@ -44,4 +91,5 @@ class ConfigWidget(QWidget):
# if they've removed everything, clear it so they get the
# default next time.
del prefs['personal.ini']
+
diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py
index b68f5b79..a58ee250 100644
--- a/calibre-plugin/dialogs.py
+++ b/calibre-plugin/dialogs.py
@@ -7,18 +7,22 @@ __license__ = 'GPL v3'
__copyright__ = '2011, Jim Miller'
__docformat__ = 'restructuredtext en'
-from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QGridLayout,
- QPushButton, QProgressDialog, QString, QLabel,
- QTextEdit, QLineEdit, QInputDialog, QComboBox,
- QProgressDialog, QTimer )
+from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
+ QPushButton, QProgressDialog, QString, QLabel, QCheckBox,
+ QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard,
+ QProgressDialog, QTimer, QApplication )
from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+OVERWRITE='Overwrite'
+ADDNEW='Add New'
+SKIP='Skip'
+
class DownloadDialog(QDialog):
- def __init__(self, gui, icon, do_user_config, start_downloads):
+ def __init__(self, gui, prefs, icon, do_user_config, start_downloads):
QDialog.__init__(self, gui)
self.gui = gui
self.do_user_config = do_user_config
@@ -33,8 +37,12 @@ class DownloadDialog(QDialog):
self.l.addWidget(QLabel('Story URL(s), one per line:'))
self.url = QTextEdit(self)
+ self.url.setToolTip('URLs for stories, one per line.')
self.url.setLineWrapMode(QTextEdit.NoWrap)
-# self.url.setText('''http://test1.com?sid=6700
+ clipboard = QApplication.instance().clipboard()
+ self.url.setText(clipboard.text())
+ #'''http://test1.com?sid=6
+#''')
# http://test1.com?sid=6701
# http://test1.com?sid=6702
# http://test1.com?sid=6703
@@ -44,34 +52,64 @@ class DownloadDialog(QDialog):
# http://test1.com?sid=6707
# http://test1.com?sid=6708
# http://test1.com?sid=6709
-# ''')
self.l.addWidget(self.url)
# self.url = QLineEdit(self)
# self.url.setText('http://test1.com?sid=12345')
# self.l.addWidget(self.url)
- self.l.addWidget(QLabel('Output Format:'))
+ self.ffdl_button = QPushButton(
+ 'Download Stories', self)
+ self.ffdl_button.setToolTip('Start download(s).')
+ self.ffdl_button.clicked.connect(self.ffdl)
+ self.l.addWidget(self.ffdl_button)
+
+ horz = QHBoxLayout()
+ label = QLabel('Output &Format:')
+ horz.addWidget(label)
self.fileform = QComboBox(self)
self.fileform.addItem('epub')
self.fileform.addItem('mobi')
self.fileform.addItem('html')
self.fileform.addItem('txt')
- self.l.addWidget(self.fileform)
+ self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+ self.fileform.setToolTip('Choose output format to create. May set default from plugin configuration.')
+ label.setBuddy(self.fileform)
+ horz.addWidget(self.fileform)
+ self.l.addLayout(horz)
- self.ffdl_button = QPushButton(
- 'Download Stories', self)
- self.ffdl_button.clicked.connect(self.ffdl)
- self.l.addWidget(self.ffdl_button)
+ horz = QHBoxLayout()
+ label = QLabel('On &Collision?')
+ label.setToolTip("What to do if there's already an existing story with the same title and author.")
+ horz.addWidget(label)
+ self.collision = QComboBox(self)
+ self.collision.addItem(OVERWRITE)
+ self.collision.addItem(ADDNEW)
+ self.collision.addItem(SKIP)
+ self.collision.setCurrentIndex(self.collision.findText(prefs['collision']))
+ self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.')
+ label.setBuddy(self.collision)
+ horz.addWidget(self.collision)
+ self.l.addLayout(horz)
+ horz = QHBoxLayout()
+ horz.addStretch(1)
+
+ self.updatemeta = QCheckBox('Update &Metadata?',self)
+ self.updatemeta.setChecked(prefs['updatemeta'])
+ self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+ horz.addWidget(self.updatemeta)
+ self.l.addLayout(horz)
+
+ horz = QHBoxLayout()
+ self.about_button = QPushButton('About', self)
+ self.about_button.clicked.connect(self.about)
+ horz.addWidget(self.about_button)
self.conf_button = QPushButton(
'Configure this plugin', self)
self.conf_button.clicked.connect(self.config)
- self.l.addWidget(self.conf_button)
-
- self.about_button = QPushButton('About', self)
- self.about_button.clicked.connect(self.about)
- self.l.addWidget(self.about_button)
+ horz.addWidget(self.conf_button)
+ self.l.addLayout(horz)
self.resize(self.sizeHint())
@@ -91,7 +129,9 @@ class DownloadDialog(QDialog):
def ffdl(self):
self.start_downloads(unicode(self.url.toPlainText()),
- unicode(self.fileform.currentText()))
+ unicode(self.fileform.currentText()),
+ unicode(self.collision.currentText()),
+ self.updatemeta.isChecked())
self.hide()
def config(self):
@@ -180,7 +220,10 @@ class MetadataProgressDialog(QProgressDialog):
current = self.loop_list[self.i]
try:
retval = self.getadapter_function(current,self.fileform)
- self.loop_good.append((current,retval))
+ if retval:
+ self.loop_good.append((current,retval))
+ else:
+ self.loop_bad.append((current,'Duplicate--skipped.'))
except Exception as e:
self.loop_bad.append((current,e))
@@ -203,8 +246,8 @@ class MetadataProgressDialog(QProgressDialog):
_('Could not get metadata for %d of %d stories.') %
(len(self.loop_bad), len(self.loop_list)),
msg).exec_()
- else:
- info_dialog(self.gui, "Starting Downloads",
- "Got metadata and started download for %d stories."%len(self.loop_good),
- show_copy_button=False).exec_()
+ # else:
+ # info_dialog(self.gui, "Starting Downloads",
+ # "Got metadata and started download for %d stories."%len(self.loop_good),
+ # show_copy_button=False).exec_()
self.gui = None
diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py
index e14fa63f..00a7a2c3 100644
--- a/calibre-plugin/ffdl_plugin.py
+++ b/calibre-plugin/ffdl_plugin.py
@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
from StringIO import StringIO
import ConfigParser
+from functools import partial
# The class that all interface action plugins must inherit from
from calibre.ptempfile import PersistentTemporaryFile
@@ -19,7 +20,18 @@ from calibre.gui2.threaded_jobs import ThreadedJob
from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
from calibre_plugins.fanfictiondownloader_plugin.config import prefs
-from calibre_plugins.fanfictiondownloader_plugin.dialogs import DownloadDialog, MetadataProgressDialog, UserPassDialog
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
+ DownloadDialog, MetadataProgressDialog, UserPassDialog, OVERWRITE, ADDNEW, SKIP)
+
+# because calibre immediately transforms html into zip and don't want
+# to have an 'if html'. db.has_format is cool with the case mismatch,
+# but if I'm doing it anyway...
+formmapping = {
+ 'epub':'EPUB',
+ 'mobi':'MOBI',
+ 'html':'ZIP',
+ 'txt':'TXT'
+ }
class FanFictionDownLoaderPlugin(InterfaceAction):
@@ -34,7 +46,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
action_spec = ('FanFictionDownLoader', None,
'Download FanFiction stories from various web sites', None)
- action_type = 'current'
+ action_type = 'global'
def genesis(self):
# This method is called once per plugin, do initial setup here
@@ -72,11 +84,22 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
# things.
self.db = self.gui.current_db
+ rows = self.gui.library_view.selectionModel().selectedRows()
+ if rows:
+ book_ids = self.gui.library_view.get_selected_ids()
+ print("book_ids: %s"%book_ids)
+
+ row = self.gui.library_view.currentIndex()
+ if row.isValid():
+ print("current id:%d"%self.gui.library_view.model().id(row))
+ #self.db.get_identifiers()['url']
+
# self.gui is the main calibre GUI. It acts as the gateway to access
# all the elements of the calibre user interface, it should also be the
# parent of the dialog
# DownloadDialog just collects URLs, format and presents buttons.
d = DownloadDialog(self.gui,
+ prefs,
self.qaction.icon(),
do_user_config, # method for config button
self.start_downloads, # method to start downloads
@@ -87,31 +110,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
# No need to do anything with perfs here, but we could.
prefs
- def start_downloads(self,urls,fileform):
+ def start_downloads(self,urls,fileform,
+ collision,updatemeta):
self.ffdlconfig = ConfigParser.SafeConfigParser()
self.ffdlconfig.readfp(StringIO(get_resources("defaults.ini")))
self.ffdlconfig.readfp(StringIO(prefs['personal.ini']))
url_list = get_url_list(urls)
- '''
- http://test1.com?sid=6700
- http://test1.com?sid=6701
- http://test1.com?sid=6702
- http://test1.com?sid=6703
- http://test1.com?sid=6704
- http://test1.com?sid=6705
- http://test1.com?sid=6706
- '''
self.fetchmeta_qpd = \
MetadataProgressDialog(self.gui,
url_list,
fileform,
- self.get_adapter_for_story,
- self.download_list,
+ partial(self.get_adapter_for_story, collision=collision),
+ partial(self.download_list,collision=collision,updatemeta=updatemeta),
self.db)
- def get_adapter_for_story(self,url,fileform):
+ def get_adapter_for_story(self,url,fileform,collision=SKIP):
'''
Returns adapter object for story at URL. To be called from
MetadataProgressDialog 'loop' to build up list of adapters. Also
@@ -145,25 +160,34 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
# let exceptions percolate up.
story = adapter.getStoryMetadataOnly()
- mi = MetaInformation(story.getMetadata("title"),
- (story.getMetadata("author"),)) # author is a list.
-
add=True
- identicalbooks = self.db.find_identical_books(mi)
- if identicalbooks:
- add=False
- if question_dialog(self.gui, 'Add Duplicate?', ''+
- "%s by %s is already in your library. Create a new one?"%
- (story.getMetadata("title"),story.getMetadata("author")),
- show_copy_button=False):
- add=True
+ if collision != ADDNEW:
+ mi = MetaInformation(story.getMetadata("title"),
+ (story.getMetadata("author"),)) # author is a list.
+
+ identicalbooks = self.db.find_identical_books(mi)
+ print(identicalbooks)
+ ## more than one match will need to be handled differently.
+ if identicalbooks and collision == SKIP:
+ add=False
+ # book_id = identicalbooks.pop()
+ # print("formats:"+self.db.formats(book_id,index_is_id=True))
+ # print("has format:%s"%self.db.has_format(book_id,formmapping[fileform],index_is_id=True))
+ # if self.db.has_format(book_id,formmapping[fileform],index_is_id=True):
+ # if question_dialog(self.gui, 'Update?', '
'+
+ # "%s by %s is already in your library more than once. Add/Replace this format?"%
+ # (story.getMetadata("title"),story.getMetadata("author")),
+ # show_copy_button=False):
+ # add=True
if add:
return adapter
else:
return None
- def download_list(self,adaptertuple_list,fileform):
+ def download_list(self,adaptertuple_list,fileform,
+ collision=ADDNEW,
+ updatemeta=True):
'''
Called by MetadataProgressDialog to start story downloads BG processing.
adapter_list is a list of tuples of (url,adapter)
@@ -174,38 +198,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
'Downloading FanFiction Stories',
func=self.do_story_downloads,
args=(adaptertuple_list, fileform, self.db),
- kwargs={},
+ kwargs={'collision':collision,'updatemeta':updatemeta},
callback=self._get_stories_completed)
+
self.gui.job_manager.run_threaded_job(job)
self.gui.status_bar.show_message('Downloading %d stories'%len(adaptertuple_list))
def _get_stories_completed(self, job):
print("_get_stories_completed")
- # remove_dir(job.tdir)
- if job.failed:
- return self.gui.job_exception(job, dialog_title='Failed to download stories')
-# self.gui.status_bar.show_message('Download Stories completed', 3000)
- # not really, but leave it for now.
- book_pages_map, book_words_map = job.result
-
- # if len(book_pages_map) + len(book_words_map) == 0:
- # # Must have been some sort of error in processing this book
- # msg = 'Failed to generate any statistics. View Log for details'
- # p = ErrorNotification(job.details, 'Count log', 'Count Pages failed', msg,
- # show_copy_button=False, parent=self.gui)
- # else:
- # payload = (book_pages_map, job.pages_custom_column, book_words_map, job.words_custom_column)
- # all_ids = set(book_pages_map.keys()) | set(book_words_map.keys())
- # msg = '
Count Pages plugin found %d statistics(s). ' % len(all_ids) + \
- # 'Proceed with updating your library?'
- # p = ProceedNotification(self._update_database_columns,
- # payload, job.details,
- # 'Count log', 'Count complete', msg,
- # show_copy_button=False, parent=self.gui)
- # p.show()
- #info_dialog(self.gui,'FFDL Complete','It worked?')
def do_story_downloads(self, adaptertuple_list, fileform, db,
**kwargs): # lambda x,y:x lambda makes small anonymous function.
@@ -214,20 +216,30 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
Master job, loop to download this list of stories
'''
print("do_story_downloads")
+ abort = kwargs['abort']
notifications=kwargs['notifications']
+ log = kwargs['log']
notifications.put((0.01, 'Start Downloading Stories'))
- count = 0
+ count = 0.01
total = len(adaptertuple_list)
# Queue all the jobs
for (url,adapter) in adaptertuple_list:
- self.do_story_download(adapter,fileform,db)
+ if abort.is_set():
+ notifications.put(1.0,'Aborting...')
+ return
+ notifications.put((float(count)/total,
+ 'Downloading %s'%adapter.getStoryMetadataOnly().getMetadata("title")))
+ log.prints(log.INFO,'Downloading %s'%adapter.getStoryMetadataOnly().getMetadata("title"))
+ try:
+ self.do_story_download(adapter,fileform,db,kwargs['collision'],kwargs['updatemeta'])
+ except Exception as e:
+ log.prints(log.ERROR,'Failed Downloading %s: %s'%
+ (adapter.getStoryMetadataOnly().getMetadata("title"),e))
+
count = count + 1
- notifications.put((float(count)/total, 'Downloading Stories'))
- # return the map as the job result
- # return book_pages_map, book_words_map
- return {},{}
+ return
- def do_story_download(self,adapter,fileform,db):
+ def do_story_download(self,adapter,fileform,db,collision,updatemeta):
print("do_story_download")
story = adapter.getStoryMetadataOnly()
@@ -237,24 +249,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
writer = writers.getWriter(fileform,adapter.config,adapter)
tmp = PersistentTemporaryFile("."+fileform)
+ print("%s by %s"%(story.getMetadata("title"), story.getMetadata("author")))
print("tmp: "+tmp.name)
-
+
writer.writeStory(tmp)
- print("post write tmp: "+tmp.name)
-
mi.set_identifiers({'url':story.getMetadata("storyUrl")})
- mi.publisher = story.getMetadata("site")
-
+ mi.publisher = story.getMetadata("site")
mi.tags = writer.getTags()
mi.languages = ['en']
mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")
mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")
mi.comments = story.getMetadata("description")
- (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True)
+ identicalbooks = self.db.find_identical_books(mi)
+ print(identicalbooks)
+ addedcount=0
+ if identicalbooks and collision == OVERWRITE:
+ ## more than one match? add to first off the list.
+ book_id = identicalbooks.pop()
+ if updatemeta:
+ db.set_metadata(book_id,mi)
+ db.add_format_with_hooks(book_id, fileform, tmp, index_is_id=True)
+ else:
+ (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True)
+
+
# Otherwise list of books doesn't update right away.
- self.gui.library_view.model().books_added(addedcount)
+ if addedcount:
+ self.gui.library_view.model().books_added(addedcount)
del adapter
del writer
diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py
index 35dbec82..3964172f 100644
--- a/fanficdownloader/adapters/adapter_tthfanficorg.py
+++ b/fanficdownloader/adapters/adapter_tthfanficorg.py
@@ -174,10 +174,10 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
BtVS = True
for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
- if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover']:
+ if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
self.story.addToList('category',cat.string)
else:
- if cat.string == 'Non-BtVS/AtS Stories':
+ if 'Non-BtVS' in cat.string:
BtVS = False
if BtVS:
self.story.addToList('category','Buffy: The Vampire Slayer')