mirror of
https://github.com/JimmXinu/FanFicFare.git
synced 2025-12-26 10:47:14 +01:00
First calibre-plugin version with working background downloading.
This commit is contained in:
Jim Miller
parent
f634629b98
commit
cbc8cadd91
6 changed files with 309 additions and 109 deletions
|
|
@ -79,3 +79,12 @@ class InterfacePluginDemo(InterfaceActionBase):
|
|||
ac.apply_settings()
|
||||
|
||||
|
||||
|
||||
|
||||
# For testing, run from command line with this:
|
||||
# calibre-debug -e __init__.py
|
||||
if __name__ == '__main__':
|
||||
from PyQt4.Qt import QApplication
|
||||
from calibre.gui2.preferences import test_widget
|
||||
app = QApplication([])
|
||||
test_widget('Advanced', 'Plugins')
|
||||
|
|
|
|||
72
calibre-plugin/jobs.py
Normal file
72
calibre-plugin/jobs.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Jim Miller'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import os, traceback, time
|
||||
|
||||
import ConfigParser
|
||||
|
||||
from calibre.ebooks import DRMError
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from calibre.ebooks.metadata import MetaInformation
|
||||
|
||||
from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
|
||||
|
||||
def do_story_downloads(adaptertuple_list, fileform, db,
|
||||
abort=None, log=None, notifications=[]): # lambda x,y:x lambda makes small anonymous function.
|
||||
'''
|
||||
Master job, to launch child jobs to download this list of stories
|
||||
'''
|
||||
print("do_story_downloads")
|
||||
notifications.put((0.01, 'Start Downloading Stories'))
|
||||
count = 0
|
||||
total = len(adaptertuple_list)
|
||||
# Queue all the jobs
|
||||
for (url,adapter) in adaptertuple_list:
|
||||
do_story_download(adapter,fileform,db)
|
||||
count = count + 1
|
||||
notifications.put((float(count)/total, 'Downloading Stories'))
|
||||
# return the map as the job result
|
||||
# return book_pages_map, book_words_map
|
||||
return {},{}
|
||||
|
||||
def do_story_download(adapter,fileform,db):
|
||||
print("do_story_download")
|
||||
|
||||
# ffdlconfig = ConfigParser.SafeConfigParser()
|
||||
# adapter = adapters.getAdapter(ffdlconfig,url)
|
||||
|
||||
story = adapter.getStoryMetadataOnly()
|
||||
|
||||
mi = MetaInformation(story.getMetadata("title"),
|
||||
(story.getMetadata("author"),)) # author is a list.
|
||||
|
||||
writer = writers.getWriter(fileform,adapter.config,adapter)
|
||||
tmp = PersistentTemporaryFile("."+fileform)
|
||||
print("tmp: "+tmp.name)
|
||||
|
||||
writer.writeStory(tmp)
|
||||
|
||||
print("post write tmp: "+tmp.name)
|
||||
|
||||
mi.set_identifiers({'url':story.getMetadata("storyUrl")})
|
||||
mi.publisher = story.getMetadata("site")
|
||||
|
||||
mi.tags = writer.getTags()
|
||||
mi.languages = ['en']
|
||||
mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")
|
||||
mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")
|
||||
mi.comments = story.getMetadata("description")
|
||||
|
||||
(notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True)
|
||||
# Otherwise list of books doesn't update right away.
|
||||
#self.gui.library_view.model().books_added(addedcount)
|
||||
|
||||
del adapter
|
||||
del writer
|
||||
|
||||
|
|
@ -1,38 +1,26 @@
|
|||
#!/usr/bin/env python
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
from __future__ import (unicode_literals, division,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Fanficdownloader team'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
from StringIO import StringIO
|
||||
from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QGridLayout, QPushButton, QProgressDialog, QString,
|
||||
QLabel, QLineEdit, QInputDialog, QComboBox, QProgressDialog, QTimer )
|
||||
|
||||
from PyQt4.Qt import (QDialog, QVBoxLayout, QGridLayout, QPushButton,
|
||||
QLabel, QLineEdit, QInputDialog, QComboBox )
|
||||
from calibre.gui2 import error_dialog, warning_dialog, question_dialog
|
||||
|
||||
from calibre.ptempfile import PersistentTemporaryFile
|
||||
from calibre.ebooks.metadata import MetaInformation
|
||||
from calibre.gui2 import question_dialog
|
||||
|
||||
from calibre_plugins.fanfictiondownloader_plugin.config import prefs
|
||||
from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
|
||||
|
||||
import ConfigParser
|
||||
class DownloadDialog(QDialog):
|
||||
|
||||
class DemoDialog(QDialog):
|
||||
|
||||
def __init__(self, gui, icon, do_user_config):
|
||||
def __init__(self, gui, icon, do_user_config, pluginaction):
|
||||
QDialog.__init__(self, gui)
|
||||
self.gui = gui
|
||||
self.do_user_config = do_user_config
|
||||
|
||||
# The current database shown in the GUI
|
||||
# db is an instance of the class LibraryDatabase2 from database.py
|
||||
# This class has many, many methods that allow you to do a lot of
|
||||
# things.
|
||||
self.db = gui.current_db
|
||||
self.pluginaction = pluginaction
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
|
@ -46,12 +34,12 @@ class DemoDialog(QDialog):
|
|||
self.l.addWidget(self.url)
|
||||
|
||||
self.l.addWidget(QLabel('Output Format:'))
|
||||
self.format = QComboBox(self)
|
||||
self.format.addItem('epub')
|
||||
self.format.addItem('mobi')
|
||||
self.format.addItem('html')
|
||||
self.format.addItem('txt')
|
||||
self.l.addWidget(self.format)
|
||||
self.fileform = QComboBox(self)
|
||||
self.fileform.addItem('epub')
|
||||
self.fileform.addItem('mobi')
|
||||
self.fileform.addItem('html')
|
||||
self.fileform.addItem('txt')
|
||||
self.l.addWidget(self.fileform)
|
||||
|
||||
self.ffdl_button = QPushButton(
|
||||
'Download Story', self)
|
||||
|
|
@ -84,81 +72,9 @@ class DemoDialog(QDialog):
|
|||
text.decode('utf-8'))
|
||||
|
||||
def ffdl(self):
|
||||
|
||||
config = ConfigParser.SafeConfigParser()
|
||||
config.readfp(StringIO(get_resources("defaults.ini")))
|
||||
config.readfp(StringIO(prefs['personal.ini']))
|
||||
print("URL:"+unicode(self.url.text()))
|
||||
|
||||
adapter = adapters.getAdapter(config,unicode(self.url.text()))
|
||||
|
||||
try:
|
||||
adapter.getStoryMetadataOnly()
|
||||
except exceptions.FailedToLogin:
|
||||
print("Login Failed, Need Username/Password.")
|
||||
userpass = UserPassDialog(self.gui,adapter.getSiteDomain())
|
||||
userpass.exec_() # exec_ will make it act modal
|
||||
if userpass.status:
|
||||
adapter.username = userpass.user.text()
|
||||
adapter.password = userpass.passwd.text()
|
||||
else:
|
||||
del adapter
|
||||
return
|
||||
except exceptions.AdultCheckRequired:
|
||||
if question_dialog(self.gui, 'Are You Adult?', '<p>'+
|
||||
"This story requires that you be an adult. Please confirm you are an adult in your locale:",
|
||||
show_copy_button=False):
|
||||
adapter.is_adult=True
|
||||
else:
|
||||
del adapter
|
||||
return
|
||||
# except exceptions.StoryDoesNotExist
|
||||
|
||||
story = adapter.getStoryMetadataOnly()
|
||||
fileform = unicode(self.format.currentText())
|
||||
|
||||
mi = MetaInformation(story.getMetadata("title"),
|
||||
(story.getMetadata("author"),)) # author is a list.
|
||||
|
||||
add=True
|
||||
identicalbooks = self.db.find_identical_books(mi)
|
||||
if identicalbooks:
|
||||
add=False
|
||||
if question_dialog(self.gui, 'Add Duplicate?', '<p>'+
|
||||
"That story is already in your library. Create a new one?",
|
||||
show_copy_button=False):
|
||||
add=True
|
||||
|
||||
if add:
|
||||
writer = writers.getWriter(fileform,config,adapter)
|
||||
tmp = PersistentTemporaryFile("."+fileform)
|
||||
print("tmp: "+tmp.name)
|
||||
|
||||
writer.writeStory(tmp)
|
||||
|
||||
mi.set_identifiers({'url':story.getMetadata("storyUrl")})
|
||||
mi.publisher = story.getMetadata("site")
|
||||
|
||||
mi.tags = writer.getTags()
|
||||
mi.languages = ['en']
|
||||
mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")
|
||||
mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")
|
||||
mi.comments = story.getMetadata("description")
|
||||
|
||||
(notadded,addedcount)=self.db.add_books([tmp],[fileform],[mi], add_duplicates=True)
|
||||
# Otherwise list of books doesn't update right away.
|
||||
self.gui.library_view.model().books_added(addedcount)
|
||||
|
||||
self.pluginaction.start_downloads(unicode(self.url.text()),
|
||||
unicode(self.fileform.currentText()))
|
||||
self.hide()
|
||||
|
||||
# QMessageBox.about(self, 'FFDL Metadata',
|
||||
# str(adapter.getStoryMetadataOnly()).decode('utf-8'))
|
||||
del adapter
|
||||
try:
|
||||
del writer
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def config(self):
|
||||
self.do_user_config(parent=self)
|
||||
|
|
@ -203,3 +119,57 @@ class UserPassDialog(QDialog):
|
|||
def cancel(self):
|
||||
self.status=False
|
||||
self.hide()
|
||||
|
||||
class QueueProgressDialog(QProgressDialog):
|
||||
|
||||
def __init__(self, gui, title, loop_list, fileform, loop_function, enqueue_function, db):
|
||||
QProgressDialog.__init__(self, title, QString(), 0, len(loop_list), gui)
|
||||
self.setWindowTitle(title)
|
||||
self.setMinimumWidth(500)
|
||||
self.gui = gui
|
||||
self.db = db
|
||||
self.loop_list = loop_list
|
||||
self.fileform = fileform
|
||||
self.loop_function = loop_function
|
||||
self.enqueue_function = enqueue_function
|
||||
# self.book_ids, self.tdir, self.format_order, self.queue, self.db = \
|
||||
# book_ids, tdir, format_order, queue, db
|
||||
# self.pages_algorithm, self.pages_custom_column = pages_algorithm, pages_col
|
||||
# self.words_algorithm, self.words_custom_column = words_algorithm, words_col
|
||||
self.i, self.loop_bad, self.loop_good = 0, [], []
|
||||
self.setValue(0)
|
||||
self.setLabelText("Fetching metadata for %d of %d"%(0,len(self.loop_list)))
|
||||
QTimer.singleShot(0, self.do_loop)
|
||||
self.exec_()
|
||||
|
||||
def do_loop(self):
|
||||
current = self.loop_list[self.i]
|
||||
self.i += 1
|
||||
|
||||
try:
|
||||
retval = self.loop_function(current,self.fileform)
|
||||
self.loop_good.append((current,retval))
|
||||
except Exception as e:
|
||||
self.loop_bad.append(current)
|
||||
|
||||
self.setValue(self.i)
|
||||
self.setLabelText("Fetching metadata for %d of %d"%(self.i,len(self.loop_list)))
|
||||
if self.i >= len(self.loop_list):
|
||||
return self.do_queue()
|
||||
else:
|
||||
QTimer.singleShot(0, self.do_loop)
|
||||
|
||||
def do_queue(self):
|
||||
self.hide()
|
||||
if self.loop_bad != []:
|
||||
res = []
|
||||
for current in self.loop_bad:
|
||||
res.append('%s'%current)
|
||||
msg = '%s' % '\n'.join(res)
|
||||
warning_dialog(self.gui, _('Could not get metadata for some stories'),
|
||||
_('Could not get metadata for %d of %d stories.') %
|
||||
(len(self.loop_bad), len(self.loop_list)),
|
||||
msg).exec_()
|
||||
self.gui = None
|
||||
# Queue a job to process these ePub/Mobi books
|
||||
self.enqueue_function(self.loop_good,self.fileform)
|
||||
|
|
|
|||
|
|
@ -7,15 +7,23 @@ __license__ = 'GPL v3'
|
|||
__copyright__ = '2011, Fanficdownloader team'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
if False:
|
||||
# This is here to keep my python error checker from complaining about
|
||||
# the builtin functions that will be defined by the plugin loading system
|
||||
# You do not need this code in your plugins
|
||||
get_icons = get_resources = None
|
||||
from StringIO import StringIO
|
||||
import ConfigParser
|
||||
|
||||
from PyQt4.Qt import (QMessageBox)
|
||||
|
||||
# The class that all interface action plugins must inherit from
|
||||
from calibre.gui2.actions import InterfaceAction
|
||||
from calibre_plugins.fanfictiondownloader_plugin.plugin import DemoDialog
|
||||
from calibre.ebooks.metadata import MetaInformation
|
||||
from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
|
||||
from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob
|
||||
|
||||
from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
|
||||
|
||||
from calibre_plugins.fanfictiondownloader_plugin.config import prefs
|
||||
from calibre_plugins.fanfictiondownloader_plugin.plugin import (DownloadDialog, QueueProgressDialog,UserPassDialog)
|
||||
|
||||
from calibre_plugins.fanfictiondownloader_plugin.jobs import do_story_downloads
|
||||
|
||||
class InterfacePlugin(InterfaceAction):
|
||||
|
||||
|
|
@ -59,15 +67,155 @@ class InterfacePlugin(InterfaceAction):
|
|||
# method is defined on the base plugin class
|
||||
do_user_config = base_plugin_object.do_user_config
|
||||
|
||||
# The current database shown in the GUI
|
||||
# db is an instance of the class LibraryDatabase2 from database.py
|
||||
# This class has many, many methods that allow you to do a lot of
|
||||
# things.
|
||||
self.db = self.gui.current_db
|
||||
|
||||
# self.gui is the main calibre GUI. It acts as the gateway to access
|
||||
# all the elements of the calibre user interface, it should also be the
|
||||
# parent of the dialog
|
||||
d = DemoDialog(self.gui, self.qaction.icon(), do_user_config)
|
||||
d = DownloadDialog(self.gui, self.qaction.icon(), do_user_config, self)
|
||||
d.show()
|
||||
|
||||
def apply_settings(self):
|
||||
from calibre_plugins.fanfictiondownloader_plugin.config import prefs
|
||||
# In an actual non trivial plugin, you would probably need to
|
||||
# do something based on the settings in prefs
|
||||
prefs
|
||||
|
||||
|
||||
def start_downloads(self,url,fileform):
|
||||
self.ffdlconfig = ConfigParser.SafeConfigParser()
|
||||
self.ffdlconfig.readfp(StringIO(get_resources("defaults.ini")))
|
||||
self.ffdlconfig.readfp(StringIO(prefs['personal.ini']))
|
||||
|
||||
## XXX including code for Things to Come, namely, a list of
|
||||
## URLs rather than just one.
|
||||
url_list = [url,
|
||||
#"http://test1.com?sid=6700",
|
||||
#"http://test1.com?sid=6701",
|
||||
# "http://test1.com?sid=6702",
|
||||
# "http://test1.com?sid=6703",
|
||||
# "http://test1.com?sid=6704",
|
||||
# "http://test1.com?sid=6705",
|
||||
# "http://test1.com?sid=6706"
|
||||
]
|
||||
|
||||
self.fetchmeta_qpd = \
|
||||
QueueProgressDialog(self.gui,
|
||||
"Getting Metadata for Stories",
|
||||
url_list,
|
||||
fileform,
|
||||
self.get_adapter_for_story,
|
||||
self.enqueue_story_list_for_download,
|
||||
self.db)
|
||||
|
||||
def get_adapter_for_story(self,url,fileform):
|
||||
'''
|
||||
Returns adapter object for story at URL. To be called from
|
||||
QueueProgressDialog 'loop' to build up list of adapters. Also
|
||||
pops dialogs for is adult, user/pass, duplicate
|
||||
'''
|
||||
|
||||
print("URL:"+url)
|
||||
adapter = adapters.getAdapter(self.ffdlconfig,url)
|
||||
|
||||
try:
|
||||
adapter.getStoryMetadataOnly()
|
||||
except exceptions.FailedToLogin:
|
||||
print("Login Failed, Need Username/Password.")
|
||||
userpass = UserPassDialog(self.gui,url)
|
||||
userpass.exec_() # exec_ will make it act modal
|
||||
if userpass.status:
|
||||
adapter.username = userpass.user.text()
|
||||
adapter.password = userpass.passwd.text()
|
||||
# else:
|
||||
# del adapter
|
||||
# return
|
||||
except exceptions.AdultCheckRequired:
|
||||
if question_dialog(self.gui, 'Are You Adult?', '<p>'+
|
||||
"%s requires that you be an adult. Please confirm you are an adult in your locale:"%url,
|
||||
show_copy_button=False):
|
||||
adapter.is_adult=True
|
||||
# else:
|
||||
# del adapter
|
||||
# return
|
||||
|
||||
# let exceptions percolate up.
|
||||
story = adapter.getStoryMetadataOnly()
|
||||
|
||||
mi = MetaInformation(story.getMetadata("title"),
|
||||
(story.getMetadata("author"),)) # author is a list.
|
||||
|
||||
add=True
|
||||
identicalbooks = self.db.find_identical_books(mi)
|
||||
if identicalbooks:
|
||||
add=False
|
||||
if question_dialog(self.gui, 'Add Duplicate?', '<p>'+
|
||||
"That story is already in your library. Create a new one?",
|
||||
show_copy_button=False):
|
||||
add=True
|
||||
|
||||
if add:
|
||||
return adapter
|
||||
else:
|
||||
return None
|
||||
|
||||
def enqueue_story_list_for_download(self,adaptertuple_list,fileform):
|
||||
'''
|
||||
Called by QueueProgressDialog to enqueue story downloads for BG processing.
|
||||
adapter_list is a list of tuples of (url,adapter)
|
||||
'''
|
||||
print("enqueue_story_list_for_download")
|
||||
print(adaptertuple_list)
|
||||
|
||||
func = 'arbitrary_n'
|
||||
args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_story_downloads',
|
||||
(adaptertuple_list, fileform)] # adaptertuple_list
|
||||
desc = 'Download FanFiction Stories'
|
||||
|
||||
print("pre self.gui.job_manager.run_job")
|
||||
# job = self.gui.job_manager.run_job(
|
||||
# self.Dispatcher(self._get_stories_completed), func, args=args,
|
||||
# description=desc)
|
||||
|
||||
job = ThreadedJob('FanFictionDownload',
|
||||
'Downloading FanFiction Stories',
|
||||
func=do_story_downloads,
|
||||
args=(adaptertuple_list, fileform, self.db),
|
||||
kwargs={},
|
||||
callback=self._get_stories_completed)
|
||||
|
||||
self.gui.job_manager.run_threaded_job(job)
|
||||
print("post self.gui.job_manager.run_job")
|
||||
# job.tdir = tdir
|
||||
# job.pages_custom_column = pages_custom_column
|
||||
# job.words_custom_column = words_custom_column
|
||||
self.gui.status_bar.show_message('Downloading %d stories'%len(adaptertuple_list))
|
||||
|
||||
def _get_stories_completed(self, job):
|
||||
print("_get_stories_completed")
|
||||
# remove_dir(job.tdir)
|
||||
if job.failed:
|
||||
return self.gui.job_exception(job, dialog_title='Failed to download stories')
|
||||
# self.gui.status_bar.show_message('Download Stories completed', 3000)
|
||||
# not really, but leave it for now.
|
||||
book_pages_map, book_words_map = job.result
|
||||
|
||||
# if len(book_pages_map) + len(book_words_map) == 0:
|
||||
# # Must have been some sort of error in processing this book
|
||||
# msg = 'Failed to generate any statistics. <b>View Log</b> for details'
|
||||
# p = ErrorNotification(job.details, 'Count log', 'Count Pages failed', msg,
|
||||
# show_copy_button=False, parent=self.gui)
|
||||
# else:
|
||||
# payload = (book_pages_map, job.pages_custom_column, book_words_map, job.words_custom_column)
|
||||
# all_ids = set(book_pages_map.keys()) | set(book_words_map.keys())
|
||||
# msg = '<p>Count Pages plugin found <b>%d statistics(s)</b>. ' % len(all_ids) + \
|
||||
# 'Proceed with updating your library?'
|
||||
# p = ProceedNotification(self._update_database_columns,
|
||||
# payload, job.details,
|
||||
# 'Count log', 'Count complete', msg,
|
||||
# show_copy_button=False, parent=self.gui)
|
||||
# p.show()
|
||||
#info_dialog(self.gui,'FFDL Complete','It worked?')
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class TestSiteAdapter(BaseSiteAdapter):
|
|||
if self.story.getMetadata('storyId') == '666':
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
|
||||
if self.story.getMetadata('storyId') == '670':
|
||||
if self.story.getMetadata('storyId').startswith('670'):
|
||||
time.sleep(2.0)
|
||||
|
||||
if self.getConfig("username"):
|
||||
|
|
@ -131,7 +131,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
if self.story.getMetadata('storyId') == '667':
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
|
||||
|
||||
if self.story.getMetadata('storyId') == '670':
|
||||
if self.story.getMetadata('storyId').startswith('670'):
|
||||
time.sleep(2.0)
|
||||
|
||||
if "chapter=1" in url :
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ class BaseSiteAdapter(Configurable):
|
|||
return re.match(self.getSiteURLPattern(), self.url)
|
||||
|
||||
def __init__(self, config, url):
|
||||
self.config = config
|
||||
Configurable.__init__(self, config)
|
||||
self.addConfigSection(self.getSiteDomain())
|
||||
self.addConfigSection("overrides")
|
||||
|
|
|
|||
Loading…
Reference in a new issue