#!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from __future__ import (unicode_literals, division, absolute_import, print_function) __license__ = 'GPL v3' __copyright__ = '2012, Jim Miller' __docformat__ = 'restructuredtext en' import time, os, copy from ConfigParser import SafeConfigParser from StringIO import StringIO from functools import partial from datetime import datetime from PyQt4.Qt import (QApplication, QMenu, QToolButton) from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir from calibre.ebooks.metadata import MetaInformation, authors_to_string from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog from calibre.gui2.dialogs.message_box import ViewLog # The class that all interface action plugins must inherit from from calibre.gui2.actions import InterfaceAction from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin_icon_resources, get_icon, create_menu_action_unique, get_library_uuid) from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge from calibre_plugins.fanfictiondownloader_plugin.dcsource import get_dcsource from calibre_plugins.fanfictiondownloader_plugin.config import (prefs) from calibre_plugins.fanfictiondownloader_plugin.dialogs import ( AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog, MetadataProgressDialog, UserPassDialog, AboutDialog, OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, NotGoingToDownload ) # because calibre immediately transforms html into zip and don't want # to have an 'if html'. db.has_format is cool with the case mismatch, # but if I'm doing it anyway... formmapping = { 'epub':'EPUB', 'mobi':'MOBI', 'html':'ZIP', 'txt':'TXT' } PLUGIN_ICONS = ['images/icon.png'] sendlists = ["Send to Nook", "Send to Kindle", "Send to Droid", "Add to Nook", "Add to Kindle", "Add to Droid"] readlists = ["000"] class FanFictionDownLoaderPlugin(InterfaceAction): name = 'FanFictionDownLoader' # Declare the main action associated with this plugin # The keyboard shortcut can be None if you dont want to use a keyboard # shortcut. Remember that currently calibre has no central management for # keyboard shortcuts, so try to use an unusual/unused shortcut. # (text, icon_path, tooltip, keyboard shortcut) # icon_path isn't in the zip--icon loaded below. action_spec = (name, None, 'Download FanFiction stories from various web sites', None) action_type = 'global' # make button menu drop down only #popup_type = QToolButton.InstantPopup def genesis(self): # This method is called once per plugin, do initial setup here # Read the plugin icons and store for potential sharing with the config widget icon_resources = self.load_resources(PLUGIN_ICONS) set_plugin_icon_resources(self.name, icon_resources) # Show the config dialog # The config dialog can also be shown from within # Preferences->Plugins, which is why the do_user_config # method is defined on the base plugin class do_user_config = self.interface_action_base_plugin.do_user_config base = self.interface_action_base_plugin self.version = base.name+" v%d.%d.%d"%base.version # Set the icon for this interface action # The get_icons function is a builtin function defined for all your # plugin code. It loads icons from the plugin zip file. It returns # QIcon objects, if you want the actual data, use the analogous # get_resources builtin function. # Note that if you are loading more than one icon, for performance, you # should pass a list of names to get_icons. In this case, get_icons # will return a dictionary mapping names to QIcons. Names that # are not found in the zip file will result in null QIcons. icon = get_icon('images/icon.png') # The qaction is automatically created from the action_spec defined # above self.qaction.setIcon(icon) # Call function when plugin triggered. self.qaction.triggered.connect(self.plugin_button) # Assign our menu to this action self.menu = QMenu(self.gui) self.qaction.setMenu(self.menu) self.menu.aboutToShow.connect(self.about_to_show_menu) self.actions_unique_map = {} self.add_action = self.create_menu_item_ex(self.menu, '&Add New from URL(s)', image='plus.png', unique_name='Add New FanFiction Book(s) from URL(s)', shortcut_name='Add New FanFiction Book(s) from URL(s)', triggered=self.add_dialog ) self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png', unique_name='Update Existing FanFiction Book(s)', shortcut_name='Update Existing FanFiction Book(s)', triggered=self.update_existing) self.menu.addSeparator() self.add_send_action = self.create_menu_item_ex(self.menu, 'Add Selected to 000 and Send Lists', image='plusplus.png', unique_name='Add Selected to 000 and Send Lists', shortcut_name='Add Selected to 000 and Send Lists', triggered=partial(self.update_lists,add=True)) self.add_remove_action = self.create_menu_item_ex(self.menu, 'Remove Selected from 000, add to Send Lists', image='minusminus.png', unique_name='Remove Selected from 000, add to Send Lists', shortcut_name='Remove Selected from 000, add to Send Lists', triggered=partial(self.update_lists,add=False)) self.menu.addSeparator() self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png', unique_name='Get URLs from Selected Books', shortcut_name='Get URLs from Selected Books', triggered=self.get_list_urls) self.menu.addSeparator() self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False, image= 'config.png', unique_name='Configure FanFictionDownLoader', shortcut_name='Configure FanFictionDownLoader', triggered=partial(do_user_config,parent=self.gui)) self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False, image= 'images/icon.png', unique_name='About FanFictionDownLoader', shortcut_name='About FanFictionDownLoader', triggered=self.about) def about_to_show_menu(self): self.update_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 ) self.add_send_action.setEnabled( prefs['addtolists'] and len(self.gui.library_view.get_selected_ids()) > 0 ) self.add_remove_action.setEnabled( prefs['addtolists'] and len(self.gui.library_view.get_selected_ids()) > 0 ) self.get_list_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 ) def about(self): # Get the about text from a file inside the plugin zip file # The get_resources function is a builtin function defined for all your # plugin code. It loads files from the plugin zip file. It returns # the bytes from the specified file. # # Note that if you are loading more than one file, for performance, you # should pass a list of names to get_resources. In this case, # get_resources will return a dictionary mapping names to bytes. Names that # are not found in the zip file will not be in the returned dictionary. text = get_resources('about.txt') AboutDialog(self.gui,self.qaction.icon(),text).exec_() def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None, shortcut=None, triggered=None, is_checked=None, shortcut_name=None, unique_name=None): ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip, shortcut, triggered, is_checked, shortcut_name, unique_name) self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name return ac def plugin_button(self): if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['updatedefault']: self.update_existing() else: self.add_dialog() def update_lists(self,add=True): if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['addtolists']: self._update_reading_lists(self.gui.library_view.get_selected_ids(),add) self.gui.library_view.model().refresh_ids(self.gui.library_view.get_selected_ids()) def get_list_urls(self): if len(self.gui.library_view.get_selected_ids()) > 0: url_list = [] for book_id in self.gui.library_view.get_selected_ids(): url = self._get_story_url(self.gui.current_db, book_id) if url != None: url_list.append(url) if url_list: d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui) d.setWindowIcon(get_icon('bookmarks.png')) d.exec_() else: info_dialog(self.gui, _('List of URLs'), _('No URLs found in selected books.'), show=True, show_copy_button=False) def add_dialog(self): #print("add_dialog()") url_list = self.get_urls_clip() url_list_text = "\n".join(url_list) # self.gui is the main calibre GUI. It acts as the gateway to access # all the elements of the calibre user interface, it should also be the # parent of the dialog # AddNewDialog just collects URLs, format and presents buttons. d = AddNewDialog(self.gui, prefs, self.qaction.icon(), url_list_text, ) d.exec_() if d.result() != d.Accepted: return url_list = get_url_list(d.get_urlstext()) add_books = self._convert_urls_to_books(url_list) #print("add_books:%s"%add_books) #print("options:%s"%d.get_ffdl_options()) options = d.get_ffdl_options() options['version'] = self.version print(self.version) self.start_downloads( options, add_books ) def update_existing(self): #print("update_existing()") previous = self.gui.library_view.currentIndex() db = self.gui.current_db book_ids = self.gui.library_view.get_selected_ids() books = self._convert_calibre_ids_to_books(db, book_ids) #print("update books:%s"%books) d = UpdateExistingDialog(self.gui, 'Update Existing List', prefs, self.qaction.icon(), books, ) d.exec_() if d.result() != d.Accepted: return update_books = d.get_books() #print("update_books:%s"%update_books) #print("options:%s"%d.get_ffdl_options()) # only if there's some good ones. if 0 < len(filter(lambda x : x['good'], update_books)): options = d.get_ffdl_options() options['version'] = self.version print(self.version) self.start_downloads( options, update_books ) def get_urls_clip(self): url_list = [] if prefs['urlsfromclip']: for url in unicode(QApplication.instance().clipboard().text()).split(): if( self._is_good_downloader_url(url) ): url_list.append(url) return url_list def apply_settings(self): # No need to do anything with perfs here, but we could. prefs def start_downloads(self, options, books): #print("start_downloads:%s"%books) # create and pass temp dir. tdir = PersistentTemporaryDirectory(prefix='fanfictiondownloader_') options['tdir']=tdir self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000) MetadataProgressDialog(self.gui, books, options, partial(self.get_metadata_for_book, options = options), partial(self.start_download_list, options = options)) # MetadataProgressDialog calls get_metadata_for_book for each 'good' story, # get_metadata_for_book updates book for each, # MetadataProgressDialog calls start_download_list at the end which goes # into the BG, or shows list if no 'good' books. def get_metadata_for_book(self,book, options={'fileform':'epub', 'collision':ADDNEW, 'updatemeta':True}): ''' Update passed in book dict with metadata from website and necessary data. To be called from MetadataProgressDialog 'loop'. Also pops dialogs for is adult, user/pass. ''' # The current database shown in the GUI # db is an instance of the class LibraryDatabase2 from database.py # This class has many, many methods that allow you to do a lot of # things. db = self.gui.current_db fileform = options['fileform'] collision = options['collision'] updatemeta= options['updatemeta'] if not book['good']: # book has already been flagged bad for whatever reason. return url = book['url'] print("url:%s"%url) ## was self.ffdlconfig, but we need to be able to change it ## when doing epub update. ffdlconfig = SafeConfigParser() ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini"))) ffdlconfig.readfp(StringIO(prefs['personal.ini'])) adapter = adapters.getAdapter(ffdlconfig,url) options['personal.ini'] = prefs['personal.ini'] ## three tries, that's enough if both user/pass & is_adult needed, ## or a couple tries of one or the other for x in range(0,2): try: adapter.getStoryMetadataOnly() except exceptions.FailedToLogin: print("Login Failed, Need Username/Password.") userpass = UserPassDialog(self.gui,url) userpass.exec_() # exec_ will make it act modal if userpass.status: adapter.username = userpass.user.text() adapter.password = userpass.passwd.text() except exceptions.AdultCheckRequired: if question_dialog(self.gui, 'Are You Adult?', '

'+ "%s requires that you be an adult. Please confirm you are an adult in your locale:"%url, show_copy_button=False): adapter.is_adult=True # let other exceptions percolate up. story = adapter.getStoryMetadataOnly() writer = writers.getWriter(options['fileform'],adapter.config,adapter) book['title'] = story.getMetadata("title", removeallentities=True) book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True) book['publisher'] = story.getMetadata("site") book['tags'] = writer.getTags() book['pubdate'] = story.getMetadataRaw('datePublished') book['timestamp'] = story.getMetadataRaw('dateCreated') book['comments'] = story.getMetadata("description") #, removeallentities=True) comments handles entities better. # adapter.opener is the element with a threadlock. But del # adapter.opener doesn't work--subproc fails when it tries # to pull in the adapter object that hasn't been imported yet. # book['adapter'] = adapter book['is_adult'] = adapter.is_adult book['username'] = adapter.username book['password'] = adapter.password book['icon'] = 'plus.png' if collision in (CALIBREONLY): book['icon'] = 'metadata.png' # Dialogs should prevent this case now. if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub': raise NotGoingToDownload("Cannot update non-epub format.") book_id = None if book['calibre_id'] != None: # updating an existing book. Update mode applies. print("update existing id:%s"%book['calibre_id']) book_id = book['calibre_id'] # No handling needed: OVERWRITEALWAYS,CALIBREONLY # only care about collisions when not ADDNEW elif collision != ADDNEW: # 'new' book from URL. collision handling applies. print("from URL") # find dups mi = MetaInformation(story.getMetadata("title", removeallentities=True), (story.getMetadata("author", removeallentities=True),)) # author is a list. identicalbooks = db.find_identical_books(mi) ## removed for being overkill. # for ib in identicalbooks: # # only *really* identical if URL matches, too. # # XXX make an option? # if self._get_story_url(db,ib) == url: # identicalbooks.append(ib) #print("identicalbooks:%s"%identicalbooks) if collision == SKIP and identicalbooks: raise NotGoingToDownload("Skipping duplicate story.","list_remove.png") if len(identicalbooks) > 1: raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png") if collision == CALIBREONLY and not identicalbooks: raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png") if len(identicalbooks)>0: book_id = identicalbooks.pop() book['calibre_id'] = book_id book['icon'] = 'edit-redo.png' if book_id != None and collision != ADDNEW: if options['collision'] in (CALIBREONLY): book['comment'] = 'Metadata collected.' # don't need temp file created below. return ## newer/chaptercount checks are the same for both: # Update epub, but only if more chapters. if collision == UPDATE: # 'book' can exist without epub. If there's no existing epub, # let it go and it will download it. if db.has_format(book_id,fileform,index_is_id=True): toupdateio = StringIO() (epuburl,chaptercount) = doMerge(toupdateio, [StringIO(db.format(book_id,'EPUB', index_is_id=True))], titlenavpoints=False, striptitletoc=True, forceunique=False) urlchaptercount = int(story.getMetadata('numChapters')) if chaptercount == urlchaptercount: # and not onlyoverwriteifnewer: raise NotGoingToDownload("Already contains %d chapters."%chaptercount,'edit-undo.png') elif chaptercount > urlchaptercount: raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d." % (chaptercount,urlchaptercount),'dialog_error.png') if collision == OVERWRITE and \ db.has_format(book_id,formmapping[fileform],index_is_id=True): # check make sure incoming is newer. lastupdated=story.getMetadataRaw('dateUpdated').date() fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8]).date() if fileupdated > lastupdated: raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png') # For update, provide a tmp file copy of the existing epub so # it can't change underneath us. if collision in (UPDATE,UPDATEALWAYS) and \ db.has_format(book['calibre_id'],'EPUB',index_is_id=True): tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'], suffix='.epub', dir=options['tdir']) db.copy_format_to(book_id,fileform,tmp,index_is_id=True) print("existing epub tmp:"+tmp.name) book['epub_for_update'] = tmp.name if book['good']: # there shouldn't be any !'good' books at this point. # if still 'good', make a temp file to write the output to. tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'], suffix='.'+options['fileform'], dir=options['tdir']) print("title:"+book['title']) print("outfile:"+tmp.name) book['outfile'] = tmp.name return def start_download_list(self,book_list, options={'fileform':'epub', 'collision':ADDNEW, 'updatemeta':True}): ''' Called by MetadataProgressDialog to start story downloads BG processing. adapter_list is a list of tuples of (url,adapter) ''' #print("start_download_list:book_list:%s"%book_list) ## No need to BG process when CALIBREONLY! Fake it. if options['collision'] in (CALIBREONLY): class NotJob(object): def __init__(self,result): self.failed=False self.result=result notjob = NotJob(book_list) self.download_list_completed(notjob,options=options) return # ## XXX show list before starting download. # d = DisplayStoryListDialog(self.gui, # 'Download List', # prefs, # self.qaction.icon(), # book_list, # label_text='Status of stories to be downloaded' # ) # d.exec_() # if d.result() != d.Accepted: # return for book in book_list: if book['good']: break else: ## No good stories to try to download, go straight to ## list. d = DisplayStoryListDialog(self.gui, 'Nothing to Download', prefs, self.qaction.icon(), book_list, label_text='None of the URLs/stories given can be/need to be downloaded.' ) d.exec_() return func = 'arbitrary_n' cpus = self.gui.job_manager.server.pool_size args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_download_worker', (book_list, options, cpus)] desc = 'Download FanFiction Book' job = self.gui.job_manager.run_job( self.Dispatcher(partial(self.download_list_completed,options=options)), func, args=args, description=desc) self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000) def download_list_completed(self, job, options={}): if job.failed: self.gui.job_exception(job, dialog_title='Failed to Download Stories') return db = self.gui.current_db d = DisplayStoryListDialog(self.gui, 'Downloads finished, confirm to update Calibre', prefs, self.qaction.icon(), job.result, label_text='Stories will not be added or updated in Calibre without confirmation.' ) d.exec_() if d.result() == d.Accepted: ## in case the user removed any from the list. book_list = d.get_books() good_list = filter(lambda x : x['good'], book_list) total_good = len(good_list) self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000) for book in good_list: print("add/update %s %s"%(book['title'],book['url'])) mi = self._make_mi_from_book(book) if options['collision'] != CALIBREONLY: self._add_or_update_book(book,options,prefs,mi) if options['collision'] == CALIBREONLY or \ (options['updatemeta'] and book['good']) : self._update_metadata(db, book['calibre_id'], book, mi) add_list = filter(lambda x : x['good'] and x['added'], book_list) update_list = filter(lambda x : x['good'] and not x['added'], book_list) update_ids = [ x['calibre_id'] for x in update_list ] if len(add_list): ## even shows up added to searchs. Nice. self.gui.library_view.model().books_added(len(add_list)) if update_ids: self.gui.library_view.model().refresh_ids(update_ids) self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000) if len(update_list) + len(add_list) != total_good: d = DisplayStoryListDialog(self.gui, 'Updates completed, final status', prefs, self.qaction.icon(), book_list, label_text='Stories have be added or updated in Calibre, some had additional problems.' ) d.exec_() print("all done, remove temp dir.") remove_dir(options['tdir']) def _add_or_update_book(self,book,options,prefs,mi=None): db = self.gui.current_db if mi == None: mi = self._make_mi_from_book(book) book_id = book['calibre_id'] if book_id == None: book_id = db.create_book_entry(mi, add_duplicates=True) book['calibre_id'] = book_id book['added'] = True else: book['added'] = False if not db.add_format_with_hooks(book_id, options['fileform'], book['outfile'], index_is_id=True): book['comment'] = "Adding format to book failed for some reason..." book['good']=False book['icon']='dialog_error.png' if prefs['deleteotherforms']: fmts = db.formats(book['calibre_id'], index_is_id=True).split(',') for fmt in fmts: if fmt != formmapping[options['fileform']]: print("remove f:"+fmt) db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False if prefs['addtolists']: self._update_reading_lists([book_id],add=True) return book_id def _update_metadata(self, db, book_id, book, mi): if prefs['keeptags']: mi = copy.deepcopy(mi) old_tags = db.get_tags(book_id) # remove old Completed/In-Progress only if there's a new one. if 'Completed' in mi.tags or 'In-Progress' in mi.tags: old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags) # remove old Last Update tags if there are new ones. if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0: old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags) # mi.tags needs to be list, but set kills dups. mi.tags = list(set(list(old_tags)+mi.tags)) db.set_metadata(book_id,mi) def _update_reading_lists(self,book_ids,add=True): try: rl_plugin = self.gui.iactions['Reading List'] except: confirm("You don't have the plugin Reading List installed.",'fanfictiondownloader_no_readinglist_plugin_item', self) return if add: for l in readlists: if l in rl_plugin.get_list_names(): rl_plugin.add_books_to_list(l, book_ids, refresh_screen=False, display_warnings=False) else: for l in readlists: if l in rl_plugin.get_list_names(): rl_plugin.remove_books_from_list(l, book_ids, refresh_screen=False, display_warnings=False) for l in sendlists: if l in rl_plugin.get_list_names(): rl_plugin.add_books_to_list(l, book_ids, refresh_screen=False, display_warnings=False) def _find_existing_book_id(self,db,book,matchurl=True): mi = MetaInformation(book["title"],(book["author"],)) # author is a list. identicalbooks = db.find_identical_books(mi) if matchurl: # only *really* identical if URL matches, too. for ib in identicalbooks: if self._get_story_url(db,ib) == book['url']: return ib if identicalbooks: return identicalbooks.pop() return None def _make_mi_from_book(self,book): mi = MetaInformation(book['title'],(book['author'],)) # author is a list. mi.set_identifiers({'url':book['url']}) mi.publisher = book['publisher'] mi.tags = book['tags'] #mi.languages = ['en'] mi.pubdate = book['pubdate'] mi.timestamp = book['timestamp'] mi.comments = book['comments'] return mi def _convert_urls_to_books(self, urls): books = [] for url in urls: books.append(self._convert_url_to_book(url)) return books def _convert_url_to_book(self, url): book = {} book['good'] = True book['calibre_id'] = None book['title'] = 'Unknown' book['author'] = 'Unknown' book['author_sort'] = 'Unknown' book['comment'] = '' book['url'] = '' book['added'] = False self._set_book_url_and_comment(book,url) return book def _convert_calibre_ids_to_books(self, db, ids): books = [] for book_id in ids: books.append(self._convert_calibre_id_to_book(db,book_id)) return books def _convert_calibre_id_to_book(self, db, book_id): mi = db.get_metadata(book_id, index_is_id=True) book = {} book['good'] = True book['calibre_id'] = mi.id book['title'] = mi.title book['author'] = authors_to_string(mi.authors) book['author_sort'] = mi.author_sort # book['series'] = mi.series # if mi.series: # book['series_index'] = mi.series_index # else: # book['series_index'] = 0 book['comment'] = '' book['url'] = "" book['added'] = False url = self._get_story_url(db,book_id) self._set_book_url_and_comment(book,url) return book def _set_book_url_and_comment(self,book,url): if not url: book['comment'] = "No story URL found." book['good'] = False book['icon'] = 'search_delete_saved.png' else: # get normalized url or None. book['url'] = self._is_good_downloader_url(url) if book['url'] == None: book['url'] = url book['comment'] = "URL is not a valid story URL." book['good'] = False book['icon']='dialog_error.png' def _get_story_url(self, db, book_id): identifiers = db.get_identifiers(book_id,index_is_id=True) if 'url' in identifiers: # identifiers have :->| in url. #print("url from book:"+identifiers['url'].replace('|',':')) return identifiers['url'].replace('|',':') else: ## only epub has URL in it--at least where I can easily find it. if db.has_format(book_id,'EPUB',index_is_id=True): existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True) mi = get_metadata(existingepub,'EPUB') #print("mi:%s"%mi) identifiers = mi.get_identifiers() if 'url' in identifiers: #print("url from epub:"+identifiers['url'].replace('|',':')) return identifiers['url'].replace('|',':') # look for dc:source return get_dcsource(existingepub) return None def _is_good_downloader_url(self,url): # this is the accepted way to 'check for existance'? really? try: self.dummyconfig except AttributeError: self.dummyconfig = SafeConfigParser() # pulling up an adapter is pretty low over-head. If # it fails, it's a bad url. try: adapter = adapters.getAdapter(self.dummyconfig,url) url = adapter.url del adapter return url except: return None; def get_job_details(job): ''' Convert the job result into a set of parameters including a detail message summarising the success of the extraction operation. This is used by both the threaded and worker approaches to extraction ''' extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result if not hasattr(job, 'html_details'): job.html_details = job.details det_msg = [] for i, title in failed_ids: if i in no_format_ids: msg = title + ' (No formats)' else: msg = title + ' (ISBN not found)' det_msg.append(msg) if same_isbn_ids: if det_msg: det_msg.append('----------------------------------') for i, title in same_isbn_ids: msg = title + ' (Same ISBN)' det_msg.append(msg) if len(extracted_ids) > 0: if det_msg: det_msg.append('----------------------------------') for i, title, last_modified, isbn in extracted_ids: msg = '%s (Extracted %s)'%(title, isbn) det_msg.append(msg) det_msg = '\n'.join(det_msg) return extracted_ids, same_isbn_ids, failed_ids, det_msg def get_url_list(urls): def f(x): if x.strip(): return True else: return False # set removes dups. return set(filter(f,urls.strip().splitlines()))