diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 86742304..0345bb19 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -172,15 +172,17 @@ class FakeLineEdit(): def text(self): pass -class CollectURLDialog(QDialog): +class CollectURLDialog(SizePersistedDialog): ''' Collect single url for get urls. ''' - def __init__(self, gui, title): - QDialog.__init__(self, gui) + def __init__(self, gui, title, url_text): + SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls') self.gui = gui self.status=False + self.setMinimumWidth(300) + self.l = QGridLayout() self.setLayout(self.l) @@ -189,6 +191,7 @@ class CollectURLDialog(QDialog): self.l.addWidget(QLabel("URL:"),1,0) self.url = QLineEdit(self) + self.url.setText(url_text) self.l.addWidget(self.url,1,1) self.ok_button = QPushButton('OK', self) @@ -199,15 +202,16 @@ class CollectURLDialog(QDialog): self.cancel_button.clicked.connect(self.cancel) self.l.addWidget(self.cancel_button,2,1) - self.resize(self.sizeHint()) + # restore saved size. + self.resize_dialog() def ok(self): self.status=True - self.hide() + self.accept() def cancel(self): self.status=False - self.hide() + self.reject() class UserPassDialog(QDialog): ''' diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index f02aab9e..6ca25fb2 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -238,13 +238,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self._update_reading_lists(self.gui.library_view.get_selected_ids(),add) def get_urls_from_page(self): - d = CollectURLDialog(self.gui,"Get Story URLs from Web Page") + + if prefs['urlsfromclip']: + try: + urltxt = self.get_urls_clip(storyurls=False)[0] + except: + urltxt = "" + + d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt) d.exec_() if not d.status: return - print("URL:%s"%d.url.text()) + print("get_urls_from_page URL:%s"%d.url.text()) - url_list = get_urls_from_page("%s"%d.url.text()) + ffdlconfig = SafeConfigParser() + ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini"))) + ffdlconfig.readfp(StringIO(prefs['personal.ini'])) + url_list = get_urls_from_page("%s"%d.url.text(),ffdlconfig) if url_list: d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui) @@ -361,13 +371,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction): options['version'] = self.version print(self.version) self.start_downloads( options, update_books ) - - def get_urls_clip(self): + + def get_urls_clip(self,storyurls=True): url_list = [] if prefs['urlsfromclip']: for url in unicode(QApplication.instance().clipboard().text()).split(): - if( self._is_good_downloader_url(url) ): + if not storyurls or self._is_good_downloader_url(url): url_list.append(url) + return url_list def apply_settings(self): diff --git a/downloader.py b/downloader.py index 7377de01..58e607aa 100644 --- a/downloader.py +++ b/downloader.py @@ -128,7 +128,7 @@ def main(): config.set("overrides",var,val) if options.list: - retlist = get_urls_from_page(args[0]) + retlist = get_urls_from_page(args[0], config) print "\n".join(retlist) return @@ -193,7 +193,8 @@ def main(): # update now handled by pre-populating the old # images and chapters in the adapter rather than # merging epubs. - (url,chaptercount, + (url, + chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, diff --git a/fanficdownloader/geturls.py b/fanficdownloader/geturls.py index 292e4aae..5492fd9f 100644 --- a/fanficdownloader/geturls.py +++ b/fanficdownloader/geturls.py @@ -25,14 +25,38 @@ from gziphttp import GZipProcessor import adapters -def get_urls_from_page(url): - - opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor()) - soup = BeautifulSoup(opener.open(url).read()) - +def get_urls_from_page(url,config=None): + normalized = set() # normalized url retlist = [] # orig urls. - config = ConfigParser.SafeConfigParser() + if not config: + config = ConfigParser.SafeConfigParser() + + data = None + + # special stuff to log into archiveofourown.org, if possible. + # Unlike most that show the links to 'adult' stories, but protect + # them, AO3 doesn't even show them if not logged in. Only works + # with saved user/pass--not going to prompt for list. + if 'archiveofourown.org' in url: + ao3adapter = adapters.getAdapter(config,"http://www.archiveofourown.org/works/0","EPUB") + if ao3adapter.getConfig("username"): + if ao3adapter.getConfig("is_adult"): + addurl = "?view_adult=true" + else: + addurl="" + # just to get an authenticity_token. + data = ao3adapter._fetchUrl(url+addurl) + # login the session. + ao3adapter.performLogin(url,data) + # get the list page with logged in session. + data = ao3adapter._fetchUrl(url) + + if not data: + opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor()) + data = opener.open(url).read() + + soup = BeautifulSoup(data) for a in soup.findAll('a'): if a.has_key('href'):