Improvements to Get Story URLs from URL, AO3 login and better dialogs in PI.

calibre-plugin/dialogs.py

@@ -172,15 +172,17 @@ class FakeLineEdit():
     def text(self):
         pass
     
-class CollectURLDialog(QDialog):
+class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title):
-        QDialog.__init__(self, gui)
+    def __init__(self, gui, title, url_text): 
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
         self.gui = gui
         self.status=False
 
+        self.setMinimumWidth(300)
+        
         self.l = QGridLayout()
         self.setLayout(self.l)
 
@@ -189,6 +191,7 @@ class CollectURLDialog(QDialog):
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
+        self.url.setText(url_text)
         self.l.addWidget(self.url,1,1)
    
         self.ok_button = QPushButton('OK', self)
@@ -199,15 +202,16 @@ class CollectURLDialog(QDialog):
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,1)
 
-        self.resize(self.sizeHint())
+        # restore saved size.
+        self.resize_dialog()
 
     def ok(self):
         self.status=True
-        self.hide()
+        self.accept()
 
     def cancel(self):
         self.status=False
-        self.hide()
+        self.reject()
 
 class UserPassDialog(QDialog):
     '''

calibre-plugin/ffdl_plugin.py

@@ -238,13 +238,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
-        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page")
+
+        if prefs['urlsfromclip']:
+            try:
+                urltxt = self.get_urls_clip(storyurls=False)[0]
+            except:
+                urltxt = ""
+        
+        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt)
         d.exec_()
         if not d.status:
             return
-        print("URL:%s"%d.url.text())
+        print("get_urls_from_page URL:%s"%d.url.text())
 
-        url_list = get_urls_from_page("%s"%d.url.text())
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
+        url_list = get_urls_from_page("%s"%d.url.text(),ffdlconfig)
 
         if url_list:
             d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
@@ -361,13 +371,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['version'] = self.version
             print(self.version)
             self.start_downloads( options, update_books )
-        
-    def get_urls_clip(self):
+            
+    def get_urls_clip(self,storyurls=True):
         url_list = []
         if prefs['urlsfromclip']:
             for url in unicode(QApplication.instance().clipboard().text()).split():
-                if( self._is_good_downloader_url(url) ):
+                if not storyurls or self._is_good_downloader_url(url):
                     url_list.append(url)
+                    
         return url_list
 
     def apply_settings(self):

downloader.py

@@ -128,7 +128,7 @@ def main():
            config.set("overrides",var,val)
 
    if options.list:
-       retlist = get_urls_from_page(args[0])
+       retlist = get_urls_from_page(args[0], config)
        print "\n".join(retlist)
                
        return
@@ -193,7 +193,8 @@ def main():
                    # update now handled by pre-populating the old
                    # images and chapters in the adapter rather than
                    # merging epubs.
-                   (url,chaptercount,
+                   (url,
+                    chaptercount,
                     adapter.oldchapters,
                     adapter.oldimgs,
                     adapter.oldcover,

fanficdownloader/geturls.py

@@ -25,14 +25,38 @@ from gziphttp import GZipProcessor
 
 import adapters
 
-def get_urls_from_page(url):
-    
-    opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
-    soup = BeautifulSoup(opener.open(url).read())
-    
+def get_urls_from_page(url,config=None):
+
     normalized = set() # normalized url
     retlist = [] # orig urls.
-    config = ConfigParser.SafeConfigParser()
+    if not config:
+        config = ConfigParser.SafeConfigParser()
+
+    data = None
+    
+    # special stuff to log into archiveofourown.org, if possible.
+    # Unlike most that show the links to 'adult' stories, but protect
+    # them, AO3 doesn't even show them if not logged in.  Only works
+    # with saved user/pass--not going to prompt for list.
+    if 'archiveofourown.org' in url:
+        ao3adapter = adapters.getAdapter(config,"http://www.archiveofourown.org/works/0","EPUB")
+        if ao3adapter.getConfig("username"):
+            if ao3adapter.getConfig("is_adult"):
+                addurl = "?view_adult=true"
+            else:
+                addurl=""
+            # just to get an authenticity_token.
+            data = ao3adapter._fetchUrl(url+addurl)
+            # login the session.
+            ao3adapter.performLogin(url,data)
+            # get the list page with logged in session.
+            data = ao3adapter._fetchUrl(url)
+
+    if not data:
+        opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        data = opener.open(url).read()
+    
+    soup = BeautifulSoup(data)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):