Bump Test Version 4.57.7

Calibre?

.gitignore

@@ -20,6 +20,9 @@
 # pycharm project specific settings files
 .idea
 
+# vscode project specific settings file
+.vscode
+
 cleanup.sh
 FanFictionDownLoader.zip
 *.epub

README.md

@@ -52,22 +52,15 @@ Test versions are available at:
 
 - The [test plugin] is posted at MobileRead.
 - The test version of CLI for pip install is uploaded to the testpypi repository and can be installed with:
-
-> `pip install --extra-index-url https://testpypi.python.org/pypi --upgrade FanFicFare`
-
+```
+pip install --extra-index-url https://test.pypi.org/simple/ --upgrade FanFicFare
+```
 
 ### Other Releases
 
 Other versions may be available depending on your OS.  I(JimmXinu) don't directly support these:
 
-- **Arch Linux**: The CLI can also be obtained on Arch Linux from the OS repositories:
-
-```
-pacman -S fanficfare
-```
-
-...or from git via the [AUR package](https://aur.archlinux.org/packages/fanficfare-git)
-(which will also update the calibre plugin, if calibre is installed).
+- **Arch Linux**: The latest CLI release can be obtained from the [fanficfare](https://aur.archlinux.org/packages/fanficfare) AUR package. It will install the calibre plugin, if calibre is installed.
 
 
 [this post in the old FFDL thread]: https://www.mobileread.com/forums/showthread.php?p=1982785#post1982785

calibre-plugin/.tx/config

@@ -1,8 +1,9 @@
 [main]
 host = https://www.transifex.com
 
-[calibre-plugins.fanfictiondownloader]
+[o:calibre:p:calibre-plugins:r:fanfictiondownloader]
 file_filter = translations/<lang>.po
+source_file = translations/en.po
 source_lang = en
-type = PO
+type        = PO
 

calibre-plugin/__init__.py

@@ -33,7 +33,7 @@ except NameError:
 from calibre.customize import InterfaceActionBase
 
 # pulled out from FanFicFareBase for saving in prefs.py
-__version__ = (4, 12, 0)
+__version__ = (4, 57, 7)
 
 ## Apparently the name for this class doesn't matter--it was still
 ## 'demo' for the first few versions.

calibre-plugin/action_chains.py

@@ -0,0 +1,20 @@
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2024, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+## References:
+## https://www.mobileread.com/forums/showthread.php?p=4435205&postcount=65
+## https://www.mobileread.com/forums/showthread.php?p=4102834&postcount=389
+
+from calibre_plugins.action_chains.events import ChainEvent
+
+class FanFicFareDownloadFinished(ChainEvent):
+
+    # replace with the name of your event
+    name = 'FanFicFare Download Finished'
+
+    def get_event_signal(self):
+        return self.gui.iactions['FanFicFare'].download_finished_signal

calibre-plugin/common_utils.py

@@ -2,7 +2,6 @@
 
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
-import six
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>, 2018, Jim Miller'
@@ -15,13 +14,16 @@ from PyQt5.Qt import (QApplication, Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLa
                       QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
                       QTextEdit, QListWidget, QAbstractItemView, QCursor)
 
+from calibre.constants import numeric_version as calibre_version
 from calibre.constants import iswindows, DEBUG
 from calibre.gui2 import UNDEFINED_QDATETIME, gprefs, info_dialog
 from calibre.gui2.actions import menu_action_unique_name
 from calibre.gui2.keyboard import ShortcutConfig
 from calibre.utils.config import config_dir
 from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
-from fanficfare.six import text_type as unicode
+
+import fanficfare.six as six
+from six import text_type as unicode
 
 # Global definition of our plugin name. Used for common functions that require this.
 plugin_name = None
@@ -40,8 +42,41 @@ def set_plugin_icon_resources(name, resources):
     plugin_name = name
     plugin_icon_resources = resources
 
+# print_tracebacks_for_missing_resources first appears in cal 6.2.0
+if calibre_version >= (6,2,0):
+    def get_icons_nolog(icon_name,plugin_name):
+        return get_icons(icon_name,
+                         plugin_name,
+                         print_tracebacks_for_missing_resources=False)
+else:
+    get_icons_nolog = get_icons
 
-def get_icon(icon_name):
+def get_icon_6plus(icon_name):
+    '''
+    Retrieve a QIcon for the named image from
+    1. Calibre's image cache
+    2. resources/images
+    3. the icon theme
+    4. the plugin zip
+    Only plugin zip has images/ in the image name for backward
+    compatibility.
+    '''
+    icon = None
+    if icon_name:
+        icon = QIcon.ic(icon_name)
+        ## both .ic and get_icons return an empty QIcon if not found.
+        if not icon or icon.isNull():
+            # don't need a tracestack from get_icons just because
+            # there's no icon in the theme
+            icon = get_icons_nolog(icon_name.replace('images/',''),
+                                   plugin_name)
+        if not icon or icon.isNull():
+            icon = get_icons(icon_name,plugin_name)
+    if not icon:
+        icon = QIcon()
+    return icon
+
+def get_icon_old(icon_name):
     '''
     Retrieve a QIcon for the named image from the zip file if it exists,
     or if not then from Calibre's image cache.
@@ -55,6 +90,11 @@ def get_icon(icon_name):
             return QIcon(pixmap)
     return QIcon()
 
+# get_icons changed in Cal6.
+if calibre_version >= (6,0,0):
+    get_icon = get_icon_6plus
+else:
+    get_icon = get_icon_old
 
 def get_pixmap(icon_name):
     '''

calibre-plugin/config.py

@@ -2,7 +2,6 @@
 
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
-import six
 
 __license__   = 'GPL v3'
 __copyright__ = '2021, Jim Miller'
@@ -23,7 +22,9 @@ from PyQt5.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QLabel,
 
 from calibre.gui2 import dynamic, info_dialog
 from calibre.gui2.complete2 import EditWithComplete
-from fanficfare.six import text_type as unicode
+from calibre.gui2.dialogs.confirm_delete import confirm
+import fanficfare.six as six
+from six import text_type as unicode
 
 try:
     from calibre.ebooks.covers import generate_cover as cal_generate_cover
@@ -278,7 +279,6 @@ class ConfigWidget(QWidget):
             prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
             prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
             prefs['bgmeta'] = self.basic_tab.bgmeta.isChecked()
-            prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
             prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
             prefs['mark'] = self.basic_tab.mark.isChecked()
             prefs['mark_success'] = self.basic_tab.mark_success.isChecked()
@@ -333,6 +333,7 @@ class ConfigWidget(QWidget):
             prefs['calibre_gen_cover'] = self.calibrecover_tab.calibre_gen_cover.isChecked()
             prefs['plugin_gen_cover'] = self.calibrecover_tab.plugin_gen_cover.isChecked()
             prefs['gcnewonly'] = self.calibrecover_tab.gcnewonly.isChecked()
+            prefs['covernewonly'] = self.calibrecover_tab.covernewonly.isChecked()
             gc_site_settings = {}
             for (site,combo) in six.iteritems(self.calibrecover_tab.gc_dropdowns):
                 val = unicode(combo.itemData(combo.currentIndex()))
@@ -370,9 +371,11 @@ class ConfigWidget(QWidget):
             prefs['suppresstitlesort'] = self.std_columns_tab.suppresstitlesort.isChecked()
             prefs['authorcase'] = self.std_columns_tab.authorcase.isChecked()
             prefs['titlecase'] = self.std_columns_tab.titlecase.isChecked()
+            prefs['seriescase'] = self.std_columns_tab.seriescase.isChecked()
             prefs['setanthologyseries'] = self.std_columns_tab.setanthologyseries.isChecked()
 
             prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+            prefs['set_series_url'] =self.std_columns_tab.set_series_url.isChecked()
             prefs['includecomments'] =self.std_columns_tab.includecomments.isChecked()
             prefs['anth_comments_newonly'] =self.std_columns_tab.anth_comments_newonly.isChecked()
 
@@ -414,6 +417,10 @@ class ConfigWidget(QWidget):
             prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
             prefs['update_existing_only_from_email'] = self.imap_tab.update_existing_only_from_email.isChecked()
             prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
+
+            prefs['site_split_jobs'] = self.other_tab.site_split_jobs.isChecked()
+            prefs['reconsolidate_jobs'] = self.other_tab.reconsolidate_jobs.isChecked()
+
             prefs.save_to_db()
             self.plugin_action.set_popup_mode()
 
@@ -482,11 +489,6 @@ class BasicTab(QWidget):
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
 
-        self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
-        self.updateepubcover.setToolTip(_("On each download, FanFicFare offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        horz.addWidget(self.updateepubcover)
-
         self.bgmeta = QCheckBox(_('Default Background Metadata?'),self)
         self.bgmeta.setToolTip(_("On each download, FanFicFare offers an option to Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail.<br />Only available for Update/Overwrite of existing books in case URL given isn't canonical or matches to existing book by Title/Author."))
         self.bgmeta.setChecked(prefs['bgmeta'])
@@ -759,6 +761,7 @@ class BasicTab(QWidget):
                            tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
                            rejectreasons=rejecturllist.get_reject_reasons(),
                            reasonslabel=_('Add this reason to all URLs added:'),
+                           accept_storyurls=True,
                            save_size_name='fff:Add Reject List')
         d.exec_()
         if d.result() == d.Accepted:
@@ -997,7 +1000,7 @@ class CalibreCoverTab(QWidget):
 
         self.gencov_elements=[] ## used to disable/enable when gen
                                 ## cover is off/on.  This is more
-                                ## about being a visual que than real
+                                ## about being a visual cue than real
                                 ## necessary function.
 
         topl = self.l = QVBoxLayout()
@@ -1041,9 +1044,17 @@ class CalibreCoverTab(QWidget):
         horz.addWidget(self.updatecalcover)
         self.l.addLayout(horz)
 
+        self.covernewonly = QCheckBox(_("Set Calibre Cover Only for New Books"),self)
+        self.covernewonly.setToolTip(_("Set the Calibre cover from EPUB only for new\nbooks, not updates to existing books."))
+        self.covernewonly.setChecked(prefs['covernewonly'])
+        horz = QHBoxLayout()
+        horz.addItem(QtGui.QSpacerItem(20, 1))
+        horz.addWidget(self.covernewonly)
+        self.l.addLayout(horz)
+        self.l.addSpacing(5)
+
         tooltip = _("Generate a Calibre book cover image when Calibre metadata is updated.<br />"
-                    "Defaults to 'Yes, Always' for backward compatibility and because %(gc)s(Plugin)"
-                    " will only run if configured for Default or site.")%no_trans
+                    "Note that %(gc)s(Plugin) will only run if there is a %(gc)s setting configured below for Default or the appropriate site.")%no_trans
         horz = QHBoxLayout()
         label = QLabel(_('Generate Calibre Cover:'))
         label.setToolTip(tooltip)
@@ -1051,13 +1062,7 @@ class CalibreCoverTab(QWidget):
         self.gencalcover = QComboBox(self)
         for i in gencalcover_order:
             self.gencalcover.addItem(i)
-        # back compat.  If has own value, use.
-        # if prefs['gencalcover']:
         self.gencalcover.setCurrentIndex(self.gencalcover.findText(prefs_save_options[prefs['gencalcover']]))
-        # elif prefs['gencover']: # doesn't have own val, set YES if old value set.
-        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(prefs_save_options[SAVE_YES]))
-        # else: # doesn't have own value, old value not set, NO.
-        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(prefs_save_options[SAVE_NO]))
 
         self.gencalcover.setToolTip(tooltip)
         label.setBuddy(self.gencalcover)
@@ -1065,6 +1070,26 @@ class CalibreCoverTab(QWidget):
         self.l.addLayout(horz)
         self.gencalcover.currentIndexChanged.connect(self.endisable_elements)
 
+        horz = QHBoxLayout()
+        horz.addItem(QtGui.QSpacerItem(20, 1))
+        vert = QVBoxLayout()
+        horz.addLayout(vert)
+        self.l.addLayout(horz)
+
+        self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
+                                    " updated.<br />Used for both Calibre and Plugin generated covers."))
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        vert.addWidget(self.gcnewonly)
+        self.gencov_elements.append(self.gcnewonly)
+
+        self.gc_polish_cover = QCheckBox(_("Inject/update the generated cover inside EPUB"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
+                                          " cover into the EPUB ebook file.<br />Used for both Calibre and Plugin generated covers."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        vert.addWidget(self.gc_polish_cover)
+        self.gencov_elements.append(self.gc_polish_cover)
+
         # can't be local or it's destroyed when __init__ is done and
         # connected things don't fire.
         self.gencov_rdgrp = QButtonGroup()
@@ -1073,7 +1098,9 @@ class CalibreCoverTab(QWidget):
         self.gencov_gb.setLayout(horz)
 
         self.plugin_gen_cover = QRadioButton(_('Plugin %(gc)s')%no_trans,self)
-        self.plugin_gen_cover.setToolTip(_("Use plugin to create covers.  Additional settings are below."))
+        self.plugin_gen_cover.setToolTip(_("Use the %(gc)s plugin to create covers.<br>"
+                                           "Requires that you have the the %(gc)s plugin installed.<br>"
+                                           "Additional settings are below.")%no_trans)
         self.gencov_rdgrp.addButton(self.plugin_gen_cover)
         # always, new only, when no cover from site, inject yes/no...
         self.plugin_gen_cover.setChecked(prefs['plugin_gen_cover'])
@@ -1095,20 +1122,6 @@ class CalibreCoverTab(QWidget):
         #self.l.addLayout(horz)
         self.l.addWidget(self.gencov_gb)
 
-        self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
-        self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
-                                    " updated.<br />Used for both Calibre and Plugin generated covers."))
-        self.gcnewonly.setChecked(prefs['gcnewonly'])
-        self.l.addWidget(self.gcnewonly)
-        self.gencov_elements.append(self.gcnewonly)
-
-        self.gc_polish_cover = QCheckBox(_("Inject/update the cover inside EPUB"),self)
-        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
-                                          " cover into the EPUB ebook file.<br />Used for both Calibre and Plugin generated covers."))
-        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
-        self.l.addWidget(self.gc_polish_cover)
-        self.gencov_elements.append(self.gc_polish_cover)
-
         self.gcp_gb = QGroupBox(_("%(gc)s(Plugin) Settings")%no_trans)
         topl.addWidget(self.gcp_gb)
         self.l = QVBoxLayout()
@@ -1267,6 +1280,31 @@ class OtherTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        groupbox = QGroupBox()
+        self.l.addWidget(groupbox)
+
+        groupl = QVBoxLayout()
+        groupbox.setLayout(groupl)
+
+        label = QLabel("<h3>"+
+                       _("Background Job Settings")+
+                       "</h3>"
+                       )
+        label.setWordWrap(True)
+        groupl.addWidget(label)
+
+        self.site_split_jobs = QCheckBox(_('Split downloads into separate background jobs by site'),self)
+        self.site_split_jobs.setToolTip(_("Launches a separate background Job for each site in the list of stories to download/update.  Otherwise, there will be only one background job."))
+        self.site_split_jobs.setChecked(prefs['site_split_jobs'])
+        groupl.addWidget(self.site_split_jobs)
+
+        self.reconsolidate_jobs = QCheckBox(_('Reconsolidate split downloads before updating library'),self)
+        self.reconsolidate_jobs.setToolTip(_("Hold all downloads/updates launched together until they all finish.  Otherwise, there will be a 'Proceed to update' dialog for each site."))
+        self.reconsolidate_jobs.setChecked(prefs['reconsolidate_jobs'])
+        groupl.addWidget(self.reconsolidate_jobs)
+
+        self.l.addSpacing(5)
+
         label = QLabel(_("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFicFare confirmation dialogs back again."))
         label.setWordWrap(True)
         self.l.addWidget(label)
@@ -1572,22 +1610,39 @@ class StandardColumnsTab(QWidget):
                 self.titlecase.setChecked(prefs['titlecase'])
                 row.append(self.titlecase)
             elif key == 'authors':
+                self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
+                self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
+                self.set_author_url.setChecked(prefs['set_author_url'])
+                row.append(self.set_author_url)
+
                 self.suppressauthorsort = QCheckBox(_('Force Author into Author Sort?'),self)
                 self.suppressauthorsort.setToolTip(_("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc."))
                 self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
                 row.append(self.suppressauthorsort)
+
                 self.authorcase = QCheckBox(_('Fix Author Case?'),self)
                 self.authorcase.setToolTip(_("If checked, Calibre's routine for correcting the capitalization of author names will be applied.")
                                           +"\n"+_("Calibre remembers all authors in the library; changing the author case on one book will effect all books by that author.")
                                           +"\n"+_("This effects Calibre metadata only, not FanFicFare metadata in title page."))
                 self.authorcase.setChecked(prefs['authorcase'])
                 row.append(self.authorcase)
+
             elif key == 'series':
+                self.set_series_url = QCheckBox(_('Set Calibre Series URL'),self)
+                self.set_series_url.setToolTip(_("Set Calibre Series URL to Series's URL on story site."))
+                self.set_series_url.setChecked(prefs['set_series_url'])
+                row.append(self.set_series_url)
+
                 self.setanthologyseries = QCheckBox(_("Set 'Series [0]' for New Anthologies?"),self)
                 self.setanthologyseries.setToolTip(_("If checked, the Series column will be set to 'Series Name [0]' when an Anthology for a series is first created."))
                 self.setanthologyseries.setChecked(prefs['setanthologyseries'])
                 row.append(self.setanthologyseries)
 
+                self.seriescase = QCheckBox(_('Fix Series Case?'),self)
+                self.seriescase.setToolTip(_("If checked, Calibre's routine for correcting the capitalization of title will be applied.")
+                                          +"\n"+_("This effects Calibre metadata only, not FanFicFare metadata in title page."))
+                self.seriescase.setChecked(prefs['seriescase'])
+                row.append(self.seriescase)
         grid = QGridLayout()
         for rownum, row in enumerate(rows):
             for colnum, col in enumerate(row):
@@ -1600,11 +1655,6 @@ class StandardColumnsTab(QWidget):
         self.l.addWidget(label)
         self.l.addSpacing(5)
 
-        self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
-        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
-        self.set_author_url.setChecked(prefs['set_author_url'])
-        self.l.addWidget(self.set_author_url)
-
         self.includecomments = QCheckBox(_("Include Books' Comments in Anthology Comments?"),self)
         self.includecomments.setToolTip(_('''Include all the merged books' comments in the new book's comments.
 Default is a list of included titles only.'''))

calibre-plugin/dialogs.py

@@ -18,22 +18,27 @@ from datetime import datetime
 
 from PyQt5 import QtWidgets as QtGui
 from PyQt5 import QtCore
-from PyQt5.Qt import (QApplication, QDialog, QWidget, QTableWidget, QVBoxLayout, QHBoxLayout,
-                      QGridLayout, QPushButton, QFont, QLabel, QCheckBox, QIcon,
+from PyQt5.Qt import (QApplication, QDialog, QWidget, QTableWidget, QTableWidgetItem, QVBoxLayout,
+                      QHBoxLayout, QGridLayout, QPushButton, QFont, QLabel, QCheckBox, QIcon,
                       QLineEdit, QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
                       QScrollArea, QPixmap, Qt, QAbstractItemView, QTextEdit,
-                      pyqtSignal, QGroupBox, QFrame)
+                      pyqtSignal, QGroupBox, QFrame, QTextCursor)
 try:
     # qt6 Calibre v6+
     QTextEditNoWrap = QTextEdit.LineWrapMode.NoWrap
+    MoveOperations = QTextCursor.MoveOperation
+    MoveMode = QTextCursor.MoveMode
 except:
     # qt5 Calibre v2-5
     QTextEditNoWrap = QTextEdit.NoWrap
+    MoveOperations = QTextCursor
+    MoveMode = QTextCursor
 
 from calibre.gui2 import gprefs
 show_download_options = 'fff:add new/update dialogs:show_download_options'
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
+from fanficfare.exceptions import NotGoingToDownload
 from fanficfare.six import text_type as unicode, ensure_text
 
 # pulls in translation files for _() strings
@@ -151,15 +156,6 @@ class RejectUrlEntry:
 
         return retval
 
-class NotGoingToDownload(Exception):
-    def __init__(self,error,icon='dialog_error.png',showerror=True):
-        self.error=error
-        self.icon=icon
-        self.showerror=showerror
-
-    def __str__(self):
-        return self.error
-
 class DroppableQTextEdit(QTextEdit):
     def __init__(self,parent):
         QTextEdit.__init__(self,parent)
@@ -185,12 +181,32 @@ class DroppableQTextEdit(QTextEdit):
         else:
             return QTextEdit.insertFromMimeData(self, mime_data)
 
-class AddNewDialog(SizePersistedDialog):
+class HotKeyedSizePersistedDialog(SizePersistedDialog):
+
+    def __init__(self, gui, save_size_name):
+        super(HotKeyedSizePersistedDialog,self).__init__(gui, save_size_name)
+        self.keys=dict()
+
+    def addCtrlKeyPress(self,key,func):
+        # print("addKeyPress: key(0x%x)"%key)
+        # print("control: 0x%x"%QtCore.Qt.ControlModifier)
+        self.keys[key]=func
+
+    def keyPressEvent(self, event):
+        # print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
+        if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
+            func = self.keys[event.key()]
+            return func()
+        else:
+            return super(HotKeyedSizePersistedDialog,self).keyPressEvent(event)
+
+class AddNewDialog(HotKeyedSizePersistedDialog):
 
     go_signal = pyqtSignal(object, object, object, object)
 
     def __init__(self, gui, prefs, icon):
-        SizePersistedDialog.__init__(self, gui, 'fff:add new dialog')
+        super(AddNewDialog,self).__init__(gui, 'fff:add new dialog')
+
         self.prefs = prefs
 
         self.setMinimumWidth(300)
@@ -212,19 +228,25 @@ class AddNewDialog(SizePersistedDialog):
         self.toplabel=QLabel("Toplabel")
         self.l.addWidget(self.toplabel)
 
-        ## XXX add labels for series name and desc? Desc in tooltip?
-        row = 0
+        ## scrollable area for lengthy series comments.
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
         grid = QGridLayout()
+        scrollcontent.setLayout(grid)
+        self.mergeshow.append(scrollable)
+
+        row = 0
         label = QLabel('<b>'+_('Series')+':</b>')
         grid.addWidget(label,row,0)
         self.mergedname=QLabel("mergedname")
         tt = _('This name will be used with the %s setting to set the title of the new book.')%'<i>anthology_title_pattern</i>'
         label.setToolTip(tt)
-        self.mergeshow.append(label)
         self.mergedname.setToolTip(tt)
         grid.addWidget(self.mergedname,row,1,1,-1)
-        self.l.addLayout(grid)
-        self.mergeshow.append(self.mergedname)
 
         row+=1
         label = QLabel('<b>'+_('Comments')+':</b>')
@@ -232,18 +254,15 @@ class AddNewDialog(SizePersistedDialog):
         self.mergeddesc=QLabel("mergeddesc")
         tt = _('These comments about the series will be included in the Comments of the new book.')+'<i></i>' # for html for auto-wrap
         label.setToolTip(tt)
-        self.mergeshow.append(label)
         self.mergeddesc.setToolTip(tt)
         self.mergeddesc.setWordWrap(True)
         grid.addWidget(self.mergeddesc,row,1,1,-1)
-        self.l.addLayout(grid)
-        self.mergeshow.append(self.mergeddesc)
         grid.setColumnStretch(1,1)
 
         self.url = DroppableQTextEdit(self)
         self.url.setToolTip("UrlTooltip")
         self.url.setLineWrapMode(QTextEditNoWrap)
-        self.l.addWidget(self.url)
+        self.l.addWidget(self.url,1) # 1 higher 'stretch'==higher priority
 
         self.groupbox = QGroupBox(_("Show Download Options"))
         self.groupbox.setCheckable(True)
@@ -307,12 +326,6 @@ class AddNewDialog(SizePersistedDialog):
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
-        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
-        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
-        horz.addWidget(self.updateepubcover)
-        self.mergehide.append(self.updateepubcover)
-
         self.gbl.addLayout(horz)
 
         ## bgmeta not used with Add New because of stories that change
@@ -332,6 +345,9 @@ class AddNewDialog(SizePersistedDialog):
         self.button_box.rejected.connect(self.reject)
         self.l.addWidget(self.button_box)
 
+        self.addCtrlKeyPress(QtCore.Qt.Key_Return,self.ok_clicked)
+        self.addCtrlKeyPress(QtCore.Qt.Key_Enter,self.ok_clicked) # num pad
+
     def click_show_download_options(self,x):
         self.gbf.setVisible(x)
         gprefs[show_download_options] = x
@@ -442,9 +458,6 @@ class AddNewDialog(SizePersistedDialog):
         self.updatemeta.setChecked(self.prefs['updatemeta'])
         # self.bgmeta.setChecked(self.prefs['bgmeta'])
 
-        if not self.merge:
-            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
-
         self.url.setText(url_list_text)
         if url_list_text:
             self.button_box.button(QDialogButtonBox.Ok).setFocus()
@@ -477,19 +490,18 @@ class AddNewDialog(SizePersistedDialog):
             self.collision.setCurrentIndex(i)
 
     def get_fff_options(self):
-        retval =  {
-            'fileform': unicode(self.fileform.currentText()),
-            'collision': unicode(self.collision.currentText()),
-            'updatemeta': self.updatemeta.isChecked(),
-            'bgmeta': False, # self.bgmeta.isChecked(),
-            'updateepubcover': self.updateepubcover.isChecked(),
-            'smarten_punctuation':self.prefs['smarten_punctuation'],
-            'do_wordcount':self.prefs['do_wordcount'],
-            }
+        retval = dict(self.extraoptions)
+        retval.update( {
+                'fileform': unicode(self.fileform.currentText()),
+                'collision': unicode(self.collision.currentText()),
+                'updatemeta': self.updatemeta.isChecked(),
+                'bgmeta': False, # self.bgmeta.isChecked(),
+                'smarten_punctuation':self.prefs['smarten_punctuation'],
+                'do_wordcount':self.prefs['do_wordcount'],
+                } )
 
         if self.merge:
             retval['fileform']=='epub'
-            retval['updateepubcover']=True
             if self.newmerge:
                 retval['updatemeta']=True
                 retval['collision']=ADDNEW
@@ -501,7 +513,6 @@ class AddNewDialog(SizePersistedDialog):
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
 
-
 class FakeLineEdit():
     def __init__(self):
         pass
@@ -577,35 +588,83 @@ class UserPassDialog(QDialog):
         QDialog.__init__(self, gui)
         self.status=False
 
-        self.l = QGridLayout()
+        self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        grid = QGridLayout()
+        self.l.addLayout(grid)
+
         if exception and exception.passwdonly:
             self.setWindowTitle(_('Password'))
-            self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
+            grid.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
             # user isn't used, but it's easier to still have it for
             # post processing.
             self.user = FakeLineEdit()
         else:
             self.setWindowTitle(_('User/Password'))
-            self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
+            grid.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
 
-            self.l.addWidget(QLabel(_("User:")),1,0)
+            grid.addWidget(QLabel(_("User:")),1,0)
             self.user = QLineEdit(self)
-            self.l.addWidget(self.user,1,1)
+            grid.addWidget(self.user,1,1)
 
-        self.l.addWidget(QLabel(_("Password:")),2,0)
+        grid.addWidget(QLabel(_("Password:")),2,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
-        self.l.addWidget(self.passwd,2,1)
+        grid.addWidget(self.passwd,2,1)
+
+        horz = QHBoxLayout()
+        self.l.addLayout(horz)
 
         self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,3,0)
+        horz.addWidget(self.ok_button)
 
         self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,3,1)
+        horz.addWidget(self.cancel_button)
+
+        self.resize(self.sizeHint())
+
+    def ok(self):
+        self.status=True
+        self.hide()
+
+    def cancel(self):
+        self.status=False
+        self.hide()
+
+class TOTPDialog(QDialog):
+    '''
+    Need to collect Timebased One Time Password(TOTP) for some sites.
+    '''
+    def __init__(self, gui, site, exception=None):
+        QDialog.__init__(self, gui)
+        self.status=False
+
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        grid = QGridLayout()
+        self.l.addLayout(grid)
+
+        self.setWindowTitle(_('Time-based One Time Password(TOTP)'))
+        grid.addWidget(QLabel(_("Site requires a Time-based One Time Password(TOTP) for this url:\n%s")%exception.url),0,0,1,2)
+
+        grid.addWidget(QLabel(_("TOTP:")),2,0)
+        self.totp = QLineEdit(self)
+        grid.addWidget(self.totp,2,1)
+
+        horz = QHBoxLayout()
+        self.l.addLayout(horz)
+
+        self.ok_button = QPushButton(_('OK'), self)
+        self.ok_button.clicked.connect(self.ok)
+        horz.addWidget(self.ok_button)
+
+        self.cancel_button = QPushButton(_('Cancel'), self)
+        self.cancel_button.clicked.connect(self.cancel)
+        horz.addWidget(self.cancel_button)
 
         self.resize(self.sizeHint())
 
@@ -623,13 +682,15 @@ def LoopProgressDialog(gui,
                        finish_function,
                        init_label=_("Fetching metadata for stories..."),
                        win_title=_("Downloading metadata for stories"),
-                       status_prefix=_("Fetched metadata for")):
+                       status_prefix=_("Fetched metadata for"),
+                       disable_cancel=False):
     ld = _LoopProgressDialog(gui,
                              book_list,
                              foreach_function,
                              init_label,
                              win_title,
-                             status_prefix)
+                             status_prefix,
+                             disable_cancel)
 
     # Mac OS X gets upset if the finish_function is called from inside
     # the real _LoopProgressDialog class.
@@ -647,10 +708,12 @@ class _LoopProgressDialog(QProgressDialog):
                  foreach_function,
                  init_label=_("Fetching metadata for stories..."),
                  win_title=_("Downloading metadata for stories"),
-                 status_prefix=_("Fetched metadata for")):
+                 status_prefix=_("Fetched metadata for"),
+                 disable_cancel=False):
         QProgressDialog.__init__(self,
                                  init_label,
                                  _('Cancel'), 0, len(book_list), gui)
+        self.gui = gui
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.book_list = book_list
@@ -658,7 +721,6 @@ class _LoopProgressDialog(QProgressDialog):
         self.status_prefix = status_prefix
         self.i = 0
         self.start_time = datetime.now()
-        self.first = True
 
         # can't import at file load.
         from calibre_plugins.fanficfare_plugin.prefs import prefs
@@ -667,11 +729,27 @@ class _LoopProgressDialog(QProgressDialog):
         self.setLabelText('%s %d / %d' % (self.status_prefix, self.i, len(self.book_list)))
         self.setValue(self.i)
 
+        if disable_cancel:
+            self.setCancelButton(None)
+            self.reject = self.disabled_reject
+            self.closeEvent = self.disabled_closeEvent
+
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
         ## A weird way to do a loop, but that was the example I had.
-        QTimer.singleShot(0, self.do_loop)
+        ## 100 instead of 0 on the first go due to Win10(and later
+        ## qt6) not displaying dialog properly.
+        QTimer.singleShot(100, self.do_loop)
         self.exec_()
 
+    # used when disable_cancel = True
+    def disabled_reject(self):
+        pass
+
+    # used when disable_cancel = True
+    def disabled_closeEvent(self, event):
+        if event.spontaneous():
+            event.ignore()
+
     def updateStatus(self):
         remaining_time_string = ''
         if self.show_est_time and self.i > -1:
@@ -685,15 +763,6 @@ class _LoopProgressDialog(QProgressDialog):
 
     def do_loop(self):
 
-        if self.first:
-            ## Windows 10 doesn't want to show the prog dialog content
-            ## until after the timer's been called again.  Something to
-            ## do with cooperative multi threading maybe?
-            ## So this just trips the timer loop an extra time at the start.
-            self.first = False
-            QTimer.singleShot(0, self.do_loop)
-            return
-
         book = self.book_list[self.i]
         try:
             ## collision spec passed into getadapter by partial from fff_plugin
@@ -892,11 +961,6 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.updatemeta.setChecked(self.prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
 
-        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
-        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
-        horz.addWidget(self.updateepubcover)
-
         self.bgmeta = QCheckBox(_('Background Metadata?'),self)
         self.bgmeta.setToolTip(_("Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail."))
         self.bgmeta.setChecked(self.prefs['bgmeta'])
@@ -948,7 +1012,6 @@ class UpdateExistingDialog(SizePersistedDialog):
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'bgmeta': self.bgmeta.isChecked(),
-            'updateepubcover': self.updateepubcover.isChecked(),
             'smarten_punctuation':self.prefs['smarten_punctuation'],
             'do_wordcount':self.prefs['do_wordcount'],
             }
@@ -1029,6 +1092,7 @@ class StoryListTableWidget(QTableWidget):
     def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
+        rows = sorted(rows, key=lambda x: x.row(), reverse=True)
         if len(rows) == 0:
             return
         message = '<p>'+_('Are you sure you want to remove this book from the list?')
@@ -1037,7 +1101,7 @@ class StoryListTableWidget(QTableWidget):
         if not confirm(message,'fff_delete_item', self):
             return
         first_sel_row = self.currentRow()
-        for selrow in reversed(rows):
+        for selrow in rows:
             self.removeRow(selrow.row())
         if first_sel_row < self.rowCount():
             self.select_and_scroll_to_row(first_sel_row)
@@ -1048,6 +1112,19 @@ class StoryListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
+## Added to allow sorting by Notes column
+class NotesWidgetItem(QTableWidgetItem):
+    def __init__(self,content):
+        QTableWidgetItem.__init__(self)
+        self.content=content
+
+    def currentText(self):
+        return self.content.currentText()
+
+    def __lt__(self, other):
+        return (unicode(self.currentText()).lower().strip() <
+                unicode(other.currentText()).lower().strip())
+
 class RejectListTableWidget(QTableWidget):
 
     def __init__(self, parent,rejectreasons=[]):
@@ -1092,6 +1169,7 @@ class RejectListTableWidget(QTableWidget):
         self.setItem(row, 1, EditableTableWidgetItem(rej.title))
         self.setItem(row, 2, EditableTableWidgetItem(rej.auth))
 
+        # sort_func orders dropdown-constant to preserve user order.
         note_cell = EditWithComplete(self,sort_func=lambda x:1)
 
         items = [rej.note]+self.rejectreasons
@@ -1099,12 +1177,14 @@ class RejectListTableWidget(QTableWidget):
         note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
         note_cell.setToolTip(_('Select or Edit Reject Note.'))
+        self.setItem(row, 3, NotesWidgetItem(note_cell))
         self.setCellWidget(row, 3, note_cell)
         note_cell.setCursorPosition(0)
 
     def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
+        rows = sorted(rows, key=lambda x: x.row(), reverse=True)
         if len(rows) == 0:
             return
         message = '<p>'+_('Are you sure you want to remove this URL from the list?')
@@ -1113,7 +1193,7 @@ class RejectListTableWidget(QTableWidget):
         if not confirm(message,'fff_rejectlist_delete_item_again', self):
             return
         first_sel_row = self.currentRow()
-        for selrow in reversed(rows):
+        for selrow in rows:
             self.removeRow(selrow.row())
         if first_sel_row < self.rowCount():
             self.select_and_scroll_to_row(first_sel_row)
@@ -1240,6 +1320,7 @@ class EditTextDialog(SizePersistedDialog):
                  icon=None, title=None, label=None, tooltip=None,
                  read_only=False,
                  rejectreasons=[],reasonslabel=None,
+                 accept_storyurls=False,
                  save_size_name='fff:edit text dialog',
                  ):
         SizePersistedDialog.__init__(self, parent, save_size_name)
@@ -1253,7 +1334,10 @@ class EditTextDialog(SizePersistedDialog):
             self.setWindowIcon(icon)
         self.l.addWidget(self.label)
 
-        self.textedit = QTextEdit(self)
+        if accept_storyurls:
+            self.textedit = DroppableQTextEdit(self)
+        else:
+            self.textedit = QTextEdit(self)
         self.textedit.setLineWrapMode(QTextEditNoWrap)
         self.textedit.setReadOnly(read_only)
         self.textedit.setText(text)
@@ -1297,7 +1381,18 @@ class EditTextDialog(SizePersistedDialog):
     def get_reason_text(self):
         return unicode(self.reason_edit.currentText()).strip()
 
-class IniTextDialog(SizePersistedDialog):
+class QTextEditPlainPaste(QTextEdit):
+    def insertFromMimeData(self, mimeData):
+        # logger.debug("insertFromMimeData called")
+        #Ensure it is text.
+        if (mimeData.hasText()):
+            text = mimeData.text()
+            self.insertPlainText(text)
+        #In case not text.
+        else:
+            QTextEdit.insertFromMimeData(self, mimeData)
+
+class IniTextDialog(HotKeyedSizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None,
@@ -1305,9 +1400,7 @@ class IniTextDialog(SizePersistedDialog):
                  read_only=False,
                  save_size_name='fff:ini text dialog',
                  ):
-        SizePersistedDialog.__init__(self, parent, save_size_name)
-
-        self.keys=dict()
+        super(IniTextDialog,self).__init__(parent, save_size_name)
 
         self.l = QVBoxLayout()
         self.setLayout(self.l)
@@ -1318,7 +1411,7 @@ class IniTextDialog(SizePersistedDialog):
             self.setWindowIcon(icon)
         self.l.addWidget(self.label)
 
-        self.textedit = QTextEdit(self)
+        self.textedit = QTextEditPlainPaste(self)
 
         highlighter = IniHighlighter(self.textedit,
                                      sections=get_valid_sections(),
@@ -1374,6 +1467,8 @@ class IniTextDialog(SizePersistedDialog):
             self.addCtrlKeyPress(QtCore.Qt.Key_F,self.findFocus)
             self.addCtrlKeyPress(QtCore.Qt.Key_G,self.find)
 
+        self.addCtrlKeyPress(QtCore.Qt.Key_Return,self.accept)
+        self.addCtrlKeyPress(QtCore.Qt.Key_Enter,self.accept) # num pad
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
@@ -1406,19 +1501,6 @@ class IniTextDialog(SizePersistedDialog):
             # print("call parent accept")
             return SizePersistedDialog.accept(self)
 
-    def addCtrlKeyPress(self,key,func):
-        # print("addKeyPress: key(0x%x)"%key)
-        # print("control: 0x%x"%QtCore.Qt.ControlModifier)
-        self.keys[key]=func
-
-    def keyPressEvent(self, event):
-        # print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
-        if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
-            func = self.keys[event.key()]
-            return func()
-        else:
-            return SizePersistedDialog.keyPressEvent(self, event)
-
     def get_plain_text(self):
         return unicode(self.textedit.toPlainText())
 
@@ -1452,7 +1534,7 @@ class IniTextDialog(SizePersistedDialog):
         else:
             # Make the next search start from the begining again
             self.lastStart = 0
-            self.textedit.moveCursor(self.textedit.textCursor().Start)
+            self.textedit.moveCursor(MoveOperations.Start)
 
     def moveCursor(self,start,end):
 
@@ -1464,7 +1546,8 @@ class IniTextDialog(SizePersistedDialog):
 
         # Next we move the Cursor by over the match and pass the KeepAnchor parameter
         # which will make the cursor select the match's text
-        cursor.movePosition(cursor.Right,cursor.KeepAnchor,end - start)
+        cursor.movePosition(MoveOperations.Right,
+                            MoveMode.KeepAnchor,end - start)
 
         # And finally we set this new cursor as the parent's
         self.textedit.setTextCursor(cursor)
@@ -1478,15 +1561,14 @@ class IniTextDialog(SizePersistedDialog):
         cursor.setPosition(0)
 
         # Next we move the Cursor down lineno times
-        cursor.movePosition(cursor.Down,cursor.MoveAnchor,lineno-1)
+        cursor.movePosition(MoveOperations.Down,MoveMode.MoveAnchor,lineno-1)
 
         # Next we move the Cursor to the end of the line
-        cursor.movePosition(cursor.EndOfLine,cursor.KeepAnchor,1)
+        cursor.movePosition(MoveOperations.EndOfLine,MoveMode.KeepAnchor,1)
 
         # And finally we set this new cursor as the parent's
         self.textedit.setTextCursor(cursor)
 
-
 class ViewLog(SizePersistedDialog):
 
     def label_clicked(self, event, lineno=None):
@@ -1575,28 +1657,30 @@ class EmailPassDialog(QDialog):
         QDialog.__init__(self, gui)
         self.status=False
 
-        self.l = QGridLayout()
+        self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        self.setWindowTitle(_('Password'))
-        self.l.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
+        grid = QGridLayout()
+        self.l.addLayout(grid)
 
-        # self.l.addWidget(QLabel(_("Password:")),1,0)
+        self.setWindowTitle(_('Password'))
+        grid.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
+
+        # grid.addWidget(QLabel(_("Password:")),1,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
-        self.l.addWidget(self.passwd,1,0,1,2)
+        grid.addWidget(self.passwd,1,0,1,2)
+
+        horz = QHBoxLayout()
+        self.l.addLayout(horz)
 
         self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,2,0)
+        horz.addWidget(self.ok_button)
 
         self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,2,1)
-
-        # set stretch factors the same.
-        self.l.setColumnStretch(0,1)
-        self.l.setColumnStretch(1,1)
+        horz.addWidget(self.cancel_button)
 
         self.resize(self.sizeHint())
 

calibre-plugin/fff_util.py

@@ -33,8 +33,8 @@ def get_fff_config(url,fileform="epub",personalini=None):
     except Exception as e:
         logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
     configuration = Configuration(sections,fileform)
-    configuration.readfp(StringIO(ensure_text(get_resources("plugin-defaults.ini"))))
-    configuration.readfp(StringIO(ensure_text(personalini)))
+    configuration.read_file(StringIO(ensure_text(get_resources("plugin-defaults.ini"))))
+    configuration.read_file(StringIO(ensure_text(personalini)))
 
     return configuration
 

calibre-plugin/inihighlighter.py

@@ -58,7 +58,7 @@ class IniHighlighter(QSyntaxHighlighter):
                     'knownkeywords':QColor(Qt.blue).lighter(150),
                     'knownsections':Qt.darkCyan,
                     'teststories':Qt.cyan,
-                    'storyUrls':Qt.magenta,
+                    'storyUrls':QColor(Qt.magenta).lighter(150),
                     'comments':Qt.yellow
                     }
         except Exception as e:
@@ -95,7 +95,7 @@ class IniHighlighter(QSyntaxHighlighter):
         if sections:
             # *known* sections
             resections = r'('+(r'|'.join(sections))+r')'
-            resections = resections.replace('.','\.') #escape dots.
+            resections = resections.replace('.',r'\.') #escape dots.
             self.highlightingRules.append( HighlightingRule( r"^\["+resections+r"\]\s*$", colors['knownsections'], QFontBold, blocknum=2 ) )
 
         # test story sections
@@ -103,7 +103,8 @@ class IniHighlighter(QSyntaxHighlighter):
         self.highlightingRules.append( self.teststoryRule )
 
         # storyUrl sections
-        self.storyUrlRule = HighlightingRule( r"^\[https?://.*\]", colors['storyUrls'], blocknum=4 )
+        # StoryUrls are *not* checked beyond looking for https?://
+        self.storyUrlRule = HighlightingRule( r"^\[https?://.*\]", colors['storyUrls'], QFontBold, blocknum=2 )
         self.highlightingRules.append( self.storyUrlRule )
 
         # NOT comments -- but can be custom columns, so don't flag.
@@ -134,9 +135,10 @@ class IniHighlighter(QSyntaxHighlighter):
             if blocknum == 3:
                 self.setFormat( 0, len(text), self.teststoryRule.highlight )
 
-            # storyUrl section rules:
-            if blocknum == 4:
-                self.setFormat( 0, len(text), self.storyUrlRule.highlight )
+            ## changed storyUrl section to also be blocknum=1 April 2023
+            ## storyUrl section rules:
+            # if blocknum == 4:
+            #     self.setFormat( 0, len(text), self.storyUrlRule.highlight )
 
         self.setCurrentBlockState( blocknum )
 

calibre-plugin/jobs.py

@@ -2,7 +2,6 @@
 
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
-import six
 
 __license__   = 'GPL v3'
 __copyright__ = '2020, Jim Miller, 2011, Grant Drake <grant.drake@gmail.com>'
@@ -15,10 +14,8 @@ from time import sleep
 from datetime import time
 from io import StringIO
 from collections import defaultdict
+import sys
 
-from calibre.utils.ipc.server import Empty, Server
-from calibre.utils.ipc.job import ParallelJob
-from calibre.constants import numeric_version as calibre_version
 from calibre.utils.date import local_tz
 
 # pulls in translation files for _() strings
@@ -33,160 +30,100 @@ except NameError:
 #
 # ------------------------------------------------------------------------------
 
-def do_download_worker(book_list,
-                       options,
-                       cpus,
-                       merge=False,
-                       notification=lambda x,y:x):
-    '''
-    Coordinator job, to launch child jobs to do downloads.
-    This is run as a worker job in the background to keep the UI more
-    responsive and get around any memory leak issues as it will launch
-    a child job for each book as a worker process
-    '''
-    ## Now running one BG proc per site, which downloads for the same
-    ## site in serial.
-    logger.info("CPUs:%s"%cpus)
-    server = Server(pool_size=cpus)
+def do_download_worker_single(site,
+                              book_list,
+                              options,
+                              merge,
+                              notification=lambda x,y:x):
 
     logger.info(options['version'])
 
-    sites_lists = defaultdict(list)
-    [ sites_lists[x['site']].append(x) for x in book_list if x['good'] ]
+    ## same info debug calibre prints out at startup. For when users
+    ## give me job output instead of debug log.
+    from calibre.debug import print_basic_debug_info
+    print_basic_debug_info(sys.stderr)
 
-    totals = {}
-    # can't do direct assignment in list comprehension?  I'm sure it
-    # makes sense to some pythonista.
-    # [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
-    [ totals.update({x['url']:0.0}) for x in book_list if x['good']  ]
-    # logger.debug(sites_lists.keys())
-
-    # Queue all the jobs
-    jobs_running = 0
-    for site in sites_lists.keys():
-        site_list = sites_lists[site]
-        logger.info(_("Launch background process for site %s:")%site + "\n" +
-                    "\n".join([ x['url'] for x in site_list ]))
-        # logger.debug([ x['url'] for x in site_list])
-        args = ['calibre_plugins.fanficfare_plugin.jobs',
-                'do_download_site',
-                (site,site_list,options,merge)]
-        job = ParallelJob('arbitrary_n',
-                          "site:(%s)"%site,
-                          done=None,
-                          args=args)
-        job._site_list = site_list
-        job._processed = False
-        server.add_job(job)
-        jobs_running += 1
-
-    # This server is an arbitrary_n job, so there is a notifier available.
-    # Set the % complete to a small number to avoid the 'unavailable' indicator
     notification(0.01, _('Downloading FanFiction Stories'))
+    from calibre_plugins.fanficfare_plugin import FanFicFareBase
+    fffbase = FanFicFareBase(options['plugin_path'])
+    with fffbase: # so the sys.path was modified while loading the
+                  # plug impl.
+        from fanficfare.fff_profile import do_cprofile
 
-    # dequeue the job results as they arrive, saving the results
-    count = 0
-    while True:
-        job = server.changed_jobs_queue.get()
-        # logger.debug("job get job._processed:%s"%job._processed)
-        # A job can 'change' when it is not finished, for example if it
-        # produces a notification.
-        msg = None
-        try:
-            ## msg = book['url']
-            (percent,msg) = job.notifications.get_nowait()
-            # logger.debug("%s<-%s"%(percent,msg))
-            if percent == 10.0: # Only when signaling d/l done.
-                count += 1
-                totals[msg] = 1.0/len(totals)
-                # logger.info("Finished: %s"%msg)
-            else:
+        ## extra function just so I can easily use the same
+        ## @do_cprofile decorator
+        @do_cprofile
+        def profiled_func():
+            count = 0
+            totals = {}
+            # can't do direct assignment in list comprehension?  I'm sure it
+            # makes sense to some pythonista.
+            # [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
+            [ totals.update({x['url']:0.0}) for x in book_list if x['good']  ]
+            # logger.debug(sites_lists.keys())
+
+            def do_indiv_notif(percent,msg):
                 totals[msg] = percent/len(totals)
-            notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
-        except Empty:
-            pass
-        # without update, is_finished will never be set.  however, we
-        # do want to get all the notifications for status so we don't
-        # miss the 'done' ones.
-        job.update(consume_notifications=False)
+                notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
 
-        # if not job._processed:
-        #     sleep(0.5)
-        ## Can have a race condition where job.is_finished before
-        ## notifications for all downloads have been processed.
-        ## Or even after the job has been finished.
-        # logger.debug("job.is_finished(%s) or job._processed(%s)"%(job.is_finished, job._processed))
-        if not job.is_finished:
-            continue
-
-        ## only process each job once.  We can get more than one loop
-        ## after job.is_finished.
-        if not job._processed:
-            # sleep(1)
-            # A job really finished. Get the information.
-
-            ## This is where bg proc details end up in GUI log.
-            ## job.details is the whole debug log for each proc.
-            logger.info("\n\n" + ("="*80) + " " + job.details.replace('\r',''))
-            # logger.debug("Finished background process for site %s:\n%s"%(job._site_list[0]['site'],"\n".join([ x['url'] for x in job._site_list ])))
-            for b in job._site_list:
-                book_list.remove(b)
-            book_list.extend(job.result)
-            job._processed = True
-            jobs_running -= 1
-
-        ## Can't use individual count--I've seen stories all reported
-        ## finished before results of all jobs processed.
-        if jobs_running == 0:
-            book_list = sorted(book_list,key=lambda x : x['listorder'])
-            logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
-
-            good_lists = defaultdict(list)
-            bad_lists = defaultdict(list)
+            do_list = []
+            done_list = []
+            logger.info("\n\n"+_("Downloading FanFiction Stories")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
+            ## pass failures from metadata through bg job so all results are
+            ## together.
             for book in book_list:
                 if book['good']:
-                    good_lists[book['status']].append(book)
+                    do_list.append(book)
                 else:
-                    bad_lists[book['status']].append(book)
+                    done_list.append(book)
+            for book in do_list:
+                # logger.info("%s"%book['url'])
+                done_list.append(do_download_for_worker(book,options,merge,do_indiv_notif))
+                count += 1
+            return finish_download(done_list)
+        return profiled_func()
 
-            order = [_('Add'),
-                     _('Update'),
-                     _('Meta'),
-                     _('Different URL'),
-                     _('Rejected'),
-                     _('Skipped'),
-                     _('Bad'),
-                     _('Error'),
-                     ]
-            j = 0
-            for d in [ good_lists, bad_lists ]:
-                for status in order:
-                    if d[status]:
-                        l = d[status]
-                        logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in l])))
-                        for book in l:
-                            book['reportorder'] = j
-                            j += 1
-                    del d[status]
-                # just in case a status is added but doesn't appear in order.
-                for status in d.keys():
-                    logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in d[status]])))
-            break
+def finish_download(donelist):
+    book_list = sorted(donelist,key=lambda x : x['listorder'])
+    logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
 
-    server.close()
+    good_lists = defaultdict(list)
+    bad_lists = defaultdict(list)
+    for book in book_list:
+        if book['good']:
+            good_lists[book['status']].append(book)
+        else:
+            bad_lists[book['status']].append(book)
+
+    order = [_('Add'),
+             _('Update'),
+             _('Meta'),
+             _('Different URL'),
+             _('Rejected'),
+             _('Skipped'),
+             _('Bad'),
+             _('Error'),
+             ]
+    stnum = 0
+    for d in [ good_lists, bad_lists ]:
+        for status in order:
+            stnum += 1
+            if d[status]:
+                l = d[status]
+                logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in l])))
+                for book in l:
+                    # Add prior listorder to 10000 * status num for
+                    # ordering of accumulated results with multiple bg
+                    # jobs
+                    book['reportorder'] = stnum*10000 + book['listorder']
+            del d[status]
+        # just in case a status is added but doesn't appear in order.
+        for status in d.keys():
+            logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in d[status]])))
 
     # return the book list as the job result
     return book_list
 
-def do_download_site(site,book_list,options,merge,notification=lambda x,y:x):
-    # logger.info(_("Started job for %s")%site)
-    retval = []
-    for book in book_list:
-        # logger.info("%s"%book['url'])
-        retval.append(do_download_for_worker(book,options,merge,notification))
-        notification(10.0,book['url'])
-    return retval
-
 def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
     '''
     Child job, to download story when run as a worker job
@@ -196,13 +133,13 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
     fffbase = FanFicFareBase(options['plugin_path'])
     with fffbase: # so the sys.path was modified while loading the
                   # plug impl.
-        from calibre_plugins.fanficfare_plugin.dialogs import NotGoingToDownload
         from calibre_plugins.fanficfare_plugin.prefs import (
                 SAVE_YES, SAVE_YES_UNLESS_SITE, OVERWRITE, OVERWRITEALWAYS, UPDATE,
                 UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
         from calibre_plugins.fanficfare_plugin.wordcount import get_word_count
         from fanficfare import adapters, writers
         from fanficfare.epubutils import get_update_data
+        from fanficfare.exceptions import NotGoingToDownload
         from fanficfare.six import text_type as unicode
 
         from calibre_plugins.fanficfare_plugin.fff_util import get_fff_config
@@ -222,9 +159,6 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                                             options['fileform'],
                                             options['personal.ini'])
 
-            if not options['updateepubcover'] and 'epub_for_update' in book and book['collision'] in (UPDATE, UPDATEALWAYS):
-                configuration.set("overrides","never_make_cover","true")
-
             # images only for epub, html, even if the user mistakenly
             # turned it on else where.
             if options['fileform'] not in ("epub","html"):
@@ -234,18 +168,12 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
             adapter.is_adult = book['is_adult']
             adapter.username = book['username']
             adapter.password = book['password']
+            adapter.totp = book['totp']
             adapter.setChaptersRange(book['begin'],book['end'])
 
             ## each site download job starts with a new copy of the
             ## cookiejar and basic_cache from the FG process.  They
             ## are not shared between different sites' BG downloads
-            if configuration.getConfig('use_browser_cache'):
-                if 'browser_cache' in options:
-                    configuration.set_browser_cache(options['browser_cache'])
-                else:
-                    options['browser_cache'] = configuration.get_browser_cache()
-                    if 'browser_cachefile' in options:
-                        options['browser_cache'].load_cache(options['browser_cachefile'])
             if 'basic_cache' in options:
                 configuration.set_basic_cache(options['basic_cache'])
             else:
@@ -261,6 +189,17 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
             if not story.getMetadata("series") and 'calibre_series' in book:
                 adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
 
+            # logger.debug(merge)
+            # logger.debug(book.get('epub_for_update','(NONE)'))
+            # logger.debug(options.get('mergebook','(NOMERGEBOOK)'))
+
+            # is a merge, is a pre-existing anthology, and is not a pre-existing book in anthology.
+            if merge and 'mergebook' in options and 'epub_for_update' not in book:
+                # internal for plugin anthologies to mark chapters
+                # (new) in new stories
+                story.setMetadata("newforanthology","true")
+            logger.debug("metadata newforanthology:%s"%story.getMetadata("newforanthology"))
+
             # set PI version instead of default.
             if 'version' in options:
                 story.setMetadata('version',options['version'])
@@ -269,7 +208,6 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
             book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
             book['publisher'] = story.getMetadata("publisher")
             book['url'] = story.getMetadata("storyUrl", removeallentities=True)
-            book['tags'] = story.getSubjectTags(removeallentities=True)
             book['comments'] = story.get_sanitized_description()
             book['series'] = story.getMetadata("series", removeallentities=True)
 
@@ -346,20 +284,21 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                  adapter.oldchaptersmap,
                  adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
 
-                # dup handling from fff_plugin needed for anthology updates.
-                if book['collision'] == UPDATE:
-                    if chaptercount == urlchaptercount:
+                # dup handling from fff_plugin needed for anthology updates & BG metadata.
+                if book['collision'] in (UPDATE,UPDATEALWAYS):
+                    if chaptercount == urlchaptercount and book['collision'] == UPDATE:
                         if merge:
+                            ## Deliberately pass for UPDATEALWAYS merge.
                             book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                             book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                             if options['savemetacol'] != '':
                                 book['savemetacol'] = story.dump_html_metadata()
                             book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                             return book
-                        else: # not merge,
+                        else:
                             raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
-                    elif chaptercount > urlchaptercount:
-                        raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
+                    elif chaptercount > urlchaptercount and not (book['collision'] == UPDATEALWAYS and adapter.getConfig('force_update_epub_always')):
+                        raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite or force_update_epub_always to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                     elif chaptercount == 0:
                         raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
 
@@ -397,7 +336,11 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                 options['do_wordcount'] == SAVE_YES_UNLESS_SITE and not story.getMetadataRaw('numWords') ):
                 try:
                     wordcount = get_word_count(outfile)
-                   # logger.info("get_word_count:%s"%wordcount)
+                    # logger.info("get_word_count:%s"%wordcount)
+                    # clear cache for the rather unusual case of
+                    # numWords affecting other previously cached
+                    # entries.
+                    story.clear_processed_metadata_cache()
                     story.setMetadata('numWords',wordcount)
                     writer.writeStory(outfilename=outfile, forceOverwrite=True)
                     book['all_metadata'] = story.getAllMetadata(removeallentities=True)
@@ -406,8 +349,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                 except:
                     logger.error("WordCount failed")
 
-            if options['smarten_punctuation'] and options['fileform'] == "epub" \
-                    and calibre_version >= (0, 9, 39):
+            if options['smarten_punctuation'] and options['fileform'] == "epub":
                 # for smarten punc
                 from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                 from calibre.utils.logging import Log
@@ -417,12 +359,14 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                 data = {'smarten_punctuation':True}
                 opts = ALL_OPTS.copy()
                 opts.update(data)
-                O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
+                O = namedtuple('Options', ' '.join(ALL_OPTS.keys()))
                 opts = O(**opts)
 
                 log = Log(level=Log.DEBUG)
                 polish({outfile:outfile}, opts, log, logger.info)
-
+            ## here to catch tags set in chapters in literotica for
+            ## both overwrites and updates.
+            book['tags'] = story.getSubjectTags(removeallentities=True)
         except NotGoingToDownload as d:
             book['good']=False
             book['status']=_('Bad')
@@ -448,11 +392,12 @@ def inject_cal_cols(book,story,configuration):
     if 'calibre_columns' in book:
         injectini = ['[injected]']
         extra_valid = []
-        for k, v in six.iteritems(book['calibre_columns']):
+        for k in book['calibre_columns'].keys():
+            v = book['calibre_columns'][k]
             story.setMetadata(k,v['val'])
             injectini.append('%s_label:%s'%(k,v['label']))
             extra_valid.append(k)
         if extra_valid: # if empty, there's nothing to add.
             injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
-            configuration.readfp(StringIO('\n'.join(injectini)))
+            configuration.read_file(StringIO('\n'.join(injectini)))
             #print("added:\n%s\n"%('\n'.join(injectini)))

calibre-plugin/plugin-example.ini

@@ -3,22 +3,9 @@
 
 [defaults]
 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels.  Example:
-
-## [defaults]
-## titlepage_entries: category,genre, status
-## [www.whofic.com]
-## # overrides defaults.
-## titlepage_entries: category,genre, status,dateUpdated,rating
-## [epub]
-## # overrides defaults & site section
-## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
-## [www.whofic.com:epub]
-## # overrides defaults, site section & format section
-## titlepage_entries: category,genre, status,datePublished
-## [overrides]
-## # overrides all other sections
-## titlepage_entries: category
+## overridden at several levels.  See
+## https://github.com/JimmXinu/FanFicFare/wiki/INI-File for more
+## details.
 
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -29,42 +16,32 @@
 ## want to make them all look the same?  Strip them off, then add them
 ## back on with add_chapter_numbers.  Don't like the way it strips
 ## numbers or adds them back?  See chapter_title_strip_pattern and
-## chapter_title_add_pattern.
+## chapter_title_add_pattern in defaults.ini.
 #strip_chapter_numbers:true
 #add_chapter_numbers:true
 
-## Add this to genre if there's more than one category.
-#add_genre_when_multi_category: Crossover
 
 [epub]
-## include images from img tags in the body and summary of stories.
+## Include images from img tags in the body and summary of stories.
 ## Images will be converted to jpg for size if possible.  Images work
 ## in epub format only.  To get mobi or other format with images,
 ## download as epub and use Calibre to convert.
+## true by default, uncomment and set false to not include images.
 #include_images:true
 
-## Quality level to use when converting images to jpg. Range is 0-100,
-## reasonable values likely to be in the range 70-95.
-#jpg_quality: 95
-
-## If not set, the summary will have all html stripped for safety.
+## If set false, the summary will have all html stripped for safety.
 ## Both this and include_images must be true to get images in the
 ## summary.
+## true by default, uncomment and set false to not keep summary html.
 #keep_summary_html:true
 
-## If set, the first image found will be made the cover image.  If
-## keep_summary_html is true, any images in summary will be before any
+## If set true, and there isn't a specific cover image, the first
+## image found in the story will be made the cover image.  If
+## keep_summary_html is true, images in the summary will be before any
 ## in chapters.
+## true by default, uncomment and set false to turn off
 #make_firstimage_cover:true
 
-## Resize images down to width, height, preserving aspect ratio.
-## Nook size, with margin.
-#image_max_size: 580, 725
-
-## Change image to grayscale, if graphics library allows, to save
-## space.
-#grayscale_images: false
-
 
 ## Most common, I expect will be using this to save username/passwords
 ## for different sites.  Here are a few examples.  See defaults.ini
@@ -76,22 +53,6 @@
 ## default is false
 #collect_series: true
 
-[ficwad.com]
-#username:YourUsername
-#password:YourPassword
-
-[www.adastrafanfic.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.
-#is_adult:true
-
-[www.twcslibrary.net]
-#username:YourName
-#password:yourpassword
-#is_adult:true
-## default is false
-#collect_series: true
-
 [www.fimfiction.net]
 #is_adult:true
 #fail_on_password: false
@@ -100,8 +61,9 @@
 #is_adult:true
 ## tth is a little unusual--it doesn't require user/pass, but the site
 ## keeps track of which chapters you've read and won't send another
-## update until it thinks you're up to date.  This way, on download,
-## it thinks you're up to date.
+## update until it thinks you're up to date.  If you set
+## username/password, FFF will login to download.  Then the site
+## thinks you're up to date.
 #username:YourName
 #password:yourpassword
 

calibre-plugin/prefs.py

@@ -120,12 +120,13 @@ default_prefs['reject_delete_default'] = True
 
 default_prefs['updatemeta'] = True
 default_prefs['bgmeta'] = False
-default_prefs['updateepubcover'] = False
+#default_prefs['updateepubcover'] = True # removed in favor of always True Oct 2022
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['authorcase'] = False
 default_prefs['titlecase'] = False
+default_prefs['seriescase'] = False
 default_prefs['setanthologyseries'] = False
 default_prefs['mark'] = False
 default_prefs['mark_success'] = True
@@ -159,11 +160,12 @@ default_prefs['addtolistsonread'] = False
 default_prefs['autounnew'] = False
 
 default_prefs['updatecalcover'] = SAVE_YES_IF_IMG
-default_prefs['gencalcover'] = SAVE_YES
+default_prefs['covernewonly'] = False
+default_prefs['gencalcover'] = SAVE_YES_UNLESS_IMG
 default_prefs['updatecover'] = False
-default_prefs['calibre_gen_cover'] = False
-default_prefs['plugin_gen_cover'] = True
-default_prefs['gcnewonly'] = False
+default_prefs['calibre_gen_cover'] = True
+default_prefs['plugin_gen_cover'] = False
+default_prefs['gcnewonly'] = True
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
 default_prefs['gc_polish_cover'] = False
@@ -181,6 +183,7 @@ default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
 default_prefs['set_author_url'] = True
+default_prefs['set_series_url'] = True
 default_prefs['includecomments'] = False
 default_prefs['anth_comments_newonly'] = True
 
@@ -195,6 +198,11 @@ default_prefs['auto_reject_from_email'] = False
 default_prefs['update_existing_only_from_email'] = False
 default_prefs['download_from_email_immediately'] = False
 
+
+#default_prefs['single_proc_jobs'] = True # setting and code removed
+default_prefs['site_split_jobs'] = True
+default_prefs['reconsolidate_jobs'] = True
+
 def set_library_config(library_config,db,setting=PREFS_KEY_SETTINGS):
     db.prefs.set_namespaced(PREFS_NAMESPACE,
                             setting,

fanficfare/adapters/__init__.py

@@ -30,8 +30,12 @@ from .. import configurable as configurable
 
 ## must import each adapter here.
 
+from . import base_adapter
 from . import base_efiction_adapter
 from . import adapter_test1
+from . import adapter_test2
+from . import adapter_test3
+from . import adapter_test4
 from . import adapter_fanfictionnet
 from . import adapter_fictionalleyarchiveorg
 from . import adapter_fictionpresscom
@@ -49,9 +53,7 @@ from . import adapter_archiveofourownorg
 from . import adapter_ficbooknet
 from . import adapter_midnightwhispers
 from . import adapter_ksarchivecom
-from . import adapter_archiveskyehawkecom
 from . import adapter_libraryofmoriacom
-from . import adapter_wraithbaitcom
 from . import adapter_ashwindersycophanthexcom
 from . import adapter_chaossycophanthexcom
 from . import adapter_erosnsapphosycophanthexcom
@@ -60,38 +62,26 @@ from . import adapter_occlumencysycophanthexcom
 from . import adapter_phoenixsongnet
 from . import adapter_walkingtheplankorg
 from . import adapter_dokugacom
-from . import adapter_iketernalnet
 from . import adapter_storiesofardacom
-from . import adapter_destinysgatewaycom
 from . import adapter_ncisfictioncom
 from . import adapter_fanfiktionde
-from . import adapter_ponyfictionarchivenet
 from . import adapter_themasquenet
 from . import adapter_pretendercentrecom
 from . import adapter_darksolaceorg
-from . import adapter_finestoriescom
-from . import adapter_hlfictionnet
+from . import adapter_storyroomcom
 from . import adapter_dracoandginnycom
-from . import adapter_scarvesandcoffeenet
 from . import adapter_wolverineandroguecom
-from . import adapter_merlinficdtwinscouk
 from . import adapter_thehookupzonenet
-from . import adapter_bloodtiesfancom
-from . import adapter_qafficcom
 from . import adapter_efpfanficnet
 from . import adapter_imagineeficcom
-from . import adapter_potterheadsanonymouscom
 from . import adapter_storiesonlinenet
-from . import adapter_trekiverseorg
 from . import adapter_literotica
 from . import adapter_voracity2eficcom
 from . import adapter_spikeluvercom
 from . import adapter_bloodshedversecom
 from . import adapter_fictionmaniatv
-from . import adapter_themaplebookshelf
 from . import adapter_sheppardweircom
 from . import adapter_samandjacknet
-from . import adapter_csiforensicscom
 from . import adapter_tgstorytimecom
 from . import adapter_forumsspacebattlescom
 from . import adapter_forumssufficientvelocitycom
@@ -100,8 +90,6 @@ from . import adapter_ninelivesarchivecom
 from . import adapter_masseffect2in
 from . import adapter_quotevcom
 from . import adapter_mcstoriescom
-from . import adapter_buffygilescom
-from . import adapter_andromedawebcom
 from . import adapter_naiceanilmenet
 from . import adapter_adultfanfictionorg
 from . import adapter_fictionhuntcom
@@ -111,60 +99,48 @@ from . import adapter_bdsmlibrarycom
 from . import adapter_asexstoriescom
 from . import adapter_gluttonyfictioncom
 from . import adapter_valentchambercom
-from . import adapter_looselugscom
 from . import adapter_wwwgiantessworldnet
-from . import adapter_lotrgficcom
-from . import adapter_tomparisdormcom
-from . import adapter_sugarquillnet
 from . import adapter_starslibrarynet
 from . import adapter_fanficauthorsnet
 from . import adapter_fireflyfansnet
-from . import adapter_shriftweborgbfa
 from . import adapter_trekfanfictionnet
-from . import adapter_wuxiaworldcom
-from . import adapter_wwwlushstoriescom
 from . import adapter_wwwutopiastoriescom
 from . import adapter_sinfuldreamscomunicornfic
-from . import adapter_sinfuldreamscomwhisperedmuse
 from . import adapter_sinfuldreamscomwickedtemptation
 from . import adapter_asianfanficscom
-from . import adapter_webnovelcom
 from . import adapter_mttjustoncenet
 from . import adapter_narutoficorg
-from . import adapter_starskyhutcharchivenet
-from . import adapter_swordborderlineangelcom
-from . import adapter_tasteofpoisoninkubationnet
 from . import adapter_thedelphicexpansecom
 from . import adapter_wwwaneroticstorycom
 from . import adapter_lcfanficcom
-from . import adapter_noveltrovecom
 from . import adapter_inkbunnynet
 from . import adapter_alternatehistorycom
 from . import adapter_wattpadcom
 from . import adapter_novelonlinefullcom
 from . import adapter_wwwnovelallcom
-from . import adapter_wuxiaworldco
-from . import adapter_novelupdatescc
 from . import adapter_hentaifoundrycom
 from . import adapter_mugglenetfanfictioncom
-from . import adapter_swiorgru
 from . import adapter_fanficsme
 from . import adapter_fanfictalkcom
 from . import adapter_scifistoriescom
-from . import adapter_silmarillionwritersguildorg
 from . import adapter_chireadscom
 from . import adapter_scribblehubcom
 from . import adapter_fictionlive
-from . import adapter_wuxiaworldsite
 from . import adapter_thesietchcom
-from . import adapter_fastnovelnet
 from . import adapter_squidgeworldorg
 from . import adapter_novelfull
-from . import adapter_worldofxde
 from . import adapter_psychficcom
 from . import adapter_deviantartcom
-from . import adapter_merengohu
 from . import adapter_readonlymindcom
+from . import adapter_wwwsunnydaleafterdarkcom
+from . import adapter_syosetucom
+from . import adapter_kakuyomujp
+from . import adapter_fanfictionsfr
+from . import adapter_touchfluffytail
+from . import adapter_spiritfanfictioncom
+from . import adapter_superlove
+from . import adapter_cfaa
+from . import adapter_althistorycom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -248,6 +224,21 @@ def get_section_url(url):
         ## return unchanged in that case.
         return url
 
+def get_url_search(url):
+    '''
+    For adapters that have story URLs that can change.  This is
+    used for searching the Calibre library by identifiers:url for
+    sites (generally) that contain author or title that can
+    change, but also have a unique identifier that doesn't.
+
+    returns a string containing a regexp, not a compiled re object.
+    '''
+    cls =  _get_class_for(url)[0]
+    if not cls:
+        ## still apply common processing.
+        cls = base_adapter.BaseSiteAdapter
+    return cls.get_url_search(url)
+
 def getAdapter(config,url,anyurl=False):
 
     #logger.debug("trying url:"+url)

fanficfare/adapters/adapter_adastrafanficcom.py

@@ -15,201 +15,24 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 from __future__ import absolute_import
 import logging
 logger = logging.getLogger(__name__)
-import re
 
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','aaff')
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.adastrafanfic.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&warning=5"
-        else:
-            addurl=""
-
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
-            raise exceptions.AdultCheckRequired(self.url)
-
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-
-        soup = self.make_soup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
-
-
-        ## <meta name='description' content='&lt;p&gt;Description&lt;/p&gt; ...' >
-        ## Summary, strangely, is in the content attr of a <meta name='description'> tag
-        ## which is escaped HTML.  Unfortunately, we can't use it because they don't
-        ## escape (') chars in the desc, breakin the tag.
-        #meta_desc = soup.find('meta',{'name':'description'})
-        #metasoup = bs.BeautifulStoneSoup(meta_desc['content'])
-        #self.story.setMetadata('description',stripHTML(metasoup))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ''
-                while value and 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
-                svalue=svalue[:svalue.find('<span class="label">')]
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self.get_request(url)
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-
-        soup = self.make_soup(data)
-
-        span = soup.find('div', {'id' : 'story'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,span)
+from .base_otw_adapter import BaseOTWAdapter
 
 def getClass():
-    return AdAstraFanficComSiteAdapter
+    return AdastrafanficComAdapter
+
+class AdastrafanficComAdapter(BaseOTWAdapter):
+
+    def __init__(self, config, url):
+        BaseOTWAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','aaff')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.adastrafanfic.com'

fanficfare/adapters/adapter_adultfanfictionorg.py

@@ -57,8 +57,8 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
         # normalized story URL.(checking self.zone against list
         # removed--it was redundant w/getAcceptDomains and
         # getSiteURLPattern both)
-        self._setURL('http://{0}.{1}/story.php?no={2}'.format(self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
-        #self._setURL('http://' + self.zone + '.' + self.getBaseDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
+        self._setURL('https://{0}.{1}/story.php?no={2}'.format(self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
+        #self._setURL('https://' + self.zone + '.' + self.getBaseDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
 
         # Each adapter needs to have a unique site abbreviation.
         #self.story.setMetadata('siteabbrev',self.getSiteAbbrev())
@@ -68,9 +68,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%m-%d"
-
-
+        self.dateformat = "%B %d, %Y"
 
     ## Added because adult-fanfiction.org does send you to
     ## www.adult-fanfiction.org when you go to it and it also moves
@@ -113,79 +111,31 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return ("http://anime.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://anime2.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://bleach.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://books.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://buffy.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://cartoon.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://celeb.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://comics.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://ff.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://games.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://hp.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://inu.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://lotr.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://manga.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://movies.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://naruto.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://ne.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://original.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://tv.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://xmen.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://ygo.adult-fanfiction.org/story.php?no=123456789 "
-              + "http://yuyu.adult-fanfiction.org/story.php?no=123456789")
+        return ("https://anime.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://anime2.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://bleach.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://books.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://buffy.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://cartoon.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://celeb.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://comics.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://ff.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://games.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://hp.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://inu.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://lotr.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://manga.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://movies.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://naruto.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://ne.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://original.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://tv.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://xmen.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://ygo.adult-fanfiction.org/story.php?no=123456789 "
+              + "https://yuyu.adult-fanfiction.org/story.php?no=123456789")
 
     def getSiteURLPattern(self):
-        return r'http?://(anime|anime2|bleach|books|buffy|cartoon|celeb|comics|ff|games|hp|inu|lotr|manga|movies|naruto|ne|original|tv|xmen|ygo|yuyu)\.adult-fanfiction\.org/story\.php\?no=\d+$'
-
-    ##This is not working right now, so I'm commenting it out, but leaving it for future testing
-    ## Login seems to be reasonably standard across eFiction sites.
-    #def needToLoginCheck(self, data):
-        ##This adapter will always require a login
-    #    return True
-
-#    <form name="login" method="post" action="">
-#      <div class="top">E-mail: <span id="sprytextfield1">
-#        <input name="email" type="text" id="email" size="20" maxlength="255" />
-#        <span class="textfieldRequiredMsg">Email is required.</span><span class="textfieldInvalidFormatMsg">Invalid E-mail.</span></span></div>
-#      <div class="top">Password: <span id="sprytextfield2">
-#        <input name="pass1" type="password" id="pass1" size="20" maxlength="32" />
-#        <span class="textfieldRequiredMsg">password is required.</span><span class="textfieldMinCharsMsg">Minimum 8 characters8.</span><span class="textfieldMaxCharsMsg">Exceeded 32 characters.</span></span></div>
-#      <div class="top"><br /> <input name="loginsubmittop" type="hidden" id="loginsubmit" value="TRUE" />
-#        <input type="submit" value="Login" />
-#      </div>
-#    </form>
-
-
-    ##This is not working right now, so I'm commenting it out, but leaving it for future testing
-    #def performLogin(self, url, soup):
-    #    params = {}
-
-    #    if self.password:
-    #        params['email'] = self.username
-    #        params['pass1'] = self.password
-    #    else:
-    #        params['email'] = self.getConfig("username")
-    #        params['pass1'] = self.getConfig("password")
-    #    params['submit'] = 'Login'
-
-    #    # copy all hidden input tags to pick up appropriate tokens.
-    #    for tag in soup.findAll('input',{'type':'hidden'}):
-    #        params[tag['name']] = tag['value']
-
-    #    logger.debug("Will now login to URL {0} as {1} with password: {2}".format(url, params['email'],params['pass1']))
-
-    #    d = self.post_request(url, params, usecache=False)
-    #    d = self.post_request(url, params, usecache=False)
-    #    soup = self.make_soup(d)
-
-        #if not (soup.find('form', {'name' : 'login'}) == None):
-        #    logger.info("Failed to login to URL %s as %s" % (url, params['email']))
-        #    raise exceptions.FailedToLogin(url,params['email'])
-        #    return False
-        #else:
-    #    return True
+        return r'https?://(anime|anime2|bleach|books|buffy|cartoon|celeb|comics|ff|games|hp|inu|lotr|manga|movies|naruto|ne|original|tv|xmen|ygo|yuyu)\.adult-fanfiction\.org/story\.php\?no=\d+$'
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def doExtractChapterUrlsAndMetadata(self, get_cover=True):
@@ -193,173 +143,97 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
         ## You need to have your is_adult set to true to get this story
         if not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
+        else:
+            d = self.post_request('https://www.adult-fanfiction.org/globals/ajax/age-verify.php', {"verify":"1"})
+            if "Age verified successfully" not in d:
+                raise exceptions.FailedToDownload("Failed to Verify Age: {0}".format(d))
 
         url = self.url
         logger.debug("URL: "+url)
 
         data = self.get_request(url)
+        # logger.debug(data)
 
         if "The dragons running the back end of the site can not seem to find the story you are looking for." in data:
             raise exceptions.StoryDoesNotExist("{0}.{1} says: The dragons running the back end of the site can not seem to find the story you are looking for.".format(self.zone, self.getBaseDomain()))
 
         soup = self.make_soup(data)
 
-        ##This is not working right now, so I'm commenting it out, but leaving it for future testing
-        #self.performLogin(url, soup)
-
-
         ## Title
         ## Some of the titles have a backslash on the story page, but not on the Author's page
         ## So I am removing it from the title, so it can be found on the Author's page further in the code.
         ## Also, some titles may have extra spaces '  ', and the search on the Author's page removes them,
         ## so I have to here as well. I used multiple replaces to make sure, since I did the same below.
-        a = soup.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a).replace('\\','').replace('  ',' ').replace('  ',' ').replace('  ',' ').strip())
+        h1 = soup.find('h1')
+        # logger.debug("Title:%s"%h1)
+        self.story.setMetadata('title',stripHTML(h1).replace('\\','').replace('  ',' ').replace('  ',' ').replace('  ',' ').strip())
 
-        # Find the chapters:
-        chapters = soup.find('div',{'class':'dropdown-content'})
-        for i, chapter in enumerate(chapters.findAll('a')):
-            self.add_chapter(chapter,self.url+'&chapter='+unicode(i+1))
+        # Find the chapters from first list only
+        chapters = soup.select_one('select.chapter-select').select('option')
+        for chapter in chapters:
+            self.add_chapter(chapter,self.url+'&chapter='+chapter['value'])
 
 
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"profile.php\?no=\d+"))
+        a = soup.find('a', href=re.compile(r"profile.php\?id=\d+"))
         if a == None:
             # I know that the original author of fanficfare wants to always have metadata,
             # but I posit that if the story is there, even if we can't get the metadata from the
             # author page, the story should still be able to be downloaded, which is what I've done here.
             self.story.setMetadata('authorId','000000000')
-            self.story.setMetadata('authorUrl','http://www.adult-fanfiction.org')
+            self.story.setMetadata('authorUrl','https://www.adult-fanfiction.org')
             self.story.setMetadata('author','Unknown')
             logger.warning('There was no author found for the story... Metadata will not be retreived.')
-            self.setDescription(url,'>>>>>>>>>> No Summary Given <<<<<<<<<<')
+            self.setDescription(url,'>>>>>>>>>> No Summary Given, Unknown Author <<<<<<<<<<')
         else:
             self.story.setMetadata('authorId',a['href'].split('=')[1])
             self.story.setMetadata('authorUrl',a['href'])
             self.story.setMetadata('author',stripHTML(a))
 
-            ##The story page does not give much Metadata, so we go to the Author's page
+            ## The story page does not give much Metadata, so we go to
+            ## the Author's page.  Except it's actually a sub-req for
+            ## list of author's stories for that subdomain
+            author_Url = 'https://members.{0}/load-user-stories.php?subdomain={1}&uid={2}'.format(
+                self.getBaseDomain(),
+                self.zone,
+                self.story.getMetadata('authorId'))
 
-            ##Get the first Author page to see if there are multiple pages.
-            ##AFF doesn't care if the page number is larger than the actual pages,
-            ##it will continue to show the last page even if the variable is larger than the actual page
-            author_Url = '{0}&view=story&zone={1}&page=1'.format(self.story.getMetadata('authorUrl'), self.zone)
-            #author_Url = self.story.getMetadata('authorUrl')+'&view=story&zone='+self.zone+'&page=1'
-
-            ##I'm resetting the author page to the zone for this story
-            self.story.setMetadata('authorUrl',author_Url)
-
-            logger.debug('Getting the author page: {0}'.format(author_Url))
+            logger.debug('Getting the load-user-stories page: {0}'.format(author_Url))
             adata = self.get_request(author_Url)
 
-            if "The member you are looking for does not exist." in adata:
-                raise exceptions.StoryDoesNotExist("{0}.{1} says: The member you are looking for does not exist.".format(self.zone, self.getBaseDomain()))
-                #raise exceptions.StoryDoesNotExist(self.zone+'.'+self.getBaseDomain() +" says: The member you are looking for does not exist.")
+            none_found = "No stories found in this category."
+            if none_found in adata:
+                raise exceptions.StoryDoesNotExist("{0}.{1} says: {2}".format(self.zone, self.getBaseDomain(), none_found))
 
             asoup = self.make_soup(adata)
+            # logger.debug(asoup)
 
-            ##Getting the number of pages
-            pages=asoup.find('div',{'class' : 'pagination'}).findAll('li')[-1].find('a')
-            if not pages == None:
-                pages = pages['href'].split('=')[-1]
-            else:
-                pages = 0
+            story_card = asoup.select_one('div.story-card:has(a[href="{0}"])'.format(url))
+            # logger.debug(story_card)
 
-            ##If there is only 1 page of stories, check it to get the Metadata,
-            if pages == 0:
-                a = asoup.findAll('li')
-                for lc2 in a:
-                    if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
-                        break
-            ## otherwise go through the pages
-            else:
-                page=1
-                i=0
-                while i == 0:
-                    ##We already have the first page, so if this is the first time through, skip getting the page
-                    if page != 1:
-                        author_Url = '{0}&view=story&zone={1}&page={2}'.format(self.story.getMetadata('authorUrl'), self.zone, unicode(page))
-                        logger.debug('Getting the author page: {0}'.format(author_Url))
-                        adata = self.get_request(author_Url)
-                        ##This will probably never be needed, since AFF doesn't seem to care what number you put as
-                        ## the page number, it will default to the last page, even if you use 1000, for an author
-                        ## that only hase 5 pages of stories, but I'm keeping it in to appease Saint Justin Case (just in case).
-                        if "The member you are looking for does not exist." in adata:
-                            raise exceptions.StoryDoesNotExist("{0}.{1} says: The member you are looking for does not exist.".format(self.zone, self.getBaseDomain()))
-                    # we look for the li element that has the story here
-                    asoup = self.make_soup(adata)
+            ## Category
+            ## I've only seen one category per story so far, but just in case:
+            for cat in story_card.select('div.story-card-category'):
+                # remove Category:, old code suggests Located: is also
+                # possible, so removing by <strong>
+                cat.find("strong").decompose()
+                self.story.addToList('category',stripHTML(cat))
 
-                    a = asoup.findAll('li')
-                    for lc2 in a:
-                        if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
-                            i=1
-                            break
-                    page = page + 1
-                    if page > int(pages):
-                        break
+            self.setDescription(url,story_card.select_one('div.story-card-description'))
 
-            ##Split the Metadata up into a list
-            ##We have to change the soup type to a string, then remove the newlines, and double spaces,
-            ##then changes the <br/> to '-:-', which seperates the different elemeents.
-            ##Then we strip the HTML elements from the string.
-            ##There is also a double <br/>, so we have to fix that, then remove the leading and trailing '-:-'.
-            ##They are always in the same order.
-            ## EDIT 09/26/2016: Had some trouble with unicode errors... so I had to put in the decode/encode parts to fix it
-            liMetadata = unicode(lc2).replace('\n','').replace('\r','').replace('\t',' ').replace('  ',' ').replace('  ',' ').replace('  ',' ')
-            liMetadata = stripHTML(liMetadata.replace(r'<br/>','-:-').replace('<!-- <br /-->','-:-'))
-            liMetadata = liMetadata.strip('-:-').strip('-:-').encode('utf-8')
-            for i, value in enumerate(liMetadata.decode('utf-8').split('-:-')):
-                if i == 0:
-                    # The value for the title has been manipulated, so may not be the same as gotten at the start.
-                    # I'm going to use the href from the lc2 retrieved from the author's page to determine if it is correct.
-                    if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))['href'] != url:
-                        raise exceptions.StoryDoesNotExist('Did not find story in author story list: {0}'.format(author_Url))
-                elif i == 1:
-                    ##Get the description
-                    self.setDescription(url,stripHTML(value.strip()))
-                else:
-                    # the rest of the values can be missing, so instead of hardcoding the numbers, we search for them.
-                    if 'Located :' in value:
-                        self.story.setMetadata('category',value.replace(r'&gt;',r'>').replace(r'Located :',r'').strip())
-                    elif 'Category :' in value:
-                        # Get the Category
-                        self.story.setMetadata('category',value.replace(r'&gt;',r'>').replace(r'Located :',r'').strip())
-                    elif 'Content Tags :' in value:
-                        # Get the Erotic Tags
-                        value = stripHTML(value.replace(r'Content Tags :',r'')).strip()
-                        for code in re.split(r'\s',value):
-                            self.story.addToList('eroticatags',code)
-                    elif 'Posted :' in value:
-                        # Get the Posted Date
-                        value = value.replace(r'Posted :',r'').strip()
-                        if value.startswith('008'):
-                            # It is unknown how the 200 became 008, but I'm going to change it back here
-                            value = value.replace('008','200')
-                        elif value.startswith('0000'):
-                            # Since the date is showing as 0000,
-                            # I'm going to put the memberdate here
-                            value = asoup.find('div',{'id':'contentdata'}).find('p').get_text(strip=True).replace('Member Since','').strip()
-                        self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-                    elif 'Edited :' in value:
-                        # Get the 'Updated' Edited date
-                        # AFF has the time for the Updated date, and we only want the date,
-                        # so we take the first 10 characters only
-                        value = value.replace(r'Edited :',r'').strip()[0:10]
-                        if value.startswith('008'):
-                            # It is unknown how the 200 became 008, but I'm going to change it back here
-                            value = value.replace('008','200')
-                            self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                        elif value.startswith('0000') or '-00-' in value:
-                            # Since the date is showing as 0000,
-                            # or there is -00- in the date,
-                            # I'm going to put the Published date here
-                            self.story.setMetadata('dateUpdated', self.story.getMetadata('datPublished'))
-                        else:
-                            self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                    else:
-                        # This catches the blank elements, and the Review and Dragon Prints.
-                        # I am not interested in these, so do nothing
-                        zzzzzzz=0
+            for tag in story_card.select('span.story-tag'):
+                self.story.addToList('eroticatags',stripHTML(tag))
+
+            ## created/updates share formatting
+            for meta in story_card.select('div.story-card-meta-item span:last-child'):
+                meta = stripHTML(meta)
+                if 'Created: ' in meta:
+                    meta = meta.replace('Created: ','')
+                    self.story.setMetadata('datePublished', makeDate(meta, self.dateformat))
+
+                if 'Updated: ' in meta:
+                    meta = meta.replace('Updated: ','')
+                    self.story.setMetadata('dateUpdated', makeDate(meta, self.dateformat))
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):
@@ -367,10 +241,11 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         soup = self.make_soup(self.get_request(url))
-        chaptertag = soup.find('div',{'class' : 'pagination'}).parent.findNext('td')
+        chaptertag = soup.select_one('div.chapter-body')
         if None == chaptertag:
             raise exceptions.FailedToDownload("Error downloading Chapter: {0}!  Missing required element!".format(url))
-        # Change td to a div.
-        chaptertag.name='div'
+        ## chapter text includes a copy of story title, author,
+        ## chapter title, & eroticatags specific to the chapter.  Did
+        ## before, too.
 
         return self.utf8FromSoup(url,chaptertag)

fanficfare/adapters/adapter_althistorycom.py

@@ -0,0 +1,40 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2026 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+import re
+
+from .base_xenforo2forum_adapter import BaseXenForo2ForumAdapter
+
+def getClass():
+    return AltHistoryComAdapter
+
+## NOTE: This is a different site than www.alternatehistory.com.
+
+class AltHistoryComAdapter(BaseXenForo2ForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForo2ForumAdapter.__init__(self, config, url)
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ahc')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'althistory.com'
+

fanficfare/adapters/adapter_andromedawebcom.py

@@ -1,280 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ####### Not all lables are captured. they are not formtted correctly on the
-# #######  webpage.
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return AndromedaWebComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class AndromedaWebComAdapter(BaseSiteAdapter):  # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        # XXX Most sites don't have the /fiction part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','awc') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y" # XXX
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.andromeda-web.com'  # XXX
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self.post_request(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=2"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self.get_request(url)
-
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-
-        # fiction/viewstory.php?sid=1882&warning=4
-        # fiction/viewstory.php?sid=1654&ageconsent=ok&warning=2
-        #print data
-        m = re.search(r"'fiction/viewstory.php\?sid=10(&warning=2)'",data)
-        m = re.search(r"'fiction/viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                data = self.get_request(url)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        pagetitle = soup.find('div',{'id':'content'})
-
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl)
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"fiction/viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^fiction/viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('fiction/viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'class' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_archiveofourownorg.py

@@ -18,55 +18,20 @@
 from __future__ import absolute_import
 import logging
 logger = logging.getLogger(__name__)
-import re
-import json
 
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-
-from .base_adapter import BaseSiteAdapter,  makeDate
+from .base_otw_adapter import BaseOTWAdapter
 
 def getClass():
     return ArchiveOfOurOwnOrgAdapter
 
-class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
+class ArchiveOfOurOwnOrgAdapter(BaseOTWAdapter):
 
     def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        self.addurl = ""
-
-        self.full_work_soup = None
-        self.full_work_chapters = None
-        self.use_full_work_soup = True
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-
-            # normalized story URL.
-            self._setURL('https://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
+        BaseOTWAdapter.__init__(self, config, url)
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ao3')
 
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%b-%d"
-
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -84,532 +49,21 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         return ['archiveofourown.org',
                 'archiveofourown.com',
                 'archiveofourown.net',
+                'archiveofourown.gay',
                 'download.archiveofourown.org',
                 'download.archiveofourown.com',
                 'download.archiveofourown.net',
                 'ao3.org',
                 ]
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/works/123456 https://"+cls.getSiteDomain()+"/collections/Some_Archive/works/123456 https://"+cls.getSiteDomain()+"/works/123456/chapters/78901"
-
-    def getSiteURLPattern(self):
-        # https://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        # logger.debug(r"https?://" + r"|".join([x.replace('.','\.') for x in self.getAcceptDomains()]) + r"(/collections/[^/]+)?/works/0*(?P<id>\d+)")
-        return r"https?://(" + r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()]) + r")(/collections/[^/]+)?/works/0*(?P<id>\d+)"
-
-    @classmethod
-    def get_section_url(cls,url):
-        ## minimal URL used for section names in INI and reject list
-        ## for comparison
-        # logger.debug("pre--url:%s"%url)
-        ## https://archiveofourown.org/works/19334905/chapters/71697933
-        # http://archiveofourown.org/works/34686793/chapters/89043733
-        url = re.sub(r'^https?://(.*/works/\d+).*$',r'https://\1',url)
-        # logger.debug("post-url:%s"%url)
+    def mod_url_request(self, url):
         return url
 
-    ## Login
-    def needToLoginCheck(self, data):
-        if 'This work is only available to registered users of the Archive.' in data \
-                or "The password or user name you entered doesn't match our records" in data:
-            return True
+    def mod_url_request(self, url):
+        ## add / to *not* replace media.archiveofourown.org
+        if self.getConfig("use_archive_transformativeworks_org",False):
+            return url.replace("/archiveofourown.org","/archive.transformativeworks.org")
+        elif self.getConfig("use_archiveofourown_gay",False):
+            return url.replace("/archiveofourown.org","/archiveofourown.gay")
         else:
-            return False
-
-    def performLogin(self, url, data):
-
-        params = {}
-        if self.password:
-            params['user[login]'] = self.username
-            params['user[password]'] = self.password
-        else:
-            params['user[login]'] = self.getConfig("username")
-            params['user[password]'] = self.getConfig("password")
-        params['user[remember_me]'] = '1'
-        params['commit'] = 'Log in'
-        params['utf8'] = u'\x2713' # utf8 *is* required now.  hex code works better than actual character for some reason. u'✓'
-
-        # authenticity_token now comes from a completely separate json call.
-        token_json = json.loads(self.get_request('https://' + self.getSiteDomain() + "/token_dispenser.json"))
-        params['authenticity_token'] = token_json['token']
-
-        loginUrl = 'https://' + self.getSiteDomain() + '/users/login'
-        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                            params['user[login]']))
-
-        d = self.post_request(loginUrl, params)
-
-        if 'href="/users/logout"' not in d :
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['user[login]']))
-            raise exceptions.FailedToLogin(url,params['user[login]'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            self.addurl = "?view_adult=true"
-        else:
-            self.addurl=""
-
-        metaurl = self.url+self.addurl
-        url = self.url+'/navigate'+self.addurl
-        logger.info("url: "+url)
-        logger.info("metaurl: "+metaurl)
-
-        data = self.get_request(url)
-        meta = self.get_request(metaurl)
-
-        if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
-            if self.addurl:
-                ## "?view_adult=true" doesn't work on base story
-                ## URL anymore, which means we have to
-                metasoup = self.make_soup(meta)
-                a = metasoup.find('a',text='Proceed')
-                metaurl = 'https://'+self.host+a['href']
-                meta = self.get_request(metaurl)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        # need to log in for this one, or always_login.
-        if self.needToLoginCheck(data) or \
-                ( self.getConfig("always_login") and 'href="/users/logout"' not in data ):
-            self.performLogin(url,data)
-            data = self.get_request(url,usecache=False)
-            meta = self.get_request(metaurl,usecache=False)
-
-        soup = self.make_soup(data)
-        for tag in soup.findAll('div',id='admin-banner'):
-            tag.extract()
-        metasoup = self.make_soup(meta)
-        for tag in metasoup.findAll('div',id='admin-banner'):
-            tag.extract()
-
-
-        ## Title
-        a = soup.find('a', href=re.compile(r"/works/\d+$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        if self.getConfig("always_login"):
-            # deliberately using always_login instead of checking for
-            # actual login so we don't have a case where these show up
-            # for a user only when they get user-restricted stories.
-            try:
-                # is bookmarked if has update /bookmarks/ form --
-                # create bookmark form uses different url
-                self.story.setMetadata('bookmarked',
-                                       None != metasoup.find('form',action=re.compile(r'^/bookmarks/')))
-                self.story.extendList('bookmarktags',
-                                      metasoup.find('input',id='bookmark_tag_string')['value'].split(', '))
-                self.story.setMetadata('bookmarkprivate',
-                                       metasoup.find('input',id='bookmark_private').has_attr('checked'))
-                self.story.setMetadata('bookmarkrec',
-                                       metasoup.find('input',id='bookmark_rec').has_attr('checked'))
-            except KeyError:
-                pass
-            self.story.setMetadata('bookmarksummary',
-                                   stripHTML(metasoup.find('textarea',id='bookmark_notes')))
-
-        if metasoup.find('img',alt='(Restricted)'):
-            self.story.setMetadata('restricted','Restricted')
-
-        # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/.+"))
-        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
-            self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl','https://' + self.getSiteDomain() + '/')
-            self.story.setMetadata('authorId','0')
-        else:
-            for a in alist:
-                self.story.addToList('authorId',a['href'].split('/')[-1])
-                self.story.addToList('authorUrl','https://'+self.host+a['href'])
-                self.story.addToList('author',a.text)
-
-        byline = metasoup.find('h3',{'class':'byline'})
-        if byline:
-            self.story.setMetadata('byline',stripHTML(byline))
-
-        # byline:
-        # <h3 class="byline heading">
-        # Hope Roy [archived by <a href="/users/ssa_archivist/pseuds/ssa_archivist" rel="author">ssa_archivist</a>]
-        # </h3>
-        # stripped:"Hope Roy [archived by ssa_archivist]"
-        m = re.match(r'(?P<author>.*) \[archived by ?(?P<archivist>.*)\]',stripHTML(byline))
-        if( m and
-            len(alist) == 1 and
-            self.getConfig('use_archived_author') ):
-            self.story.setMetadata('author',m.group('author'))
-
-        newestChapter = None
-        self.newestChapterNum = None # save for comparing during update.
-        # Scan all chapters to find the oldest and newest, on AO3 it's
-        # possible for authors to insert new chapters out-of-order or
-        # change the dates of earlier ones by editing them--That WILL
-        # break epub update.
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+r"/chapters/\d+$"))
-        self.story.setMetadata('numChapters',len(chapters))
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        if len(chapters)==1:
-            self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+chapters[0]['href'])
-        else:
-            for index, chapter in enumerate(chapters):
-                # strip just in case there's tags, like <i> in chapter titles.
-                # (2013-09-21)
-                date = stripHTML(chapter.findNext('span'))[1:-1]
-                chapterDate = makeDate(date,self.dateformat)
-                self.add_chapter(chapter,'https://'+self.host+chapter['href'],
-                                 {'date':chapterDate.strftime(self.getConfig("datechapter_format",self.getConfig("datePublished_format","%Y-%m-%d")))})
-                if newestChapter == None or chapterDate > newestChapter:
-                    newestChapter = chapterDate
-                    self.newestChapterNum = index
-
-        a = metasoup.find('blockquote',{'class':'userstuff'})
-        if a != None:
-            a.name='div' # Change blockquote to div.
-            self.setDescription(url,a)
-            #self.story.setMetadata('description',a.text)
-
-        a = metasoup.find('dd',{'class':"rating tags"})
-        if a != None:
-            self.story.setMetadata('rating',stripHTML(a.text))
-
-        d = metasoup.find('dd',{'class':"language"})
-        if d != None:
-            self.story.setMetadata('language',stripHTML(d.text))
-
-        a = metasoup.find('dd',{'class':"fandom tags"})
-        if a != None:
-            fandoms = a.findAll('a',{'class':"tag"})
-            for fandom in fandoms:
-                self.story.addToList('fandoms',fandom.string)
-
-        a = metasoup.find('dd',{'class':"warning tags"})
-        if a != None:
-            warnings = a.findAll('a',{'class':"tag"})
-            for warning in warnings:
-                self.story.addToList('warnings',warning.string)
-
-        a = metasoup.find('dd',{'class':"freeform tags"})
-        if a != None:
-            genres = a.findAll('a',{'class':"tag"})
-            for genre in genres:
-                self.story.addToList('freeformtags',genre.string)
-
-        a = metasoup.find('dd',{'class':"category tags"})
-        if a != None:
-            genres = a.findAll('a',{'class':"tag"})
-            for genre in genres:
-                if genre != "Gen":
-                    self.story.addToList('ao3categories',genre.string)
-
-        a = metasoup.find('dd',{'class':"character tags"})
-        if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
-
-        a = metasoup.find('dd',{'class':"relationship tags"})
-        if a != None:
-            ships = a.findAll('a',{'class':"tag"})
-            for ship in ships:
-                self.story.addToList('ships',ship.string)
-
-        a = metasoup.find('dd',{'class':"collections"})
-        if a != None:
-            collections = a.findAll('a')
-            for collection in collections:
-                self.story.addToList('collections',collection.string)
-
-        stats = metasoup.find('dl',{'class':'stats'})
-        dt = stats.findAll('dt')
-        dd = stats.findAll('dd')
-        for x in range(0,len(dt)):
-            label = dt[x].text
-            value = dd[x].text
-
-            if 'Words:' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Comments:' in label:
-                self.story.setMetadata('comments', value)
-
-            if 'Kudos:' in label:
-                self.story.setMetadata('kudos', value)
-
-            if 'Hits:' in label:
-                self.story.setMetadata('hits', value)
-
-            if 'Bookmarks:' in label:
-                self.story.setMetadata('bookmarks', value)
-
-            if 'Chapters:' in label:
-                self.story.setMetadata('chapterslashtotal', value)
-                if value.split('/')[0] == value.split('/')[1]:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Completed' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-
-        # Find Series name from series URL.
-        ddseries = metasoup.find('dd',{'class':"series"})
-
-        if ddseries:
-            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
-                series_name = stripHTML(a)
-                series_url = 'https://'+self.host+a['href']
-                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
-                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
-                self.story.setMetadata('series%02dUrl'%i,series_url)
-                if i == 0:
-                    self.setSeries(series_name, series_index)
-                    self.story.setMetadata('seriesUrl',series_url)
-
-    def hookForUpdates(self,chaptercount):
-        if self.newestChapterNum and self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
-            logger.info("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
-            self.oldchapters = self.oldchapters[:self.newestChapterNum]
-        return len(self.oldchapters)
-
-    ## Normalize chapter URLs because a) site has changed from http to
-    ## https and b) in case of title change.  That way updates to
-    ## existing stories don't re-download all chapters.
-    def normalize_chapterurl(self,url):
-        url = re.sub(r"https?://("+self.getSiteDomain()+r"/works/\d+/chapters/\d+)(\?view_adult=true)?$",
-                     r"https://\1",url)
-        return url
-
-    # grab the text for an individual chapter.
-    def getChapterTextNum(self, url, index):
-        ## FYI: Chapter urls used to include ?view_adult=true in each
-        ## one.  With cookiejar being passed now, that's not
-        ## necessary.  However, there is a corner case with plugin--If
-        ## a user-required story is attempted after gathering metadata
-        ## for one that needs adult, but not user AND the user doesn't
-        ## enter a valid user, the is_adult cookie from before can be
-        ## lost.
-        logger.debug('Getting chapter text for: %s index: %s' % (url,index))
-
-        save_chapter_soup = self.make_soup('<div class="story"></div>')
-        ## use the div because the full soup will also have <html><body>.
-        ## need save_chapter_soup for .new_tag()
-        save_chapter=save_chapter_soup.find('div')
-
-        whole_dl_soup = chapter_dl_soup = None
-
-        if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and self.num_chapters() > 1:
-            logger.debug("USE view_full_work")
-            ## Assumed view_adult=true was cookied during metadata
-            if not self.full_work_soup:
-                self.full_work_soup = self.make_soup(self.get_request(self.url+"?view_full_work=true"+self.addurl.replace('?','&')))
-                ## AO3 has had several cases now where chapter numbers
-                ## are missing, breaking the link between
-                ## <div id=chapter-##> and Chapter ##.
-                ## But they should all still be there and in the right
-                ## order, so array[index]
-                self.full_work_chapters = self.full_work_soup.find_all('div',{'id':re.compile(r'chapter-\d+')})
-                if len(self.full_work_chapters) != self.num_chapters():
-                    ## sanity check just in case.
-                    self.use_full_work_soup = False
-                    self.full_work_soup = None
-                    logger.warning("chapter count in view_full_work(%s) disagrees with num of chapters(%s)--ending use_view_full_work"%(len(self.full_work_chapters),self.num_chapters()))
-            whole_dl_soup = self.full_work_soup
-
-        if whole_dl_soup:
-            chapter_dl_soup = self.full_work_chapters[index]
-        else:
-            whole_dl_soup = chapter_dl_soup = self.make_soup(self.get_request(url+self.addurl))
-            if None == chapter_dl_soup:
-                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        exclude_notes=self.getConfigList('exclude_notes')
-
-        def append_tag(elem,tag,string=None,classes=None):
-            '''bs4 requires tags be added separately.'''
-            new_tag = save_chapter_soup.new_tag(tag)
-            if string:
-                new_tag.string=string
-            if classes:
-                new_tag['class']=[classes]
-            elem.append(new_tag)
-            return new_tag
-
-        ## These are the over-all work's 'Notes at the beginning'.
-        ## They only appear on the first chapter in individual chapter
-        ## pages and before chapter-1 div.  Appending removes
-        ## headnotes from whole_dl_soup, so be sure to only do it on
-        ## the first chapter.
-        head_notes_div = append_tag(save_chapter,'div',classes="fff_chapter_notes fff_head_notes")
-        if 'authorheadnotes' not in exclude_notes and index == 0:
-            headnotes = whole_dl_soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-            if headnotes != None:
-                ## Also include ul class='associations'.
-                ulassoc = headnotes.find('ul', {'class' : "associations"})
-                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
-                if headnotes != None or ulassoc != None:
-                    append_tag(head_notes_div,'b',"Author's Note:")
-                if ulassoc != None:
-                    # fix relative links--all examples so far have been.
-                    for alink in ulassoc.find_all('a'):
-                        if 'http' not in alink['href']:
-                            alink['href']='https://' + self.getSiteDomain() + alink['href']
-                    head_notes_div.append(ulassoc)
-                if headnotes != None:
-                    head_notes_div.append(headnotes)
-
-        ## Can appear on every chapter
-        if 'chaptersummary' not in exclude_notes:
-            chapsumm = chapter_dl_soup.find('div', {'id' : "summary"})
-            if chapsumm != None:
-                chapsumm = chapsumm.find('blockquote')
-                append_tag(head_notes_div,'b',"Summary for the Chapter:")
-                head_notes_div.append(chapsumm)
-
-        ## Can appear on every chapter
-        if 'chapterheadnotes' not in exclude_notes:
-            chapnotes = chapter_dl_soup.find('div', {'id' : "notes"})
-            if chapnotes != None:
-                chapnotes = chapnotes.find('blockquote')
-                if chapnotes != None:
-                    append_tag(head_notes_div,'b',"Notes for the Chapter:")
-                    head_notes_div.append(chapnotes)
-
-        text = chapter_dl_soup.find('div', {'class' : "userstuff module"})
-        chtext = text.find('h3', {'class' : "landmark heading"})
-        if chtext:
-            chtext.extract()
-        save_chapter.append(text)
-
-        foot_notes_div = append_tag(save_chapter,'div',classes="fff_chapter_notes fff_foot_notes")
-        ## Can appear on every chapter
-        if 'chapterfootnotes' not in exclude_notes:
-            chapfoot = chapter_dl_soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-            if chapfoot != None:
-                chapfoot = chapfoot.find('blockquote')
-                append_tag(foot_notes_div,'b',"Notes for the Chapter:")
-                foot_notes_div.append(chapfoot)
-
-        skip_on_update_tags = []
-        ## These are the over-all work's 'Notes at the end'.
-        ## They only appear on the last chapter in individual chapter
-        ## pages and after chapter-# div.  Appending removes
-        ## headnotes from whole_dl_soup, so be sure to only do it on
-        ## the last chapter.
-        if 'authorfootnotes' not in exclude_notes and index+1 == self.num_chapters():
-            footnotes = whole_dl_soup.find('div', {'id' : "work_endnotes"})
-            if footnotes != None:
-                footnotes = footnotes.find('blockquote')
-                if footnotes:
-                    b = append_tag(foot_notes_div,'b',"Author's Note:")
-                    skip_on_update_tags.append(b)
-                    skip_on_update_tags.append(footnotes)
-                    foot_notes_div.append(footnotes)
-
-        ## It looks like 'Inspired by' links now all appear in the ul
-        ## class=associations tag in authorheadnotes.  This code is
-        ## left in case I'm wrong and there are still stories with div
-        ## id=children inspired links at the end.
-        if 'inspiredlinks' not in exclude_notes and index+1 == self.num_chapters():
-            inspiredlinks = whole_dl_soup.find('div', {'id' : "children"})
-            if inspiredlinks != None:
-                if inspiredlinks:
-                    inspiredlinks.find('h3').name='b' # don't want a big h3 at the end.
-                    # fix relative links--all examples so far have been.
-                    for alink in inspiredlinks.find_all('a'):
-                        if 'http' not in alink['href']:
-                            alink['href']='https://' + self.getSiteDomain() + alink['href']
-                    skip_on_update_tags.append(inspiredlinks)
-                    foot_notes_div.append(inspiredlinks)
-
-        ## remove empty head/food notes div(s)
-        if not head_notes_div.find(True):
-            head_notes_div.extract()
-        if not foot_notes_div.find(True):
-            foot_notes_div.extract()
-        ## AO3 story end notes end up in the 'last' chapter, but if
-        ## updated, then there's a new 'last' chapter.  This option
-        ## applies the 'skip_on_ffdl_update' class to those tags which
-        ## means they will be removed during epub reading for update.
-        ## Results: only the last chapter will have end notes.
-        ## Side-effect: An 'Update Always' that doesn't add a new
-        ## lasts chapter will remove the end notes.
-        if self.getConfig("remove_authorfootnotes_on_update"):
-            for skip_tag in skip_on_update_tags:
-                if skip_tag.has_attr('class'):
-                    skip_tag['class'].append('skip_on_ffdl_update')
-                else:
-                    skip_tag['class']=['skip_on_ffdl_update']
-                # logger.debug(skip_tag)
-
-        return self.utf8FromSoup(url,save_chapter)
-
-    def before_get_urls_from_page(self,url,normalize):
-        # special stuff to log into archiveofourown.org, if possible.
-        # Unlike most that show the links to 'adult' stories, but protect
-        # them, AO3 doesn't even show them if not logged in.  Only works
-        # with saved user/pass--not going to prompt for list.
-        if self.getConfig("username"):
-            if self.getConfig("is_adult"):
-                if '?' in url:
-                    addurl = "&view_adult=true"
-                else:
-                    addurl = "?view_adult=true"
-            else:
-                addurl=""
-            # just to get an authenticity_token.
-            data = self.get_request(url+addurl)
-            # login the session.
-            self.performLogin(url,data)
-            # get the list page with logged in session.
-
-    def get_series_from_page(self,url,data,normalize=False):
-        '''
-        This method is to make it easier for adapters to detect a
-        series URL, pick out the series metadata and list of storyUrls
-        to return without needing to override get_urls_from_page
-        entirely.
-        '''
-
-        if 'This work is only available to registered users of the Archive' in data:
-            raise exceptions.FailedToDownload("This work is only available to registered users of the Archive -- set username/password in personal.ini under [archiveofourown.org]")
-        ## easiest way to get all the weird URL possibilities and stay
-        ## up to date with future changes.
-        m = re.match(self.getSiteURLPattern().replace('/works/','/series/'),url)
-        if m:
-            soup = self.make_soup(data)
-            retval = {}
-            retval['urllist']=[ 'https://'+self.host+a['href'] for a in soup.select('h4.heading a:first-child') ]
-            retval['name']=stripHTML(soup.select_one("h2.heading"))
-            desc=soup.select_one("div.wrapper dd blockquote.userstuff")
-            if desc:
-                desc.name='div' # change blockquote to div to match stories.
-                retval['desc']=desc
-            stats=stripHTML(soup.select_one("dl.series dl.stats"))
-            if 'Complete:Yes' in stats:
-                retval['status'] = "Completed"
-            elif 'Complete:No' in stats:
-                retval['status'] = "In-Progress"
-            return retval
-        ## return dict with at least {'urllist':['storyUrl','storyUrl',...]}
-        ## optionally 'name' and 'desc'?
-        return {}
+            return url

fanficfare/adapters/adapter_archiveskyehawkecom.py

@@ -1,174 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return ArchiveSkyeHawkeComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ash')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%m-%d"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'archive.skyehawke.com'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['archive.skyehawke.com','www.skyehawke.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
-
-    def getSiteURLPattern(self):
-        return r"https?://(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title
-        a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=stripHTML(a).split('"')[1]
-        self.story.setMetadata('title',title)
-
-        # Find authorid and URL from... author url.
-        author = a.find('a')
-        self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+author['href'])
-        self.story.setMetadata('author',author.string)
-
-        authorSoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
-
-        chapter=soup.find('select',{'name':'chapter'}).findAll('option')
-
-        for i in range(1,len(chapter)):
-            ch=chapter[i]
-            self.add_chapter(ch,ch['value'])
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        box=soup.find('div', {'class': "container borderridge"})
-        sum=box.find('span').text
-        self.setDescription(url,sum)
-
-        boxes=soup.findAll('div', {'class': "container bordersolid"})
-        for box in boxes:
-            if box.find('b') != None and box.find('b').text == "History and Story Information":
-
-                for b in box.findAll('b'):
-                    if "words" in b.nextSibling:
-                        self.story.setMetadata('numWords', b.text)
-                    if "archived" in b.previousSibling:
-                        self.story.setMetadata('datePublished', makeDate(stripHTML(b.text), self.dateformat))
-                    if "updated" in b.previousSibling:
-                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(b.text), self.dateformat))
-                    if "fandom" in b.nextSibling:
-                        self.story.addToList('category', b.text)
-
-                for br in box.findAll('br'):
-                    br.replaceWith('split')
-                genre=box.text.split("Genre:")[1].split("split")[0]
-                if not "Unspecified" in genre:
-                    self.story.addToList('genre',genre)
-
-
-            if box.find('span') != None and box.find('span').text == "WARNING":
-
-                rating=box.findAll('span')[1]
-                rating.find('br').replaceWith('split')
-                rating=rating.text.replace("This story is rated",'').split('split')[0]
-                self.story.setMetadata('rating',rating)
-                logger.debug(self.story.getMetadata('rating'))
-
-                warnings=box.find('ol')
-                if warnings != None:
-                    warnings=warnings.text.replace(']', '').replace('[', '').split('  ')
-                    for warning in warnings:
-                        self.story.addToList('warnings',warning)
-
-
-        for asoup in authorSoup.findAll('div', {'class':"story bordersolid"}):
-            if asoup.find('a')['href'] == 'story.php?no='+self.story.getMetadata('storyId'):
-                if '[ Completed ]' in asoup.text:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-                chars=asoup.findNext('div').text.split('Characters')[1].split(']')[0]
-                for char in chars.split(','):
-                    if not "None" in char:
-                        self.story.addToList('characters',char)
-                break
-
-
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div',{'class':"chapter bordersolid"}).findNext('div').findNext('div')
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_asexstoriescom.py

@@ -79,7 +79,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
         data1 = self.get_request(self.url)
         soup1 = self.make_soup(data1)
         #strip comments from soup
-        [comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
+        [comment.extract() for comment in soup1.find_all(string=lambda text:isinstance(text, Comment))]
 
         if 'Page Not Found.' in data1:
             raise exceptions.StoryDoesNotExist(self.url)
@@ -92,7 +92,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('title', title.string)
 
         # Author
-        author = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl'})[1].find('a')
+        author = soup1.find('div',{'class':'story-info'}).find_all('div',{'class':'story-info-bl'})[1].find('a')
         authorurl = author['href']
         self.story.setMetadata('author', author.string)
         self.story.setMetadata('authorUrl', authorurl)
@@ -112,7 +112,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
         ### add it before the rest of the pages, if any
         self.add_chapter('1', self.url)
 
-        chapterTable = soup1.find('div',{'class':'pages'}).findAll('a')
+        chapterTable = soup1.find('div',{'class':'pages'}).find_all('a')
 
         if chapterTable is not None:
             # Multi-chapter story
@@ -124,7 +124,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
                     self.add_chapter(chapterTitle, chapterUrl)
 
 
-        rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
+        rated = soup1.find('div',{'class':'story-info'}).find_all('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
         self.story.setMetadata('rating',rated)
         
         self.story.setMetadata('dateUpdated', makeDate('01/01/2001', '%m/%d/%Y'))

fanficfare/adapters/adapter_ashwindersycophanthexcom.py

@@ -48,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
 
 
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','asph')
@@ -64,10 +64,10 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -92,7 +92,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         params['intent'] = ''
         params['submit'] = 'Submit'
 
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
+        loginUrl = 'https://' + self.getSiteDomain() + '/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
 
@@ -130,20 +130,20 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
         asoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
 
         try:
             # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
+            a = soup.find_all('option')[1]['value']
             self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
+            url = 'https://'+self.host+'/'+a
             soup = self.make_soup(self.get_request(url))
         except:
             pass
 
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
+        for info in asoup.find_all('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
                 self.story.setMetadata('title',stripHTML(a))
@@ -151,13 +151,13 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
 
 
         # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        chapters=soup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
         if len(chapters) == 0:
             self.add_chapter(self.story.getMetadata('title'),url)
         else:
             for chapter in chapters:
                 # just in case there's tags, like <i> in chapter titles.
-                self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
+                self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href'])
 
 
         # eFiction sites don't help us out a lot with their meta data
@@ -170,7 +170,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
             except:
                 return ""
 
-        cats = info.findAll('a',href=re.compile('categories.php'))
+        cats = info.find_all('a',href=re.compile('categories.php'))
         for cat in cats:
             self.story.addToList('category',cat.string)
 
@@ -188,7 +188,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         ## <td><span class="sb"><b>Published:</b> 04/08/2007</td>
 
         ## one story had <b>Updated...</b> in the description. Restrict to sub-table
-        labels = info.find('table').findAll('b')
+        labels = info.find('table').find_all('b')
         for labelspan in labels:
             value = labelspan.nextSibling
             label = stripHTML(labelspan)

fanficfare/adapters/adapter_asianfanficscom.py

@@ -111,11 +111,17 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
     def doExtractChapterUrlsAndMetadata(self,get_cover=True):
         url = self.url
         logger.info("url: "+url)
-        data = self.get_request(url)
+        soup = None
+        try:
+            data = self.get_request(url)
+            soup = self.make_soup(data)
+        except exceptions.HTTPErrorFFF as e:
+            if e.status_code != 404:
+                raise
+            data = self.decode_data(e.data)
 
-        soup = self.make_soup(data)
-
-        if self.loginNeededCheck(data):
+        # logger.debug(data)
+        if not soup or self.loginNeededCheck(data):
             # always login if not already to avoid lots of headaches
             self.performLogin(url,data)
             # refresh website after logging in
@@ -140,8 +146,8 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
 
         # Find authorid and URL from... author url.
         mainmeta = soup.find('footer', {'class': 'main-meta'})
-        alist = mainmeta.find('span', text='Author(s)')
-        alist = alist.parent.findAll('a', href=re.compile(r"/profile/u/[^/]+"))
+        alist = mainmeta.find('span', string='Author(s)')
+        alist = alist.parent.find_all('a', href=re.compile(r"/profile/u/[^/]+"))
         for a in alist:
             self.story.addToList('authorId',a['href'].split('/')[-1])
             self.story.addToList('authorUrl','https://'+self.host+a['href'])
@@ -153,10 +159,10 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
         chapters=soup.find('select',{'name':'chapter-nav'})
         hrefattr=None
         if chapters:
-            chapters=chapters.findAll('option')
+            chapters=chapters.find_all('option')
             hrefattr='value'
         else: # didn't find <select name='chapter-nav', look for alternative
-            chapters=soup.find('div',{'class':'widget--chapters'}).findAll('a')
+            chapters=soup.find('div',{'class':'widget--chapters'}).find_all('a')
             hrefattr='href'
         for index, chapter in enumerate(chapters):
             if chapter.text != 'Foreword' and 'Collapse chapters' not in chapter.text:
@@ -165,9 +171,9 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
 
 
         # find timestamp
-        a = soup.find('span', text='Updated')
+        a = soup.find('span', string='Updated')
         if a == None:
-            a = soup.find('span', text='Published') # use published date if work was never updated
+            a = soup.find('span', string='Published') # use published date if work was never updated
         a = a.parent.find('time')
         chapterDate = makeDate(a['datetime'],self.dateformat)
         if newestChapter == None or chapterDate > newestChapter:
@@ -175,7 +181,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
             self.newestChapterNum = index
 
         # story status
-        a = mainmeta.find('span', text='Completed')
+        a = mainmeta.find('span', string='Completed')
         if a:
             self.story.setMetadata('status', 'Completed')
         else:
@@ -194,37 +200,37 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
             self.setDescription(url,a)
 
         # story tags
-        a = mainmeta.find('span',text='Tags')
+        a = mainmeta.find('span',string='Tags')
         if a:
-            tags = a.parent.findAll('a')
+            tags = a.parent.find_all('a')
             for tag in tags:
                 self.story.addToList('tags', tag.text)
 
         # story tags
-        a = mainmeta.find('span',text='Characters')
+        a = mainmeta.find('span',string='Characters')
         if a:
             self.story.addToList('characters', a.nextSibling)
 
         # published on
-        a = soup.find('span', text='Published')
+        a = soup.find('span', string='Published')
         a = a.parent.find('time')
         self.story.setMetadata('datePublished', makeDate(a['datetime'], self.dateformat))
 
         # updated on
-        a = soup.find('span', text='Updated')
+        a = soup.find('span', string='Updated')
         if a:
             a = a.parent.find('time')
             self.story.setMetadata('dateUpdated', makeDate(a['datetime'], self.dateformat))
 
         # word count
-        a = soup.find('span', text='Total Word Count')
+        a = soup.find('span', string='Total Word Count')
         if a:
             a = a.find_next('span')
             self.story.setMetadata('numWords', int(a.text.split()[0]))
 
         # upvote, subs, and views
         a = soup.find('div',{'class':'title-meta'})
-        spans = a.findAll('span', recursive=False)
+        spans = a.find_all('span', recursive=False)
         self.story.setMetadata('upvotes', re.search(r'\(([^)]+)', spans[0].find('span').text).group(1))
         self.story.setMetadata('subscribers', re.search(r'\(([^)]+)', spans[1].find('span').text).group(1))
         if len(spans) > 2: # views can be private
@@ -246,13 +252,39 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
 
         data = self.get_request(url)
         soup = self.make_soup(data)
+        # logger.debug(data)
+
+        ageform = soup.select_one('form[action="/account/toggle_age"]')
+        # logger.debug(ageform)
+        if ageform and (self.is_adult or self.getConfig("is_adult")):
+            params = {}
+            params['is_of_age']=ageform.select_one('input#is_of_age')['value']
+            params['current_url']=ageform.select_one('input#current_url')['value']
+            params['csrf_aff_token']=ageform.select_one('input[name="csrf_aff_token"]')['value']
+            loginUrl = 'https://' + self.getSiteDomain() + '/account/mark_over_18'
+            logger.info("Will now toggle age to URL (%s)" % (loginUrl))
+            # logger.debug(params)
+            data = self.post_request(loginUrl, params)
+            soup = self.make_soup(data)
+            # logger.debug(data)
+
         content = soup.find('div', {'id': 'user-submitted-body'})
 
+        if self.getConfig('inject_chapter_image'):
+            logger.debug("Injecting chapter image")
+            imgdiv = soup.select_one('div#bodyText div.bot-spacer')
+            if imgdiv:
+                content.insert(0, "\n")
+                content.insert(0, imgdiv)
+                content.insert(0, "\n")
+
         if self.getConfig('inject_chapter_title'):
             logger.debug("Injecting full-length chapter title")
             title = soup.find('h1', {'id' : 'chapter-title'}).text
             newTitle = soup.new_tag('h3')
             newTitle.string = title
+            content.insert(0, "\n")
             content.insert(0, newTitle)
+            content.insert(0, "\n")
 
         return self.utf8FromSoup(url,content)

fanficfare/adapters/adapter_bdsmlibrarycom.py

@@ -126,7 +126,7 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
 
         # Find the chapters:
         # The update date is with the chapter links... so we will update it here as well
-        for chapter in soup.findAll('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+r"&chapterid=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+r"&chapterid=\d+$")):
             value = chapter.findNext('td').findNext('td').string.replace('(added on','').replace(')','').strip()
             self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
             self.add_chapter(chapter,'https://'+self.getSiteDomain()+chapter['href'])
@@ -134,11 +134,11 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
 
         # Get the MetaData
         # Erotia Tags
-        tags = soup.findAll('a',href=re.compile(r'/stories/search.php\?selectedcode'))
+        tags = soup.find_all('a',href=re.compile(r'/stories/search.php\?selectedcode'))
         for tag in tags:
             self.story.addToList('eroticatags',tag.text)
 
-        for td in soup.findAll('td'):
+        for td in soup.find_all('td'):
             if len(td.text)>0:
                 if 'Added on:' in td.text and '<table' not in unicode(td):
                     value = td.text.replace('Added on:','').strip()
@@ -169,20 +169,20 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToDownload("Error downloading Chapter: {0}!  Missing required element!".format(url))
 
         #strip comments from soup
-        [comment.extract() for comment in chaptertag.findAll(text=lambda text:isinstance(text, Comment))]
+        [comment.extract() for comment in chaptertag.find_all(string=lambda text:isinstance(text, Comment))]
 
         # BDSM Library basically wraps it's own html around the document,
         # so we will be removing the script, title and meta content from the
         # storyblock
-        for tag in chaptertag.findAll('head') + chaptertag.findAll('style') + chaptertag.findAll('title') + chaptertag.findAll('meta') + chaptertag.findAll('o:p') + chaptertag.findAll('link'):
+        for tag in chaptertag.find_all('head') + chaptertag.find_all('style') + chaptertag.find_all('title') + chaptertag.find_all('meta') + chaptertag.find_all('o:p') + chaptertag.find_all('link'):
             tag.extract()
 
-        for tag in chaptertag.findAll('o:smarttagtype'):
+        for tag in chaptertag.find_all('o:smarttagtype'):
             tag.name = 'span'
 
         ## I'm going to take the attributes off all of the tags
         ## because they usually refer to the style that we removed above.
-        for tag in chaptertag.findAll(True):
+        for tag in chaptertag.find_all(True):
             tag.attrs = None
 
         return self.utf8FromSoup(url,chaptertag)

fanficfare/adapters/adapter_bloodshedversecom.py

@@ -117,7 +117,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
 
         summary_div = list_box.find('div', {'class': 'list_summary'})
         if not self.getConfig('keep_summary_html'):
-            summary = ''.join(summary_div(text=True))
+            summary = ''.join(summary_div(string=True))
         else:
             summary = self.utf8FromSoup(author_url, summary_div)
 
@@ -157,9 +157,6 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
 
                         self.story.addToList('warnings', warning)
 
-                elif key == 'Chapters':
-                    self.story.setMetadata('numChapters', int(value))
-
                 elif key == 'Words':
                     # Apparently only numChapters need to be an integer for
                     # some strange reason. Remove possible ',' characters as to
@@ -174,7 +171,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
                     # ugly %p(am/pm) hack moved into makeDate so other sites can use it.
                     self.story.setMetadata('dateUpdated', date)
 
-        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+        if self.story.getMetadataRaw('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
             raise exceptions.AdultCheckRequired(self.url)
 
     def getChapterText(self, url):

fanficfare/adapters/adapter_bloodtiesfancom.py

@@ -1,310 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from bs4.element import Tag
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-# By virtue of being recent and requiring both is_adult and user/pass,
-# adapter_fanficcastletvnet.py is the best choice for learning to
-# write adapters--especially for sites that use the eFiction system.
-# Most sites that have ".../viewstory.php?sid=123" in the story URL
-# are eFiction.
-
-# For non-eFiction sites, it can be considerably more complex, but
-# this is still a good starting point.
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return BloodTiesFansComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','btf') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y" # XXX
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'bloodties-fans.com' # XXX
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self.post_request(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-
-            # Furthermore, there's a couple sites now with more than
-            # one warning level for different ratings.  And they're
-            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
-            # we'll try 1 first.
-            addurl = "&ageconsent=ok&warning=4" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  nfacommunity uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-
-        # viewstory.php?sid=561&warning=4
-        # viewstory.php?sid=561&warning=1
-        # viewstory.php?sid=561&warning=2
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                data = self.get_request(url)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl)
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        listbox = soup.find('div',{'class':'listbox'})
-        # <strong>Rating:</strong> M<br /> etc
-        labels = listbox.findAll('strong')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next strong tag.
-                svalue = ""
-                while not isinstance(value,Tag) or value.name != 'strong':
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Words' in label:
-                value=re.sub(r"\|",r"",value)
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                value=re.sub(r"\|",r"",value)
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                value=re.sub(r"\|",r"",value)
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        # moved outside because they changed *most*, but not *all* labels to <strong>
-        ships = listbox.findAll('a',href=re.compile(r'browse.php.type=class&(amp;)?type_id=2')) # crappy html: & vs &amp; in url.
-        shipstext = [ship.string for ship in ships]
-        for ship in shipstext:
-            self.story.addToList('ships',ship.string)
-
-        genres = listbox.findAll('a',href=re.compile(r'browse.php\?type=class&(amp;)?type_id=1')) # crappy html: & vs &amp; in url.
-        genrestext = [genre.string for genre in genres]
-        for genre in genrestext:
-            self.story.addToList('genre',genre.string)
-
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fiction/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_buffygilescom.py

@@ -1,279 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return BuffyGilesComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class BuffyGilesComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        # XXX Most sites don't have the /efiction part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/efiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','bufg')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'buffygiles.velocitygrass.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self.post_request(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self.get_request(url)
-
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-
-        # efiction/viewstory.php?sid=1882&warning=4
-        # efiction/viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        m = re.search(r"'efiction/viewstory.php\?sid=542(&warning=5)'",data)
-        m = re.search(r"'efiction/viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                data = self.get_request(url)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        pagetitle = soup.find('div',{'id':'pagetitle'})
-
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/efiction/'+chapter['href']+addurl)
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"efiction/viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^efiction/viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('efiction/viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_cfaa.py

@@ -1,6 +1,6 @@
-# -*- coding: utf-8 -*-
+#  -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
+# Copyright 2024 FanFicFare team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,25 +16,23 @@
 #
 
 from __future__ import absolute_import
+import logging
+logger = logging.getLogger(__name__)
 
-# Software: eFiction
-from .base_efiction_adapter import BaseEfictionAdapter
-
-class TasteOfPoisonInkubationNetAdapter(BaseEfictionAdapter):
-    ''' This adapter will download stories from the
-       'Taste of Poison, the Fanfiction of Arsenic Jade' site '''
-
-    @staticmethod
-    def getSiteDomain():
-        return 'tasteofpoison.inkubation.net'
-
-    @classmethod
-    def getSiteAbbrev(self):
-        return 'topa'
-
-    @classmethod
-    def getDateFormat(self):
-        return "%d/%m/%y"
+from .base_otw_adapter import BaseOTWAdapter
 
 def getClass():
-    return TasteOfPoisonInkubationNetAdapter
+    return CFAAAdapter
+
+class CFAAAdapter(BaseOTWAdapter):
+
+    def __init__(self, config, url):
+        BaseOTWAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','cfaa')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.cfaarchive.org'

fanficfare/adapters/adapter_chaossycophanthexcom.py

@@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         self.story.setMetadata('rating', rating)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
 
@@ -134,7 +134,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
 
         # <span class="label">Rated:</span> NC-17<br /> etc
 
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
 
         value = labels[0].previousSibling
         svalue = ""
@@ -154,22 +154,22 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('numWords', value.split(' -')[0])
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -194,7 +194,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/'+a['href']
 
             seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):

fanficfare/adapters/adapter_chireadscom.py

@@ -88,8 +88,8 @@ class ChireadsComSiteAdapter(BaseSiteAdapter):
         intro = stripHTML(info.select_one('.inform-inform-txt').span)
         self.setDescription(self.url, intro)
 
-        for content in soup.findAll('div', {'id': 'content'}):
-            for a in content.findAll('a'):
+        for content in soup.find_all('div', {'id': 'content'}):
+            for a in content.find_all('a'):
                 self.add_chapter(a.get_text(), a['href'])
 
 

fanficfare/adapters/adapter_chosentwofanficcom.py

@@ -49,7 +49,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
 
 
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','chosen2')
@@ -65,10 +65,10 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return r"https?"+re.escape("://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -98,7 +98,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
         ## Title
         ## Some stories have a banner that has it's own a tag before the actual text title...
         ## so I'm checking the pagetitle div for all a tags that match the criteria, then taking the last.
-        a = soup.find('div',{'id':'pagetitle'}).findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))[-1]
+        a = soup.find('div',{'id':'pagetitle'}).find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))[-1]
         self.story.setMetadata('title',stripHTML(a))
 
         # Find authorid and URL from... author url.
@@ -106,14 +106,14 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
         # so I'm checking the pagetitle div for this as well
         a = soup.find('div',{'id':'pagetitle'}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             #self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
-            self.add_chapter(chapter,'http://{0}/{1}{2}'.format(self.host, chapter['href'],addURL))
+            self.add_chapter(chapter,'https://{0}/{1}{2}'.format(self.host, chapter['href'],addURL))
 
 
         # eFiction sites don't help us out a lot with their meta data
@@ -127,7 +127,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
                 return ""
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
         for labelspan in labels:
             val = labelspan.nextSibling
             value = unicode('')
@@ -149,27 +149,27 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
                 self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                ships = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
                 for ship in ships:
                     self.story.addToList('ships',ship.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -192,16 +192,16 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'https://'+self.host+'/'+a['href']
 
             seriessoup = self.make_soup(self.get_request(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                # this site has several links to each story.
+                if a.text == 'Latest Chapter':
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
                         self.story.setMetadata('seriesUrl',series_url)
                         break

fanficfare/adapters/adapter_csiforensicscom.py

@@ -1,220 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return CSIForensicsComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class CSIForensicsComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-
-        self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csiforensics')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'csi-forensics.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5&skin=elegantcsi"
-        else:
-            addurl="&skin=elegantcsi"
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "This story is rated NC-17, and therefore is not suitable for minors.  If you are below the age required to view such material in your locality, please return from whence you came." in data: # XXX
-            raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title
-
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Rating
-        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
-        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
-        self.story.setMetadata('rating', rate)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        smalldiv = soup.find('div', {'class' : 'small'})
-
-
-        chars = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-
-        metatext = stripHTML(smalldiv)
-
-        if 'Completed: Yes' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        word=soup.find(text=re.compile("Word count:")).split(':')
-        self.story.setMetadata('numWords', word[1])
-
-        cats = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-
-        warnings = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=class(&amp;)type_id=2(&amp;)classid=\d+'))
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)
-
-        date=soup.find('div',{'class' : 'bottom'})
-        pd=date.find(text=re.compile("Published:")).string.split(': ')
-        self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat))
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        pub=0
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Genres' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'https://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-        smalldiv.extract()
-
-        # Summary
-        summary = soup.find('div', {'class' : 'content'})
-        self.setDescription(url,summary)
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_destinysgatewaycom.py

@@ -1,222 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DestinysGatewayComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DestinysGatewayComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dgrfa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d %Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.destinysgateway.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=").replace(r"www\.",r"(www\.)?")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                data = self.get_request(url)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while value and 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_deviantartcom.py

@@ -25,6 +25,7 @@ from ..six.moves.urllib.parse import urlparse
 from .base_adapter import BaseSiteAdapter, makeDate
 from fanficfare.htmlcleanup import stripHTML
 from .. import exceptions as exceptions
+from fanficfare.dateutils import parse_relative_date_string
 
 logger = logging.getLogger(__name__)
 
@@ -73,81 +74,126 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
         return r'https?://www\.deviantart\.com/(?P<author>[^/]+)/art/(?P<id>[^/]+)/?'
 
     def performLogin(self, url):
-        data = self.get_request_raw('https://www.deviantart.com/users/login', referer=url)
+        if self.username and self.username != 'NoneGiven':
+            username = self.username
+        else:
+            username = self.getConfig('username')
+
+        # logger.debug("\n\nusername:(%s)\n\n"%username)
+        if not username:
+            logger.info("Login Required for URL %s" % url)
+            raise exceptions.FailedToLogin(url,username)
+
+        data = self.get_request_raw('https://www.deviantart.com/users/login', referer=url, usecache=False)
         data = self.decode_data(data)
         soup = self.make_soup(data)
         params = {
-            'referer': url,
+            'referer': 'https://www.deviantart.com/_sisu/do/signin', # soup.find('input', {'name': 'referer'})['value'],
+            'referer_type': soup.find('input', {'name': 'referer_type'})['value'],
             'csrf_token': soup.find('input', {'name': 'csrf_token'})['value'],
             'challenge': soup.find('input', {'name': 'challenge'})['value'],
+            'lu_token': soup.find('input', {'name': 'lu_token'})['value'],
+            'remember': 'on',
+            'username': username
+        }
+
+        loginUrl = 'https://' + self.getSiteDomain() + '/_sisu/do/step2'
+        logger.debug('Will now login to deviantARt as (%s)' % username)
+
+        result = self.post_request(loginUrl, params, usecache=False)
+        soup = self.make_soup(result)
+        if not soup.find('input', {'name': 'lu_token2'}):
+            logger.info("Login Failed for URL %s (no lu_token2 found)" % url)
+            raise exceptions.FailedToLogin(url,username)
+
+        params = {
+            'referer': 'https://www.deviantart.com/_sisu/do/signin', # soup.find('input', {'name': 'referer'})['value'],
+            'referer_type': soup.find('input', {'name': 'referer_type'})['value'],
+            'csrf_token': soup.find('input', {'name': 'csrf_token'})['value'],
+            'challenge': soup.find('input', {'name': 'challenge'})['value'],
+            'lu_token': soup.find('input', {'name': 'lu_token'})['value'],
+            'lu_token2': soup.find('input', {'name': 'lu_token2'})['value'],
+            'remember': 'on',
+            'username': ''
         }
 
         if self.password:
-            params['username'] = self.username
             params['password'] = self.password
         else:
-            params['username'] = self.getConfig('username')
             params['password'] = self.getConfig('password')
 
+        # logger.debug("\n\nparams['password']:(%s)\n\n"%params['password'])
         loginUrl = 'https://' + self.getSiteDomain() + '/_sisu/do/signin'
-        logger.debug('Will now login to deviantARt as (%s)' % params['username'])
+        logger.debug('Will now send password to deviantARt')
 
         result = self.post_request(loginUrl, params, usecache=False)
 
         if 'Log In | DeviantArt' in result:
-            logger.error('Failed to login to deviantArt as %s' % params['username'])
-            raise exceptions.FailedToLogin('https://www.deviantart.com', params['username'])
+            logger.error('Failed to login to deviantArt as %s' % username)
+            raise exceptions.FailedToLogin('https://www.deviantart.com', username)
         else:
             return True
 
     def requiresLogin(self, data):
         return '</a> has limited the viewing of this artwork to members of the DeviantArt community only' in data
 
+    def isLoggedIn(self, data):
+        return '<form id="logout-form" action="https://www.deviantart.com/users/logout" method="POST">' in data
+
     def isWatchersOnly(self, data):
-        return '<span>Watchers-Only Deviation</span>' in data
+        return '>Watchers-Only Deviation<' in data
 
     def requiresMatureContentEnabled(self, data):
         return (
             '>This content is intended for mature audiences<' in data
             or '>This deviation is intended for mature audiences<' in data
+            or '>This filter hides content that may be inappropriate for some viewers<' in data
+            or '>May contain sensitive content<' in data
+            or '>Log in to view<' in data
+            or '>This deviation has been labeled as containing themes not suitable for all deviants.<' in data
         )
 
     def extractChapterUrlsAndMetadata(self):
-        isLoggedIn = False
         logger.debug('URL: %s', self.url)
 
         data = self.get_request(self.url)
-
         soup = self.make_soup(data)
 
-        if self.requiresLogin(data):
+        ## story can require login outright, or it can show up as
+        ## watchers-only or mature-enabled without the same 'requires
+        ## login' strings.
+        if self.requiresLogin(data) or ( not self.isLoggedIn(data) and
+                                         (self.isWatchersOnly(data) or
+                                          self.requiresMatureContentEnabled(data)) ):
             if self.performLogin(self.url):
-                isLoggedIn = True
                 data = self.get_request(self.url, usecache=False)
                 soup = self.make_soup(data)
 
+        ## Check watchers only and mature enabled again, separately,
+        ## after login because they can still apply after login.
         if self.isWatchersOnly(data):
             raise exceptions.FailedToDownload(
                 'Deviation is only available for watchers.' +
                 'You must watch this author before you can download it.'
-            )
-
+                )
         if self.requiresMatureContentEnabled(data):
-            # as far as I can tell deviantArt has no way to show mature
-            # content that doesn't involve logging in or using JavaScript
-            if not isLoggedIn:
-                self.performLogin(self.url)
-                isLoggedIn = True
-                data = self.get_request(self.url, usecache=False)
-                soup = self.make_soup(data)
-                if self.requiresMatureContentEnabled(data):
-                    raise exceptions.FailedToDownload(
-                        'Deviation is set as mature, you must go into your account ' +
-                        'and enable showing of mature content.'
-                    )
+            raise exceptions.FailedToDownload(
+                'Deviation is set as mature, you must go into your account ' +
+                'and enable showing of mature content.'
+                )
 
-        appurl = soup.select_one('meta[property="da:appurl"]')['content']
-        story_id = urlparse(appurl).path.lstrip('/')
+        appurl = soup.select_one('meta[property="og:url"]')['content']
+        if appurl:
+            story_id = urlparse(appurl).path.lstrip('/')
+        else:
+            logger.debug("Looking for JS story id")
+            ## after login, this is only found in a JS block.  Dunno why.
+            ## F875A309-B0DB-860E-5079-790D0FBE5668
+            match = re.match(r'\\"deviationUuid\\":\\"(?P<id>[A-Z0-9-]+)\\",',data)
+            if match:
+                story_id = match.group('id')
+            else:
+                raise exceptions.FailedToDownload('Failed to find Story ID.')
         self.story.setMetadata('storyId', story_id)
 
         title = soup.select_one('h1').get_text()
@@ -157,7 +203,12 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
         # self.story.setMetadata('status', 'Completed')
 
         pubdate = soup.select_one('time').get_text()
-        self.story.setMetadata('datePublished', makeDate(pubdate, '%b %d, %Y'))
+
+        # Maybe do this better, but this works
+        try:
+            self.story.setMetadata('datePublished', makeDate(pubdate, '%b %d, %Y'))
+        except:
+            self.story.setMetadata('datePublished', parse_relative_date_string(pubdate))
 
         # do description here if appropriate
 
@@ -171,19 +222,35 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s', url)
         data = self.get_request(url)
+        # logger.debug(data)
         soup = self.make_soup(data)
 
         # remove comments section to avoid false matches
         comments = soup.select_one('[data-hook=comments_thread]')
-        comments.decompose()
+        if comments:
+            comments.decompose()
+        # previous search not always found in some stories.
+        # <div id="comments"></div> inside the real containing
+        # div seems more common
+        commentsdiv = soup.select_one('div#comments')
+        if commentsdiv:
+            commentsdiv.parent.decompose()
+
+        # three different 'content' tags to look for.
+        # This is the current in Oct 2024
+        content = soup.select_one('[data-editor-viewer="1"]')
 
-        content = soup.select_one('[data-id=rich-content-viewer]')
         if content is None:
-            # older story
+            # older story? I can't find any of this style in Oct2024
+            content = soup.select_one('[data-id="rich-content-viewer"]')
+
+        if content is None:
+            # olderer story, but used by some older (2018) posts
             content = soup.select_one('.legacy-journal')
-            if content is None:
-                raise exceptions.FailedToDownload(
-                    'Could not find story text. Please open a bug with the URL %s' % self.url
+
+        if content is None:
+            raise exceptions.FailedToDownload(
+                'Could not find story text. Please open a bug with the URL %s' % self.url
                 )
 
         return self.utf8FromSoup(url, content)

fanficfare/adapters/adapter_dokugacom.py

@@ -95,7 +95,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         params['Submit'] = 'Submit'
 
         # copy all hidden input tags to pick up appropriate tokens.
-        for tag in soup.findAll('input',{'type':'hidden'}):
+        for tag in soup.find_all('input',{'type':'hidden'}):
             params[tag['name']] = tag['value']
 
         loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
@@ -153,7 +153,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters:
-        chapters = soup.find('select').findAll('option')
+        chapters = soup.find('select').find_all('option')
         if len(chapters)==1:
             self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1')
         else:
@@ -168,7 +168,7 @@ class DokugaComAdapter(BaseSiteAdapter):
             asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
 
             #grab the rest of the metadata from the author's page
-            for div in asoup.findAll('div'):
+            for div in asoup.find_all('div'):
                 nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
                 if nav != None:
                     break
@@ -208,7 +208,7 @@ class DokugaComAdapter(BaseSiteAdapter):
 
         else:
             asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
-            for div in asoup.findAll('div'):
+            for div in asoup.find_all('div'):
                 nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
                 if nav != None:
                     break

fanficfare/adapters/adapter_dracoandginnycom.py

@@ -161,7 +161,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
 
@@ -181,13 +181,13 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
 
         self.setDescription(url,content.find('blockquote'))
 
-        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+        for genre in content.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
             self.story.addToList('genre',genre.string)
 
-        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+        for warning in content.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
             self.story.addToList('warnings',warning.string)
 
-        labels = content.findAll('b')
+        labels = content.find_all('b')
 
         for labelspan in labels:
             value = labelspan.nextSibling
@@ -208,22 +208,22 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('rating', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -247,7 +247,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
 
             seriessoup = self.make_soup(self.get_request(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
                 # skip 'report this' and 'TOC' links

fanficfare/adapters/adapter_efpfanficnet.py

@@ -138,7 +138,7 @@ class EFPFanFicNet(BaseSiteAdapter):
             # no selector found, so it's a one-chapter story.
             self.add_chapter(self.story.getMetadata('title'),url)
         else:
-            allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
+            allOptions = select.find_all('option', {'value' : re.compile(r'viewstory')})
             for o in allOptions:
                 url = u'https://%s/%s' % ( self.getSiteDomain(),
                                           o['value'])
@@ -170,14 +170,14 @@ class EFPFanFicNet(BaseSiteAdapter):
             if authsoup != None:
                 # last author link with offset should be the 'next' link.
                 authurl = u'https://%s/%s' % ( self.getSiteDomain(),
-                                              authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
+                                              authsoup.find_all('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
 
             # Need author page for most of the metadata.
             logger.debug("fetching author page: (%s)"%authurl)
             authsoup = self.make_soup(self.get_request(authurl))
             #print("authsoup:%s"%authsoup)
 
-            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
+            storyas = authsoup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
             for storya in storyas:
                 #print("======storya:%s"%storya)
                 storyblock = storya.findParent('div',{'class':'storybloc'})
@@ -194,7 +194,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
         # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
 
-        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        cats = noteblock.find_all('a',href=re.compile(r'browse.php\?type=categories'))
         for cat in cats:
             self.story.addToList('category',cat.string)
 
@@ -262,7 +262,7 @@ class EFPFanFicNet(BaseSiteAdapter):
 
             seriessoup = self.make_soup(self.get_request(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
             i=1
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
@@ -288,11 +288,11 @@ class EFPFanFicNet(BaseSiteAdapter):
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         # remove any header and 'o:p' tags.
-        for tag in div.findAll("head") + div.findAll("o:p"):
+        for tag in div.find_all("head") + div.find_all("o:p"):
             tag.extract()
 
         # change any html and body tags to div.
-        for tag in div.findAll("html") + div.findAll("body"):
+        for tag in div.find_all("html") + div.find_all("body"):
             tag.name='div'
 
         # remove extra bogus doctype.

fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -126,7 +126,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         self.story.setMetadata('rating', rating)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
 
@@ -144,7 +144,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
 
         # <span class="label">Rated:</span> NC-17<br /> etc
 
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
 
         value = labels[0].previousSibling
         svalue = ""
@@ -164,22 +164,22 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('numWords', value.split(' -')[0])
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -204,7 +204,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/'+a['href']
 
             seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
                 # skip 'report this' and 'TOC' links

fanficfare/adapters/adapter_fanficauthorsnet.py

@@ -53,6 +53,9 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
         #Setting the 'Zone' for each "Site"
         self.zone = self.parsedUrl.netloc.replace('.fanficauthors.net','')
 
+        # site change .nsns to -nsns
+        self.zone = self.zone.replace('.nsns','-nsns')
+
         # normalized story URL.
         self._setURL('https://{0}.{1}/{2}/'.format(
             self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
@@ -79,7 +82,10 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
     @classmethod
     def getAcceptDomains(cls):
 
+        # need both .nsns(old) and -nsns(new) because it's a domain
+        # change, not just URL change.
         return ['aaran-st-vines.nsns.fanficauthors.net',
+                'aaran-st-vines-nsns.fanficauthors.net',
                 'abraxan.fanficauthors.net',
                 'bobmin.fanficauthors.net',
                 'canoncansodoff.fanficauthors.net',
@@ -95,9 +101,12 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
                 'jeconais.fanficauthors.net',
                 'kinsfire.fanficauthors.net',
                 'kokopelli.nsns.fanficauthors.net',
+                'kokopelli-nsns.fanficauthors.net',
                 'ladya.nsns.fanficauthors.net',
+                'ladya-nsns.fanficauthors.net',
                 'lorddwar.fanficauthors.net',
                 'mrintel.nsns.fanficauthors.net',
+                'mrintel-nsns.fanficauthors.net',
                 'musings-of-apathy.fanficauthors.net',
                 'ruskbyte.fanficauthors.net',
                 'seelvor.fanficauthors.net',
@@ -108,7 +117,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
     ################################################################################################
     @classmethod
     def getSiteExampleURLs(self):
-        return ("https://aaran-st-vines.nsns.fanficauthors.net/A_Story_Name/ "
+        return ("https://aaran-st-vines-nsns.fanficauthors.net/A_Story_Name/ "
               + "https://abraxan.fanficauthors.net/A_Story_Name/ "
               + "https://bobmin.fanficauthors.net/A_Story_Name/ "
               + "https://canoncansodoff.fanficauthors.net/A_Story_Name/ "
@@ -123,10 +132,10 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
               + "https://jbern.fanficauthors.net/A_Story_Name/ "
               + "https://jeconais.fanficauthors.net/A_Story_Name/ "
               + "https://kinsfire.fanficauthors.net/A_Story_Name/ "
-              + "https://kokopelli.nsns.fanficauthors.net/A_Story_Name/ "
-              + "https://ladya.nsns.fanficauthors.net/A_Story_Name/ "
+              + "https://kokopelli-nsns.fanficauthors.net/A_Story_Name/ "
+              + "https://ladya-nsns.fanficauthors.net/A_Story_Name/ "
               + "https://lorddwar.fanficauthors.net/A_Story_Name/ "
-              + "https://mrintel.nsns.fanficauthors.net/A_Story_Name/ "
+              + "https://mrintel-nsns.fanficauthors.net/A_Story_Name/ "
               + "https://musings-of-apathy.fanficauthors.net/A_Story_Name/ "
               + "https://ruskbyte.fanficauthors.net/A_Story_Name/ "
               + "https://seelvor.fanficauthors.net/A_Story_Name/ "
@@ -136,8 +145,16 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
 
     ################################################################################################
     def getSiteURLPattern(self):
+        ## .nsns kept here to match both . and -
         return r'https?://(aaran-st-vines.nsns|abraxan|bobmin|canoncansodoff|chemprof|copperbadge|crys|deluded-musings|draco664|fp|frenchsession|ishtar|jbern|jeconais|kinsfire|kokopelli.nsns|ladya.nsns|lorddwar|mrintel.nsns|musings-of-apathy|ruskbyte|seelvor|tenhawk|viridian|whydoyouneedtoknow)\.fanficauthors\.net/([a-zA-Z0-9_]+)/'
 
+    @classmethod
+    def get_section_url(cls,url):
+        ## only changing .nsns to -nsns and only when part of the
+        ## domain.
+        url = url.replace('.nsns.fanficauthors.net','-nsns.fanficauthors.net')
+        return url
+
     ################################################################################################
     def doExtractChapterUrlsAndMetadata(self, get_cover=True):
 
@@ -163,7 +180,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
         # Find the chapters:
         # The published and update dates are with the chapter links...
         # so we have to get them from there.
-        chapters = soup.findAll('a', href=re.compile('/'+self.story.getMetadata(
+        chapters = soup.find_all('a', href=re.compile('/'+self.story.getMetadata(
             'storyId')+'/([a-zA-Z0-9_]+)/'))
 
         # Here we are getting the published date. It is the date the first chapter was "updated"
@@ -202,7 +219,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
         ## Raising AdultCheckRequired after collecting chapters gives
         ## a double chapter list.  So does genre, but it de-dups
         ## automatically.
-        if( self.story.getMetadata('rating') == 'Mature'
+        if( self.story.getMetadataRaw('rating') in ['Mature','Adult Only']
             and not (self.is_adult or self.getConfig("is_adult")) ):
             raise exceptions.AdultCheckRequired(self.url)
 
@@ -226,7 +243,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        if( self.story.getMetadata('rating') == 'Mature' and
+        if( self.story.getMetadataRaw('rating') in ['Mature','Adult Only'] and
             (self.is_adult or self.getConfig("is_adult")) ):
             addurl = "?bypass=1"
         else:
@@ -241,8 +258,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
                 "Error downloading Chapter: '{0}'!  Missing required element!".format(url))
 
         #Now, there are a lot of extranious tags within the story division.. so we will remove them.
-        for tag in story.findAll('ul',{'class':'pager'}) + story.findAll(
-            'div',{'class':'alert'}) + story.findAll('div', {'class':'btn-group'}):
+        for tag in story.find_all('ul',{'class':'pager'}) + story.find_all(
+            'div',{'class':'alert'}) + story.find_all('div', {'class':'btn-group'}):
             tag.extract()
 
         return self.utf8FromSoup(url,story)

fanficfare/adapters/adapter_fanficsme.py

@@ -134,7 +134,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
         ## restrict meta searches to header.
         fichead = soup.find('div',class_='FicHead')
         def get_meta_content(title):
-            val_label = fichead.find('div',string=title+u':')
+            val_label = fichead.find('div',string=re.compile(u'^'+title+u':'))
             if val_label:
                 return val_label.find_next('div')
 
@@ -150,7 +150,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
         self.story.setMetadata('rating',stripHTML(get_meta_content(u'Рейтинг')))
 
         ## Need to login for any rating higher than General.
-        if self.story.getMetadata('rating') != 'General' and self.needToLoginCheck(data):
+        if self.story.getMetadataRaw('rating') != 'General' and self.needToLoginCheck(data):
             self.performLogin(url)
             # reload after login.
             data = self.get_request(url,usecache=False)
@@ -168,7 +168,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',stripHTML(h))
 
         ## author(s):
-        content = get_meta_content(u'Автор')
+        content = get_meta_content(u'Авторы?')
         if content:
             alist = content.find_all('a', class_='user')
             for a in alist:
@@ -181,12 +181,8 @@ class FanFicsMeAdapter(BaseSiteAdapter):
                 self.story.setMetadata('authorUrl','https://'+self.host)
                 self.story.setMetadata('authorId','0')
 
-        # translator(s)
-        content = get_meta_content(u'Переводчик')
-        if not content:
-            # Переводчик vs Переводчи is 'Translator' vs 'TranslatorS'
-            content = get_meta_content(u'Переводчи')
-        logger.debug(content)
+        # translator(s) in different strings
+        content = get_meta_content(u'Переводчикк?и?')
         if content:
             for a in content.find_all('a', class_='user'):
                 self.story.addToList('translatorsId',a['href'].split('/user')[-1])
@@ -241,7 +237,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
 
         ## size block, only saving word count.
         content = get_meta_content(u'Размер')
-        words = stripHTML(content.find_all('li')[1])
+        words = stripHTML(content.find('a'))
         words = re.sub(r'[^0-9]','',words) # only keep numbers
         self.story.setMetadata('numWords',words)
 
@@ -301,6 +297,10 @@ class FanFicsMeAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterTextNum(self, url, index):
         logger.debug('Getting chapter text for: %s index: %s' % (url,index))
+        m = re.match(r'.*&chapter=(\d+).*',url)
+        if m:
+            index=m.group(1)
+            logger.debug("Using index(%s) from &chapter="%index)
 
         chapter_div = None
         if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and self.num_chapters() > 1:

fanficfare/adapters/adapter_fanfictalkcom.py

@@ -44,9 +44,8 @@ class FanfictalkComAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
 
-
         # normalized story URL.
-        self._setURL('https://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ahpfftc')
@@ -57,24 +56,24 @@ class FanfictalkComAdapter(BaseSiteAdapter):
 
     @classmethod
     def getAcceptDomains(cls):
-        return [cls.getSiteDomain(),'archive.hpfanfictalk.com']
+        return [cls.getSiteDomain(),'archive.hpfanfictalk.com','fanfictalk.com']
 
     @classmethod
     def getConfigSections(cls):
         "Only needs to be overriden if has additional ini sections."
-        return [cls.getConfigSection(),'archive.hpfanfictalk.com']
+        return [cls.getConfigSection(),'archive.hpfanfictalk.com','fanfictalk.com']
 
-    @staticmethod # must be @staticmethod, don't remove it.
+    @staticmethod # must be @stgetAcceptDomainsaticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'fanfictalk.com'
+        return 'archive.fanfictalk.com'
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+        return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return r"https?://(archive\.hp)?"+re.escape(self.getSiteDomain())+r"(/archive)?/viewstory\.php\?sid=\d+$"
+        return r"https?://("+r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()])+r")(/archive)?/viewstory\.php\?sid=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -118,7 +117,7 @@ class FanfictalkComAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'https://'+self.host+'/archive/'+chapter['href'])
+            self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href'])
 
         # categories
         for a in soup.select("div#sort a"):
@@ -171,14 +170,14 @@ class FanfictalkComAdapter(BaseSiteAdapter):
         # Site allows stories to be in several series at once.  FFF
         # isn't thrilled with that, we have series00, series01, etc.
         # Example:
-        # https://fanfictalk.com/archive/viewstory.php?sid=483
+        # https://archive.fanfictalk.com/viewstory.php?sid=483
 
         if self.getConfig("collect_series"):
             seriesspan = soup.find('span',label='Series')
             for i, seriesa in enumerate(seriesspan.find_all('a', href=re.compile(r"viewseries\.php\?seriesid=\d+"))):
                 # logger.debug(seriesa)
                 series_name = stripHTML(seriesa)
-                series_url = 'https://'+self.host+'/archive/'+seriesa['href']
+                series_url = 'https://'+self.host+'/'+seriesa['href']
 
                 seriessoup = self.make_soup(self.get_request(series_url))
                 storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
@@ -205,9 +204,17 @@ class FanfictalkComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
 
-        soup = self.make_soup(self.get_request(url))
+        logger.debug('Getting chapter text from: %s' % (url+addurl))
+        soup = self.make_soup(self.get_request(url+addurl))
 
         div = soup.find('div', {'id' : 'story'})
 

fanficfare/adapters/adapter_fanfictionnet.py

@@ -93,15 +93,47 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # logger.debug("post-url:%s"%url)
         return url
 
+    @classmethod
+    def get_url_search(cls,url):
+        regexp =  super(getClass(), cls).get_url_search(url)
+        regexp = re.sub(r"^(?P<keep>.*net/s/\d+/\d+/)(?P<urltitle>[^\$]*)?",
+                        r"\g<keep>(.*)",regexp)
+        logger.debug(regexp)
+        return regexp
+
     def getSiteURLPattern(self):
         return self._get_site_url_pattern()
 
-    ## not actually putting urltitle on multi-chapters below, but
-    ## one-shots will have it, so this is still useful.  normalized
-    ## chapter URLs do NOT contain the story title.
+    ## normalized chapter URLs DO contain the story title now, but
+    ## normalized to current urltitle in case of title changes.
     def normalize_chapterurl(self,url):
         return re.sub(r"https?://(www|m)\.(?P<keep>fanfiction\.net/s/\d+/\d+/).*",
-                      r"https://www.\g<keep>",url)
+                      r"https://www.\g<keep>",url)+self.urltitle
+
+    def get_request(self,url,usecache=True):
+        ## use super version if not set or isn't a chapter URL with a
+        ## title.
+        if( not self.getConfig("try_shortened_title_urls") or
+            not re.match(r"https?://www\.fanfiction\.net/s/\d+/\d+/(?P<title>[^/]+)$", url) ):
+            return super(getClass(), self).get_request(url,usecache)
+
+        ## kludgey way to attempt more than one URL variant by
+        ## removing title one letter at a time.  Note that network and
+        ## open_pages_in_browser retries still happen first.
+        titlelen = len(url.split('/')[-1])
+        maxcut = min([4,titlelen])
+        j = 0
+        while j < maxcut: # should actually leave loop either by
+                          # return or exception raise.
+            try:
+                useurl = url
+                if j: # j==0, full URL, then remove letters.
+                    useurl = url[:-j]
+                return super(getClass(), self).get_request(useurl,usecache)
+            except exceptions.HTTPErrorFFF as fffe:
+                if j >= maxcut or 'Page not found or expired' not in unicode(fffe):
+                    raise
+            j = j+1
 
     def doExtractChapterUrlsAndMetadata(self,get_cover=True):
 
@@ -135,7 +167,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## the first chapter.  It generates another server request and
         ## doesn't seem to be needed lately, so now default it to off.
         try:
-            chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            chapcount = len(soup.find('select', { 'name' : 'chapter' } ).find_all('option'))
             # get chapter part of url.
         except:
             chapcount = 1
@@ -180,7 +212,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## For 1, use the second link.
         ## For 2, fetch the crossover page and pull the two categories from there.
         pre_links = soup.find('div',{'id':'pre_story_links'})
-        categories = pre_links.findAll('a',{'class':'xcontrast_txt'})
+        categories = pre_links.find_all('a',{'class':'xcontrast_txt'})
         #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
             # Strangely, the ones with *two* links are the
@@ -219,7 +251,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
 
         grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
-        # for b in grayspan.findAll('button'):
+        # for b in grayspan.find_all('button'):
         #     b.extract()
         metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
         #logger.debug("metatext:(%s)"%metatext)
@@ -258,7 +290,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
         # Published: <span data-xutime='1384358726'>8m ago</span>
-        dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
+        dates = soup.find_all('span',{'data-xutime':re.compile(r'^\d+$')})
         if len(dates) > 1 :
             # updated get set to the same as published upstream if not found.
             self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
@@ -309,11 +341,10 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 img = soup.select_one('img.lazy.cimage')
                 cover_url=img['data-original']
             except:
-                img = soup.select_one('img.cimage:not(.lazy)')
-                if img:
-                    cover_url=img['src']
-            ## Nov 19, 2020, ffnet lazy cover images returning 0 byte
-            ## files.
+                ## Nov 2023 - src is always "/static/images/d_60_90.jpg" now
+                ## Only take cover if there's data-original
+                ## Primary motivator is to prevent unneeded author page hits.
+                pass
             logger.debug("cover_url:%s"%cover_url)
 
             authimg_url = ""
@@ -364,31 +395,37 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             # no selector found, so it's a one-chapter story.
             self.add_chapter(self.story.getMetadata('title'),url)
         else:
-            allOptions = select.findAll('option')
+            allOptions = select.find_all('option')
             for o in allOptions:
-                url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                                 self.story.getMetadata('storyId'),
-                                                 o['value'])
+                ## title URL will be put back on chapter URL during
+                ## normalize_chapterurl() anyway, but also here for
+                ## clarity
+                url = u'https://%s/s/%s/%s/%s' % ( self.getSiteDomain(),
+                                                   self.story.getMetadata('storyId'),
+                                                   o['value'],
+                                                   self.urltitle)
                 # just in case there's tags, like <i> in chapter titles.
                 title = u"%s" % o
                 title = re.sub(r'<[^>]+>','',title)
                 self.add_chapter(title,url)
-
-
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % (url))
 
-        ## AND explicitly put title URL back on chapter URL for fetch
-        ## *only*--normalized chapter URL does NOT have urltitle
-        data = self.get_request(url+self.urltitle)
+        ## title URL was put back on chapter URL during
+        ## normalize_chapterurl()
+        data = self.get_request(url)
 
-        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
+        if "Please email this error message in full to <a href='mailto:" in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
 
         soup = self.make_soup(data)
 
+        ## remove inline ads -- only seen with flaresolverr
+        for adtag in soup.select("div.google-auto-placed"):
+            adtag.decompose()
+
         div = soup.find('div', {'id' : 'storytextp'})
 
         if None == div:

fanficfare/adapters/adapter_fanfictionsfr.py

@@ -0,0 +1,157 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2024 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+from __future__ import absolute_import
+import io
+import logging
+import re
+import zipfile
+
+from bs4 import BeautifulSoup
+# py2 vs py3 transition
+
+from .base_adapter import BaseSiteAdapter, makeDate
+from fanficfare.htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+logger = logging.getLogger(__name__)
+
+
+def getClass():
+    return FanfictionsFrSiteAdapter
+
+
+class FanfictionsFrSiteAdapter(BaseSiteAdapter):
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev', 'fanfictionsfr')
+        self.story.setMetadata('langcode','fr')
+        self.story.setMetadata('language','Français')
+
+        # get storyId from url--url validation guarantees query correct
+        match = re.match(self.getSiteURLPattern(), url)
+        if not match:
+            raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
+
+        story_id = match.group('id')
+        self.story.setMetadata('storyId', story_id)
+        fandom_name = match.group('fandom')
+
+        self._setURL('https://%s/fanfictions/%s/%s/chapters.html' % (self.getSiteDomain(), fandom_name, story_id))
+
+    @staticmethod
+    def getSiteDomain():
+        return 'www.fanfictions.fr'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return 'https://%s/fanfictions/fandom/fanfiction-id/chapters.html' % cls.getSiteDomain()
+
+    def getSiteURLPattern(self):
+        return r'https?://(?:www\.)?fanfictions\.fr/fanfictions/(?P<fandom>[^/]+)/(?P<id>[^/]+)(/chapters.html)?'
+
+    def extractChapterUrlsAndMetadata(self):
+        logger.debug('URL: %s', self.url)
+
+        data = self.get_request(self.url)
+        soup = self.make_soup(data)
+
+        # detect if the fanfiction is 'suspended' (chapters unavailable)
+        alert_div = soup.find('div', id='alertInactiveFic')
+        if alert_div:
+            raise exceptions.FailedToDownload("Failed to download the fanfiction, most likely because it is suspended.")
+
+        title_element = soup.find('h1', itemprop='name')
+        self.story.setMetadata('title', stripHTML(title_element))
+
+        author_div = soup.find('div', itemprop='author')
+        author_name = stripHTML(author_div.a)
+        author_id = author_div.a['href'].split('/')[-1].replace('.html', '')
+
+        self.story.setMetadata('author', author_name)
+        self.story.setMetadata('authorId', author_id)
+
+        published_date_element = soup.find('span', class_='date-distance')
+        published_date_text = published_date_element['data-date']
+        published_date = makeDate(published_date_text, '%Y-%m-%d %H:%M:%S')
+        if published_date:
+            self.story.setMetadata('datePublished', published_date)
+
+        status_element = soup.find('p', title="Statut de la fanfiction").find('span', class_='badge')
+        french_status = stripHTML(status_element)
+        status_translation = {
+            "En cours": "In-Progress",
+            "Terminée": "Completed",
+            "One-shot": "Completed",
+        }
+        self.story.setMetadata('status', status_translation.get(french_status, french_status))
+
+        genre_elements = soup.find('div', title="Format et genres").find_all('span', class_="highlightable")
+        self.story.extendList('genre', [ stripHTML(genre) for genre in genre_elements[1:] ])
+
+        category_elements = soup.find_all('li', class_="breadcrumb-item")
+        self.story.extendList('category', [ stripHTML(category) for category in category_elements[-2].find_all('a') ])
+
+        first_description = soup.find('p', itemprop='abstract')
+        self.setDescription(self.url, first_description)
+
+        chapter_cards = soup.find_all(class_=['card', 'chapter'])
+
+        for chapter_card in chapter_cards:
+            chapter_title_tag = chapter_card.find('h2')
+            if chapter_title_tag:
+                chapter_title = stripHTML(chapter_title_tag)
+                chapter_link = 'https://'+self.getSiteDomain()+chapter_title_tag.find('a')['href']
+
+                # Clean up the chapter title by replacing multiple spaces and newline characters with a single space
+                chapter_title = re.sub(r'\s+', ' ', chapter_title)
+
+                self.add_chapter(chapter_title, chapter_link)
+
+        last_chapter_div = chapter_cards[-1]
+        updated_date_element = last_chapter_div.find('span', class_='date-distance')
+        last_chapter_update_date = updated_date_element['data-date']
+        date = makeDate(last_chapter_update_date, '%Y-%m-%d %H:%M:%S')
+        if date:
+            self.story.setMetadata('dateUpdated', date)
+
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+
+        response, redirection_url = self.get_request_redirected(url)
+
+        if "telecharger_pdf.html" in redirection_url:
+            with zipfile.ZipFile(io.BytesIO(response.encode('latin1'))) as z:
+                # Assuming there's only one text file inside the zip
+                file_list = z.namelist()
+                if len(file_list) != 1:
+                    raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Zip file should contain exactly one text file!" % url)
+                text_filename = file_list[0]
+                with z.open(text_filename) as text_file:
+                    # Decode the text file with windows-1252 encoding
+                    text = text_file.read().decode('windows-1252')
+                    return text.replace("\r\n", "<br>\r\n")
+        else:
+            soup = self.make_soup(response)
+
+            div_content = soup.find('div', id='readarea')
+            if div_content is None:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+            return self.utf8FromSoup(url, div_content)

fanficfare/adapters/adapter_fanfiktionde.py

@@ -134,7 +134,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
-        for chapter in soup.find('select').findAll('option'):
+        for chapter in soup.find('select').find_all('option'):
             self.add_chapter(chapter,'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value'])
 
         ## title="Wörter" failed with max_zalgo:1
@@ -163,11 +163,11 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         except e:
             logger.debug("Failed to find native status:%s"%e)
 
-        if head.find('span',title='Fertiggestellt'):
+        if head.find('span',title='fertiggestellt'):
             self.story.setMetadata('status', 'Completed')
-        elif head.find('span',title='Pausiert'):
+        elif head.find('span',title='pausiert'):
             self.story.setMetadata('status', 'Paused')
-        elif head.find('span',title='Abgebrochen'):
+        elif head.find('span',title='abgebrochen'):
             self.story.setMetadata('status', 'Cancelled')
         else:
             self.story.setMetadata('status', 'In-Progress')
@@ -181,13 +181,13 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
 
         # #find metadata on the author's page
         # asoup = self.make_soup(self.get_request("https://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
-        # tr=asoup.findAll('tr')
+        # tr=asoup.find_all('tr')
         # for i in range(1,len(tr)):
         #     a = tr[i].find('a')
         #     if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
         #         break
 
-        # td = tr[i].findAll('td')
+        # td = tr[i].find_all('td')
         # self.story.addToList('category',stripHTML(td[2]))
         # self.story.setMetadata('rating', stripHTML(td[5]))
         # self.story.setMetadata('numWords', stripHTML(td[6]))
@@ -204,7 +204,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         soup = self.make_soup(self.get_request(url))
 
         div = soup.find('div', {'id' : 'storytext'})
-        for a in div.findAll('script'):
+        for a in div.find_all('script'):
             a.extract()
 
         if None == div:

fanficfare/adapters/adapter_fastnovelnet.py

@@ -1,151 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-####################################################################################################
-### Adapted by Rikkit on November 7. 2017
-###=================================================================================================
-### Tested with Calibre
-####################################################################################################
-
-from __future__ import absolute_import
-import logging
-import re
-# py2 vs py3 transition
-
-from .base_adapter import BaseSiteAdapter, makeDate
-
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-logger = logging.getLogger(__name__)
-
-def getClass():
-    ''' Initializing the class '''
-    return FastNovelNetAdapter
-
-class FastNovelNetAdapter(BaseSiteAdapter):
-    ''' Adapter for FASTNOVEL.net '''
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.story.setMetadata('siteabbrev', 'fstnvl')
-
-        self.dateformat = '%d/%m/%Y'
-
-        # get storyId from url--url validation guarantees query correct
-        match = re.match(self.getSiteURLPattern(), url)
-        if not match:
-            raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
-
-        story_id = match.group('id')
-        self.story.setMetadata('storyId', story_id)
-        self._setURL('https://%s/%s/' % (self.getSiteDomain(), story_id))
-
-
-    @staticmethod
-    def getSiteDomain():
-        return 'fastnovel.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://fastnovel.net/a-story-name-id"
-
-    def getSiteURLPattern(self):
-        # https://fastnovel.net/ultimate-scheming-system-158/
-        return r"https?://fastnovel\.net/(?P<id>[^/]+)"
-
-    ## Normalized chapter URLs by changing old titlenum part to be
-    ## same as storyId.
-    def normalize_chapterurl(self,url):
-        # https://fastnovel.net/cultivation-chat-group8-29/chapter-25206.html
-        return re.sub(r"\.net/.*(?P<keep>/chapter-\d+.html)",
-                      r".net/"+self.story.getMetadata('storyId')+r"\g<keep>",url)
-
-    def extractChapterUrlsAndMetadata(self):
-        logger.debug('URL: %s', self.url)
-
-        (data,rurl) = self.get_request_redirected(self.url)
-        if rurl != self.url:
-            match = re.match(self.getSiteURLPattern(), rurl)
-            if not match:
-                ## shouldn't happen, but in case it does...
-                raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
-
-            story_id = match.group('id')
-            self.story.setMetadata('storyId', story_id)
-            self._setURL('https://%s/%s/' % (self.getSiteDomain(), story_id))
-            logger.debug("set to redirected url:%s"%self.url)
-
-        soup = self.make_soup(data)
-
-        self.story.setMetadata('title', soup.find('h1').string)
-
-        for li in soup.select('.meta-data li'):
-            label = li.select_one('label')
-            if not label:
-                continue
-
-            if label.string == "Author:":
-                for a in li.select('a'):
-                    self.story.setMetadata('authorId', a["href"].split('/')[2])
-                    self.story.setMetadata('authorUrl','https://'+self.host+a["href"])
-                    self.story.setMetadata('author', a["title"])
-
-            if label.string == "Genre:":
-                for a in li.select('a'):
-                    self.story.addToList('genre',a["title"])
-
-            if label.string == "Status:":
-                if li.select_one('strong').string.strip() == "Completed":
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if label.string == "Last updated:":
-                dateUpd = label.next_sibling.strip()
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(dateUpd), self.dateformat))
-
-        coverurl = soup.select_one('div.book-cover')["data-original"]
-        if coverurl != "https://fastnovel.net/images/novel/default.jpg":
-            self.setCoverImage(self.url, coverurl)
-
-        tags = soup.select_one('.tags')
-        if tags:
-            for a in tags.select("li.tag-item a"):
-                self.story.addToList('tags', a["title"])
-            # extract tags, because it inside description
-            tags.extract()
-
-        # remove title from description
-        soup.select_one('.film-content h3').extract()
-        desc = soup.select_one('.film-content').extract()
-        self.setDescription(self.url, desc)
-
-        for book in soup.select("#list-chapters .book"):
-            volume = book.select_one('.title a').string
-            for a in book.select(".list-chapters a.chapter"):
-                title = volume + " " + stripHTML(a)
-                self.add_chapter(title, 'https://' + self.host + a["href"])
-
-    def getChapterText(self, url):
-        data = self.get_request(url)
-        soup = self.make_soup(data)
-
-        story = soup.select_one('#chapter-body')
-        if not story:
-            raise exceptions.FailedToDownload(
-                "Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url, story)

fanficfare/adapters/adapter_ficbooknet.py

@@ -15,16 +15,16 @@
 # limitations under the License.
 #
 
-from __future__ import absolute_import
-import datetime
+from __future__ import absolute_import,unicode_literals
+# import datetime
 import logging
-logger = logging.getLogger(__name__)
+import json
 import re
-from .. import translit
+# from .. import translit
 
 
 from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
+from .. import exceptions# as exceptions
 
 # py2 vs py3 transition
 
@@ -58,7 +58,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %m %Y"
+        self.dateformat = u"%d %m %Y г., %H:%M"
 
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -67,17 +67,33 @@ class FicBookNetAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/readfic/12345 https://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"
+        return "https://"+cls.getSiteDomain()+"/readfic/12345 https://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content https://"+cls.getSiteDomain()+"/readfic/578de1cd-a8b4-7ff1-aa49-750426508b82 https://"+cls.getSiteDomain()+"/readfic/578de1cd-a8b4-7ff1-aa49-750426508b82/94793742#part_content"
 
     def getSiteURLPattern(self):
-        return r"https?://"+re.escape(self.getSiteDomain()+"/readfic/")+r"\d+"
+        return r"https?://"+re.escape(self.getSiteDomain()+"/readfic/")+r"[\d\-a-zA-Z]+"
+
+    def performLogin(self,url,data):
+        params = {}
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+
+        logger.debug("Try to login in as (%s)" % params['login'])
+        d = self.post_request('https://' + self.getSiteDomain() + '/login_check_static',params,usecache=False)
+
+        if 'Войти используя аккаунт на сайте' in d:
+            raise exceptions.FailedToLogin(url,params['login'])
+
+        return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
+    def extractChapterUrlsAndMetadata(self,get_cover=True):
         url=self.url
         logger.debug("URL: "+url)
         data = self.get_request(url)
-
         soup = self.make_soup(data)
 
         adult_div = soup.find('div',id='adultCoverWarning')
@@ -87,9 +103,11 @@ class FicBookNetAdapter(BaseSiteAdapter):
             else:
                 raise exceptions.AdultCheckRequired(self.url)
 
-
         ## Title
-        a = soup.find('section',{'class':'chapter-info'}).find('h1')
+        try:
+            a = soup.find('section',{'class':'chapter-info'}).find('h1')
+        except AttributeError:
+            raise exceptions.FailedToDownload("Error collecting meta: %s!  Missing required element!" % url)
         # kill '+' marks if present.
         sup = a.find('sup')
         if sup:
@@ -99,40 +117,12 @@ class FicBookNetAdapter(BaseSiteAdapter):
 
         # Find authorid and URL from... author url.
         # assume first avatar-nickname -- there can be a second marked 'beta'.
-        a = soup.find('a',{'class':'creator-nickname'})
+        a = soup.find('a',{'class':'creator-username'})
         self.story.setMetadata('authorId',a.text) # Author's name is unique
         self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
         self.story.setMetadata('author',a.text)
         logger.debug("Author: (%s)"%self.story.getMetadata('author'))
 
-        # Find the chapters:
-        pubdate = None
-        chapters = soup.find('ul', {'class' : 'list-of-fanfic-parts'})
-        if chapters != None:
-            for chapdiv in chapters.findAll('li', {'class':'part'}):
-                chapter=chapdiv.find('a',href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+r"/\d+#part_content$"))
-                churl='https://'+self.host+chapter['href']
-                self.add_chapter(chapter,churl)
-
-                datespan = chapdiv.find('span')
-                if pubdate == None and datespan:
-                    pubdate = translit.translit(stripHTML(datespan))
-                update = translit.translit(stripHTML(datespan))
-        else:
-            self.add_chapter(self.story.getMetadata('title'),url)
-            self.story.setMetadata('numChapters',1)
-            pubdate=translit.translit(stripHTML(soup.find('div',{'class':'title-area'}).find('span')))
-            update=pubdate
-
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-
-        if not ',' in pubdate:
-            pubdate=datetime.date.today().strftime(self.dateformat)
-        if not ',' in update:
-            update=datetime.date.today().strftime(self.dateformat)
-        pubdate=pubdate.split(',')[0]
-        update=update.split(',')[0]
-
         fullmon = {"yanvarya":"01", u"января":"01",
            "fievralya":"02", u"февраля":"02",
            "marta":"03", u"марта":"03",
@@ -146,31 +136,50 @@ class FicBookNetAdapter(BaseSiteAdapter):
            "noyabrya":"11", u"ноября":"11",
            "diekabrya":"12", u"декабря":"12" }
 
-        for (name,num) in fullmon.items():
-            if name in pubdate:
-                pubdate = pubdate.replace(name,num)
-            if name in update:
-                update = update.replace(name,num)
+        # Find the chapters:
+        pubdate = None
+        chapters = soup.find('ul', {'class' : 'list-of-fanfic-parts'})
+        if chapters is not None:
+            for chapdiv in chapters.find_all('li', {'class':'part'}):
+                chapter=chapdiv.find('a',href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+r"/\d+#part_content$"))
+                churl='https://'+self.host+chapter['href']
 
-        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
-        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
+                # Find the chapter dates.
+                date_str = chapdiv.find('span', {'title': True})['title'].replace(u"\u202fг. в", "")
+                for month_name, month_num in fullmon.items():
+                    date_str = date_str.replace(month_name, month_num)
+                chapterdate = makeDate(date_str,self.dateformat)
+                self.add_chapter(chapter,churl,
+                                 {'date':chapterdate.strftime(self.getConfig("datechapter_format",self.getConfig("datePublished_format",self.dateformat)))})
+
+                if pubdate is None and chapterdate:
+                    pubdate = chapterdate
+                update = chapterdate
+        else:
+            self.add_chapter(self.story.getMetadata('title'),url)
+            date_str = soup.find('div', {'class' : 'part-date'}).find('span', {'title': True})['title'].replace(u"\u202fг. в", "")
+            for month_name, month_num in fullmon.items():
+                date_str = date_str.replace(month_name, month_num)
+            pubdate = update = makeDate(date_str,self.dateformat)
+
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+
+        self.story.setMetadata('dateUpdated', update)
+        self.story.setMetadata('datePublished', pubdate)
         self.story.setMetadata('language','Russian')
 
-        ## after site change, I don't see word count anywhere.
-        # pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
-        # pr='https://'+self.host+pr['href']
-        # pr = self.make_soup(self.get_request(pr))
-        # pr=pr.findAll('div', {'class' : 'part_text'})
-        # i=0
-        # for part in pr:
-        #     i=i+len(stripHTML(part).split(' '))
-        # self.story.setMetadata('numWords', unicode(i))
+        dlinfo = soup.select_one('header.d-flex.flex-column.gap-12.word-break')
 
-
-        dlinfo = soup.find('div',{'class':'fanfic-main-info'})
+        series_label = dlinfo.select_one('div.description.word-break').find('strong', string='Серия:')
+        logger.debug('Series: %s'%str(series_label))
+        if series_label:
+            series_div = series_label.find_next_sibling("div")
+            # No accurate series number as for that, additional request needs to be made
+            self.setSeries(stripHTML(series_div.a), 1)
+            self.story.setMetadata('seriesUrl','https://' + self.getSiteDomain() + series_div.a.get('href'))
 
         i=0
-        fandoms = dlinfo.find('div').findAll('a', href=re.compile(r'/fanfiction/\w+'))
+        fandoms = dlinfo.select_one('div:not([class])').find_all('a', href=re.compile(r'/fanfiction/\w+'))
         for fandom in fandoms:
             self.story.addToList('category',fandom.string)
             i=i+1
@@ -179,13 +188,16 @@ class FicBookNetAdapter(BaseSiteAdapter):
 
         tags = soup.find('div',{'class':'tags'})
         if tags:
-            for genre in tags.findAll('a',href=re.compile(r'/tags/')):
+            for genre in tags.find_all('a',href=re.compile(r'/tags/')):
                 self.story.addToList('genre',stripHTML(genre))
 
-        ratingdt = dlinfo.find('strong',{'class':re.compile(r'badge-rating-.*')})
-        self.story.setMetadata('rating', stripHTML(ratingdt.find_next('span')))
+        logger.debug("category: (%s)"%self.story.getMetadata('category'))
+        logger.debug("genre: (%s)"%self.story.getMetadata('genre'))
 
-        # meta=table.findAll('a', href=re.compile(r'/ratings/'))
+        ratingdt = dlinfo.find('div',{'class':re.compile(r'badge-rating-.*')})
+        self.story.setMetadata('rating', stripHTML(ratingdt.find('span')))
+
+        # meta=table.find_all('a', href=re.compile(r'/ratings/'))
         # i=0
         # for m in meta:
         #     if i == 0:
@@ -198,12 +210,17 @@ class FicBookNetAdapter(BaseSiteAdapter):
         #     elif i == 2:
         #         self.story.addToList('warnings', m.find('b').text)
 
-        if dlinfo.find('span', {'class':'badge-status-finished'}):
+        if dlinfo.find('div', {'class':'badge-status-finished'}):
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        paircharsdt = soup.find('strong',text='Пэйринг и персонажи:')
+        try:
+            self.story.setMetadata('universe', stripHTML(dlinfo.find('a', href=re.compile('/fandom_universe/'))))
+        except AttributeError:
+            pass
+
+        paircharsdt = soup.find('strong',string='Пэйринг и персонажи:')
         # site keeps both ships and indiv chars in /pairings/ links.
         if paircharsdt:
             for paira in paircharsdt.find_next('div').find_all('a', href=re.compile(r'/pairings/')):
@@ -216,8 +233,98 @@ class FicBookNetAdapter(BaseSiteAdapter):
                     self.story.addToList('characters',stripHTML(paira))
 
         summary=soup.find('div', itemprop='description')
-        self.setDescription(url,summary)
-        #self.story.setMetadata('description', summary.text)
+        if summary:
+            # Fix for the text not displaying properly
+            summary['class'].append('part_text')
+            self.setDescription(url,summary)
+            #self.story.setMetadata('description', summary.text)
+
+        stats = soup.find('div', {'class':'hat-actions-container'})
+        targetdata = stats.find_all('span', {'class' : 'main-info'})
+        for data in targetdata:
+            svg_class = data.find('svg')['class'][1] if data.find('svg') else None
+            value = int(stripHTML(data)) if stripHTML(data).isdigit() else 0
+
+            if svg_class == 'ic_thumbs-up' and value > 0:
+                self.story.setMetadata('likes', value)
+                #logger.debug("likes: (%s)"%self.story.getMetadata('likes'))
+            elif svg_class == 'ic_bubble-dark' and value > 0:
+                self.story.setMetadata('reviews', value)
+                #logger.debug("reviews: (%s)"%self.story.getMetadata('reviews'))
+            elif svg_class == 'ic_bookmark' and value > 0:
+                self.story.setMetadata('numCollections', value)
+                logger.debug("numCollections: (%s)"%self.story.getMetadata('numCollections'))
+
+        # Grab the amount of pages and words
+        targetpages = soup.find('strong',string='Размер:').find_next('div')
+        if targetpages:
+            targetpages_text = re.sub(r"(?<!\,)\s| ", "", targetpages.text, flags=re.UNICODE | re.MULTILINE)
+
+            pages_raw = re.search(r'(\d+)(?:страницы|страниц)', targetpages_text, re.UNICODE)
+            pages = int(pages_raw.group(1))
+            if pages > 0:
+                self.story.setMetadata('pages', pages)
+                logger.debug("pages: (%s)"%self.story.getMetadata('pages'))
+
+            numWords_raw = re.search(r"(\d+)(?:слова|слов)", targetpages_text, re.UNICODE)
+            numWords = int(numWords_raw.group(1))
+            if numWords > 0:
+                self.story.setMetadata('numWords', numWords)
+                logger.debug("numWords: (%s)"%self.story.getMetadata('numWords'))
+
+        # Grab FBN Category
+        class_tag = soup.select_one('div[class^="badge-with-icon direction"]').find('span', {'class' : 'badge-text'}).text
+        if class_tag:
+            self.story.setMetadata('classification',class_tag)
+            #logger.debug("classification: (%s)"%self.story.getMetadata('classification'))
+
+        # Find dedication.
+        ded = soup.find('div', {'class' : 'js-public-beta-dedication'})
+        if ded:
+            ded['class'].append('part_text')
+            self.story.setMetadata('dedication',ded)
+
+        # Find author comment
+        comm = soup.find('div', {'class' : 'js-public-beta-author-comment'})
+        if comm:
+            comm['class'].append('part_text')
+            self.story.setMetadata('authorcomment',comm)
+
+        follows = stats.find('fanfic-follow-button')[':follow-count']
+        if int(follows) > 0:
+            self.story.setMetadata('follows', int(follows))
+            logger.debug("follows: (%s)"%self.story.getMetadata('follows'))
+
+        # Grab the amount of awards
+        numAwards = 0
+        try:
+            awards = soup.find('fanfic-reward-list')[':initial-fic-rewards-list']
+            award_list = json.loads(awards)
+            numAwards = int(len(award_list))
+            # Grab the awards, but if multiple awards have the same name, only one will be kept; only an issue with hundreds of them.
+            self.story.extendList('awards', [str(award['user_text']) for award in award_list])
+            #logger.debug("awards (%s)"%self.story.getMetadata('awards'))
+        except (TypeError, KeyError):
+            logger.debug("Could not grab the awards")
+
+        if numAwards > 0:
+            self.story.setMetadata('numAwards', numAwards)
+            logger.debug("Num Awards (%s)"%self.story.getMetadata('numAwards'))
+
+        if get_cover:
+            cover = soup.find('fanfic-cover', {'class':"jsVueComponent"})
+            if cover is not None:
+                self.setCoverImage(url,cover['src-original'])
+
+    def replace_formatting(self,tag):
+        tname = tag.name
+        ## operating on plain text because BS4 is hard to work on
+        ## text with.
+        ## stripHTML() discards whitespace around other tags, like <i>
+        txt = tag.get_text()
+        txt = txt.replace("\n","<br/>")
+        soup = self.make_soup("<"+tname+">"+txt+"</"+tname+">")
+        return soup.find(tname)
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):
@@ -227,10 +334,60 @@ class FicBookNetAdapter(BaseSiteAdapter):
         soup = self.make_soup(self.get_request(url))
 
         chapter = soup.find('div', {'id' : 'content'})
-        if chapter == None: ## still needed?
+        if chapter is None: ## still needed?
             chapter = soup.find('div', {'class' : 'public_beta_disabled'})
 
-        if None == chapter:
+        if chapter is None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
+        ## ficbook uses weird CSS white-space: pre-wrap; for
+        ## paragraphing.  Doesn't work with txt output
+        if 'part_text' in chapter['class'] and self.getConfig('replace_text_formatting'):
+            ## copy classes, except part_text
+            divclasses = chapter['class']
+            divclasses.remove('part_text')
+            chapter = self.replace_formatting(chapter)
+            chapter['class'] = divclasses
+
+        exclude_notes=self.getConfigList('exclude_notes')
+        if 'headnotes' not in exclude_notes:
+            # Find the headnote
+            head_note = soup.select_one("div.part-comment-top div.js-public-beta-comment-before")
+            if head_note:
+                # Create the structure for the headnote
+                head_notes_div_tag = soup.new_tag('div', attrs={'class': 'fff_chapter_notes fff_head_notes'})
+                head_b_tag = soup.new_tag('b')
+                head_b_tag.string = 'Примечания:'
+                if 'text-preline' in head_note['class'] and self.getConfig('replace_text_formatting'):
+                    head_blockquote_tag = self.replace_formatting(head_note)
+                    head_blockquote_tag.name = 'blockquote'
+                else:
+                    head_blockquote_tag = soup.new_tag('blockquote')
+                    head_blockquote_tag.string = stripHTML(head_note)
+                head_notes_div_tag.append(head_b_tag)
+                head_notes_div_tag.append(head_blockquote_tag)
+                # Prepend the headnotes to the chapter, <hr> to mimic the site
+                chapter.insert(0, head_notes_div_tag)
+                chapter.insert(1, soup.new_tag('hr'))
+
+        if 'footnotes' not in exclude_notes:
+            # Find the endnote
+            end_note = soup.select_one("div.part-comment-bottom div.js-public-beta-comment-after")
+            if end_note:
+                # Create the structure for the footnote
+                end_notes_div_tag = soup.new_tag('div', attrs={'class': 'fff_chapter_notes fff_foot_notes'})
+                end_b_tag = soup.new_tag('b')
+                end_b_tag.string = 'Примечания:'
+                if 'text-preline' in end_note['class'] and self.getConfig('replace_text_formatting'):
+                    end_blockquote_tag = self.replace_formatting(end_note)
+                    end_blockquote_tag.name = 'blockquote'
+                else:
+                    end_blockquote_tag = soup.new_tag('blockquote')
+                    end_blockquote_tag.string = stripHTML(end_note)
+                end_notes_div_tag.append(end_b_tag)
+                end_notes_div_tag.append(end_blockquote_tag)
+                # Append the endnotes to the chapter, <hr> to mimic the site
+                chapter.append(soup.new_tag('hr'))
+                chapter.append(end_notes_div_tag)
+
         return self.utf8FromSoup(url,chapter)

fanficfare/adapters/adapter_fictionalleyarchiveorg.py

@@ -177,7 +177,7 @@ class FictionAlleyArchiveOrgSiteAdapter(BaseSiteAdapter):
             elif key == 'Words':
                 self.story.setMetadata('numWords',val)
 
-        summary = soup.find('dt',text='Story Summary:')
+        summary = soup.find('dt',string='Story Summary:')
         if summary:
             summary = summary.find_next_sibling('dd')
             summary.name='div'
@@ -201,16 +201,16 @@ class FictionAlleyArchiveOrgSiteAdapter(BaseSiteAdapter):
         # epubutils.py
         # Yes, this still applies to fictionalley-archive.
 
-        for tag in chaptext.findAll('head') + chaptext.findAll('meta') + chaptext.findAll('script'):
+        for tag in chaptext.find_all('head') + chaptext.find_all('meta') + chaptext.find_all('script'):
             tag.extract()
 
-        for tag in chaptext.findAll('body') + chaptext.findAll('html'):
+        for tag in chaptext.find_all('body') + chaptext.find_all('html'):
             tag.name = 'div'
 
         if self.getConfig('include_author_notes'):
             row = chaptext.find_previous_sibling('div',class_='row')
             logger.debug(row)
-            andt = row.find('dt',text="Author's Note:")
+            andt = row.find('dt',string="Author's Note:")
             logger.debug(andt)
             if andt:
                 chaptext.insert(0,andt.parent.extract())

fanficfare/adapters/adapter_fictionhuntcom.py

@@ -235,7 +235,7 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
         # logger.debug(data)
         self.story.setMetadata('title',stripHTML(soup.find('h1',{'class':'Story__title'})))
 
-        summhead = soup.find('h5',text='Summary')
+        summhead = soup.find('h5',string='Summary')
         self.setDescription(url,summhead.find_next('div'))
 
         ## author:
@@ -244,12 +244,12 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorUrl',autha['href'])
         self.story.setMetadata('author',autha.string)
 
-        updlab = soup.find('label',text='Last Updated:')
+        updlab = soup.find('label',string='Last Updated:')
         if updlab:
             update = updlab.find_next('time')['datetime']
             self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
 
-        publab = soup.find('label',text='Published:')
+        publab = soup.find('label',string='Published:')
         if publab:
             pubdate = publab.find_next('time')['datetime']
             self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
@@ -280,7 +280,7 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
         # logger.debug(meta)
 
         # Find original ffnet URL
-        a = soup.find('a', text="Source")
+        a = soup.find('a', string="Source")
         self.story.setMetadata('origin',stripHTML(a))
         self.story.setMetadata('originUrl',a['href'])
 
@@ -321,8 +321,8 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
         # else:
         #     self.story.addToList('category', 'Original')
 
-        for chapa in soup.select('ul.StoryContents__chapters a'):
-            self.add_chapter(stripHTML(chapa.find('span',{'class':'chapter-title'})),chapa['href'])
+        for chapli in soup.select('ul.StoryContents__chapters li'):
+            self.add_chapter(stripHTML(chapli.select_one('span.chapter-title')),chapli.select_one('a')['href'])
 
         if self.num_chapters() == 0:
             raise exceptions.FailedToDownload("Story at %s has no chapters." % self.url)

fanficfare/adapters/adapter_fictionlive.py

@@ -27,6 +27,7 @@
 # per-user achivement tracking with fancy achievement-get animations
 # story scripting (shows script tags visible in the text, not computed values or input fields)
 
+import re
 import json
 from datetime import datetime
 
@@ -35,6 +36,8 @@ import itertools
 import logging
 logger = logging.getLogger(__name__)
 
+# __package__ = 'fanficfare.adapters' # fixes dev issues with unknown package base
+
 from .base_adapter import BaseSiteAdapter
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
@@ -52,6 +55,8 @@ class FictionLiveAdapter(BaseSiteAdapter):
         self.story_id = self.parsedUrl.path.split('/')[3]
         self.story.setMetadata('storyId', self.story_id)
 
+        self.chapter_id_to_api = {}
+
         # normalize URL. omits title in the url
         self._setURL("https://fiction.live/stories//{s_id}".format(s_id = self.story_id));
 
@@ -65,7 +70,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
 
     def getSiteURLPattern(self):
         # I'd like to thank regex101.com for helping me screw this up less
-        return r"https?://(beta\.)?fiction\.live/[^/]*/[^/]*/([a-zA-Z0-9\-]+)(/(home)?)?"
+        return r"https?://(beta\.)?fiction\.live/[^/]*/[^/]*/([a-zA-Z0-9\-]+)(/(home)?)?$"
 
     @classmethod
     def getSiteExampleURLs(cls):
@@ -74,11 +79,29 @@ class FictionLiveAdapter(BaseSiteAdapter):
                 +"https://fiction.live/Sci-fi/Example-Story-With-URL-Genre/17CharacterIDhere/ "
                 +"https://fiction.live/stories/Example-Story-With-UUID/00000000-0000-4000-0000-000000000000/")
 
+    @classmethod
+    def get_section_url(cls,url):
+        ## minimal URL used for section names in INI and reject list
+        ## for comparison
+        # logger.debug("pre--url:%s"%url)
+        url = re.sub(r"https?://(beta\.)?fiction\.live/[^/]*/[^/]*/(?P<id>[a-zA-Z0-9\-]+)(/(home)?)?$",r'https://fiction.live/stories//\g<id>',url)
+        # logger.debug("post-url:%s"%url)
+        return url
+
     def parse_timestamp(self, timestamp):
         # fiction.live date format is unix-epoch milliseconds. not a good fit for fanficfare's makeDate.
         # doesn't use a timezone object and returns tz-naive datetimes. I *think* I can leave the rest to fanficfare
         return datetime.fromtimestamp(timestamp / 1000.0, None)
 
+    def img_url_trans(self,imgurl):
+        "Apparently site changed cdn URLs for images more than once."
+        # logger.debug("pre--imgurl:%s"%imgurl)
+        imgurl = re.sub(r'(\w+)\.cloudfront\.net',r'cdn6.fiction.live/file/fictionlive',imgurl)
+        imgurl = re.sub(r'www\.filepicker\.io/api/file/(\w+)',r'cdn4.fiction.live/fp/\1',imgurl)
+        imgurl = re.sub(r'cdn[34].fiction.live/(.+)',r'cdn6.fiction.live/file/fictionlive/\1',imgurl)
+        # logger.debug("post-imgurl:%s"%imgurl)
+        return imgurl
+
     def doExtractChapterUrlsAndMetadata(self, get_cover=True):
 
         metadata_url = "https://fiction.live/api/node/{s_id}/"
@@ -150,7 +173,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
 
         tags = data['ta'] if 'ta' in data else []
 
-        if (self.story.getMetadata('rating') in {"nsfw", "adult"} or 'smut' in tags) and \
+        if (self.story.getMetadataRaw('rating') in {"nsfw", "adult"} or 'smut' in tags) and \
            not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
 
@@ -186,7 +209,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
             if show_nsfw_cover_images or not nsfw_cover:
                 coverUrl = data['i'][0]
                 self.setCoverImage(self.url, coverUrl)
-                self.story.setMetadata('cover_image', "<a href=\"" + coverUrl + "\" />") # TODO: is this needed?
 
         # gonna need these later for adding details to achievement-granting links in the text
         try:
@@ -219,6 +241,17 @@ class FictionLiveAdapter(BaseSiteAdapter):
             a, b = itertools.tee(iterable, 2)
             next(b, None)
             return list(zip(a, b))
+        
+        def map_chap_ids_to_api(chapter_ids, route_ids, times):
+            for index, bounds in enumerate(times):
+                start, end = bounds
+                end -= 1
+                chapter_url = chunkrange_url.format(s_id = data['_id'], start = start, end = end)
+                self.chapter_id_to_api[chapter_ids[index]] = chapter_url
+            
+            for route_id in route_ids:
+                chapter_url = route_chunkrange_url.format(c_id = route_id)
+                self.chapter_id_to_api[route_id] = chapter_url
 
         ## first thing to do is seperate out the appendices
         appendices, maintext, routes = [], [], []
@@ -240,22 +273,25 @@ class FictionLiveAdapter(BaseSiteAdapter):
         ## main-text chapter extraction processing. *should* now handle all the edge cases.
         ## relies on fanficfare ignoring empty chapters!
 
-        titles = [c['title'] for c in maintext]
-        titles = ["Home"] + titles
+        titles = ["Home"] + [c['title'] for c in maintext]
+        chapter_ids = ['home'] + [c['id'] for c in maintext]
+        times = [data['ct']] + [c['ct'] for c in maintext] + [self.most_recent_chunk + 2] # need to be 1 over, and add_url etc does -1
+        times = pair(times)
 
-        times = [c['ct'] for c in maintext]
-        times = [data['ct']] + times + [self.most_recent_chunk + 2] # need to be 1 over, and add_url etc does -1
+        if self.getConfig('include_appendices', True): # Add appendices after main text if desired
+            titles = titles + ["Appendix: " + a['title'][9:] for a in appendices]
+            chapter_ids = chapter_ids + [a['id'] for a in appendices]
+            times = times + [(a['ct'], a['ct'] + 2) for a in appendices]
+
+        route_ids = [r['id'] for r in routes]
+
+        map_chap_ids_to_api(chapter_ids, route_ids, times) # Map chapter ids to API URLs for use when comparing the two
 
         # doesn't actually run without the call to list.
-        list(map(add_chapter_url, titles, pair(times)))
-
-        for a in appendices: # add appendices afterwards
-            chapter_start = a['ct']
-            chapter_title = "Appendix: " + a['title'][9:] # 'Appendix: ' rather than '#special' at beginning of name
-            add_chapter_url(chapter_title, (chapter_start, chapter_start + 2)) # 1 msec range = this one chunk only
+        list(map(add_chapter_url, titles, times))
 
         for r in routes:  # add route at the end, after appendices
-            route_id = r['id']  # to get route chapter content, the route id is needed, not the timestamp
+            route_id = r['id'] # to get route chapter content, the route id is needed, not the timestamp
             chapter_title = "Route: " + r['title']  # 'Route: ' at beginning of name, since it's a multiroute chapter
             add_route_chapter_url(chapter_title, route_id)
 
@@ -285,7 +321,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
 
             text += "<div>" # chapter chunks aren't always well-delimited in their contents
 
-            # appendix chunks are mixed in with other things 
+            # appendix chunks are mixed in with other things
             if not getting_appendix and 't' in chunk and chunk['t'].startswith("#special"): # t = title = bookmark
                 continue
 
@@ -300,7 +336,8 @@ class FictionLiveAdapter(BaseSiteAdapter):
 
             text += "</div><br />\n"
 
-        return text
+        ## soup to repair the most egregious HTML errors.
+        return self.utf8FromSoup(url,self.make_soup(text))
 
     ### everything from here out is chunk data handling.
 
@@ -317,8 +354,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
         if self.achievements:
             soup = self.append_achievments(soup)
 
-        # utf8FromSoup does important processing e.g. sanitization and imageurl extraction
-        return self.utf8FromSoup(self.url, soup)
+        return str(soup)
 
     def add_spoiler_legends(self, soup):
         # find spoiler links and change link-anchor block to legend block
@@ -398,7 +434,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
                     #   so let's just ignore non-int values here
                     if not isinstance(v, int):
                         continue
-                    if 0 <= v <= len(choices):
+                    if 0 <= v < len(choices):
                         output[v] += 1
             return output
 
@@ -482,8 +518,10 @@ class FictionLiveAdapter(BaseSiteAdapter):
         # now matches the site and does *not* include dicerolls as posts!
         num_votes = str(len(posts)) + " posts" if len(posts) != 0 else "be the first to post."
 
+        posts_title = chunk['b'] if 'b' in chunk else "Reader Posts"
+
         output = ""
-        output += u"<h4><span>Reader Posts — <small> Posting " + closed
+        output += u"<h4><span>" + posts_title + " — <small> Posting " + closed
         output += u" — " + num_votes + "</small></span></h4>\n"
 
         ## so. a voter can roll with their post. these rolls are in a seperate dict, but have the **same uid**.
@@ -509,6 +547,35 @@ class FictionLiveAdapter(BaseSiteAdapter):
 
         return output
 
+    def normalize_chapterurl(self, url):
+        if url.startswith(r'https://fiction.live/api/anonkun/chapters'):
+            return url
+        
+        pattern = None
+
+        if url.startswith(r'https://fiction.live/api/anonkun/route'):
+            pattern = r"https?://(?:beta\.)?fiction\.live/[^/]*/[^/]*/[a-zA-Z0-9]+/routes/([a-zA-Z0-9]+)"
+        elif url.startswith(r'https://fiction.live/'):
+            pattern = r"https?://(?:beta\.)?fiction\.live/[^/]*/[^/]*/[a-zA-Z0-9]+/[^/]*(/[a-zA-Z0-9]+|home)"
+            # regex101 rocks
+
+        if not pattern:
+            return url
+
+        match = re.match(pattern, url)
+        if not match:
+            return url
+        
+        chapter_id = match.group(1)
+
+        if chapter_id.startswith('/'):
+            chapter_id = chapter_id[1:]
+
+        if chapter_id and chapter_id in self.chapter_id_to_api:
+            return self.chapter_id_to_api[chapter_id]
+        
+        return url
+    
     def format_unknown(self, chunk):
         raise NotImplementedError("Unknown chunk type ({}) in fiction.live story.".format(chunk))
 
@@ -523,5 +590,5 @@ class FictionLiveAdapter(BaseSiteAdapter):
 
 # TODO: support chapter urls for single-chapter / chapter-range downloads
 # complicated -- urls for getChapterText are API urls generated by add_chapters, not the public/website ones
-# in particular, may need more API reversing to figure out how to get the *end* of the chunk range 
+# in particular, may need more API reversing to figure out how to get the *end* of the chunk range
 # find in 'bm' in the metadata?

fanficfare/adapters/adapter_fictionmaniatv.py

@@ -23,7 +23,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
     SITE_ABBREVIATION = 'fmt'
     SITE_DOMAIN = 'fictionmania.tv'
 
-    BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
+    BASE_URL = 'https://' + SITE_DOMAIN + '/stories/'
     READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
     DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
 
@@ -40,10 +40,6 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
         self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
         self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
 
-        # Always single chapters, probably should use the Anthology feature to
-        # merge chapters of a story
-        self.story.setMetadata('numChapters', 1)
-
     @staticmethod
     def getSiteDomain():
         return FictionManiaTVAdapter.SITE_DOMAIN
@@ -53,7 +49,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
         return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
 
     def getSiteURLPattern(self):
-        return r'https?' + re.escape(self.BASE_URL[len('http'):]) + r'(readtextstory|readhtmlstory|readxstory|details)\.html\?storyID=\d+$'
+        return r'https?' + re.escape(self.BASE_URL[len('https'):]) + r'(readtextstory|readhtmlstory|readxstory|details)\.html\?storyID=\d+$'
 
     def extractChapterUrlsAndMetadata(self):
         url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
@@ -110,7 +106,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
                 self.story.setMetadata('rating', value)
 
             elif key == 'Complete':
-                self.story.setMetadata('status', 'Completed' if value == 'Complete' else 'In-Progress')
+                self.story.setMetadata('status', 'Completed' if value == 'yes' else 'In-Progress')
 
             elif key == 'Categories':
                 for element in cells[1]('a'):
@@ -167,14 +163,30 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
             # <div style="margin-left:10ex;margin-right:10ex">
             ## fetching SWI version now instead of text.
             htmlurl = url.replace('readtextstory','readhtmlstory')
-            soup = self.make_soup(self.get_request(htmlurl))
-            div = soup.find('div',style="margin-left:10ex;margin-right:10ex")
-            if div:
-                return self.utf8FromSoup(htmlurl,div)
-            else:
+            ## Used to find by style, but it's inconsistent now. we've seen:
+            ## margin-left:10ex;margin-right:10ex
+            ## margin-right: 5%; margin-left: 5%
+            ## margin-left:5%; margin-right:5%
+            ## margin-left:5%; margin-right:5%; background: white
+            ## And there's some without a <div> tag (or an unclosed div)
+            ## Only the comments appear to be consistent.
+            beginmarker='<!--Read or display the file-->'
+            endmarker='''<hr size=1 noshade>
+<!--review add read, top and bottom-->
+'''
+            data = self.get_request(htmlurl)
+            try:
+                ## if both markers are found, assume whatever is in between
+                ## is the chapter text.
+                soup = self.make_soup(data[data.index(beginmarker):data.index(endmarker)])
+                return self.utf8FromSoup(htmlurl,soup)
+            except Exception as e:
+                # logger.debug(e)
+                # logger.debug(soup)
                 logger.debug("Story With Images(SWI) not found, falling back to HTML.")
 
             ## fetching html version now instead of text.
+            ## Note that html and SWI pages are *not* formatted the same.
             soup = self.make_soup(self.get_request(url.replace('readtextstory','readxstory')))
             # logger.debug(soup)
 

fanficfare/adapters/adapter_fictionpresscom.py

@@ -18,6 +18,7 @@
 from __future__ import absolute_import
 import logging
 logger = logging.getLogger(__name__)
+import re
 
 # py2 vs py3 transition
 
@@ -46,6 +47,12 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
     def _get_site_url_pattern(cls):
         return r"https?://(www|m)?\.fictionpress\.com/s/(?P<id>\d+)(/\d+)?(/(?P<title>[^/]+))?/?$"
 
+    ## normalized chapter URLs DO contain the story title now, but
+    ## normalized to current urltitle in case of title changes.
+    def normalize_chapterurl(self,url):
+        return re.sub(r"https?://(www|m)\.(?P<keep>fictionpress\.com/s/\d+/\d+/).*",
+                      r"https://www.\g<keep>",url)+self.urltitle
+
 def getClass():
     return FictionPressComSiteAdapter
 

fanficfare/adapters/adapter_ficwadcom.py

@@ -66,7 +66,8 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                                                               params['username']))
         d = self.post_request(loginUrl,params,usecache=False)
 
-        if "Login attempt failed..." in d:
+        if "Login attempt failed..." in d or \
+                '<div id="error">Please enter your username and password.</div>' in d:
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['username']))
             raise exceptions.FailedToLogin(url,params['username'])
@@ -114,7 +115,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
         self.story.setMetadata('title', stripHTML(titleh4.a))
 
-        if 'Deleted story' in self.story.getMetadata('title'):
+        if 'Deleted story' in self.story.getMetadataRaw('title'):
             raise exceptions.StoryDoesNotExist("This story was deleted. %s"%self.url)
 
         # Find authorid and URL from... author url.
@@ -129,14 +130,14 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
 
         # most of the meta data is here:
-        metap = storydiv.find("p",{"class":"meta"})
+        metap = storydiv.find("div",{"class":"meta"})
         self.story.addToList('category',metap.find("a",href=re.compile(r"^/category/\d+")).string)
 
         # warnings
         # <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
         spanreq = metap.find("span",{"class":"story-warnings"})
         if spanreq: # can be no warnings.
-            for a in spanreq.findAll("a"):
+            for a in spanreq.find_all("a"):
                 self.story.addToList('warnings',a['title'])
 
         ## perhaps not the most efficient way to parse this, using
@@ -186,7 +187,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             # no list found, so it's a one-chapter story.
             self.add_chapter(self.story.getMetadata('title'),url)
         else:
-            chapterlistlis = storylistul.findAll('li')
+            chapterlistlis = storylistul.find_all('li')
             for chapterli in chapterlistlis:
                 if "blocked" in chapterli['class']:
                     # paranoia check.  We should already be logged in by now.

fanficfare/adapters/adapter_fimfictionnet.py

@@ -99,6 +99,17 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                                                                  params['username']))
                 raise exceptions.FailedToLogin(url,params['username'])
 
+    def make_soup(self,data):
+        soup = super(FimFictionNetSiteAdapter, self).make_soup(data)
+        for img in soup.select('img.lazy-img, img.user_image'):
+            ## FimF has started a 'camo' mechanism for images that
+            ## gets block by CF.  attr data-source is original source.
+            if img.has_attr('data-source'):
+                img['src'] = img['data-source']
+            elif img.has_attr('data-src'):
+                img['src'] = img['data-src']
+        return soup
+
     def doExtractChapterUrlsAndMetadata(self,get_cover=True):
 
         if self.is_adult or self.getConfig("is_adult"):
@@ -106,7 +117,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
         ## Only needed with password protected stories, which you have
         ## to have logged into in the website using this account.
-        self.performLogin(self.url)
+        if self.getConfig("always_login"):
+            self.performLogin(self.url)
 
         ##---------------------------------------------------------------------------------------------------
         ## Get the story's title page. Check if it exists.
@@ -139,7 +151,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("authorId", author['href'].split('/')[2])
         self.story.setMetadata("authorUrl", "https://%s/user/%s/%s" % (self.getSiteDomain(),
                                                                        self.story.getMetadata('authorId'),
-                                                                       self.story.getMetadata('author')))
+                                                                       # meta entry author can be changed by the user.
+                                                                       stripHTML(author)))
 
         #Rating text is replaced with full words for historical compatibility after the site changed
         #on 2014-10-27
@@ -167,12 +180,13 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
         # Cover image
         if get_cover:
-            storyImage = storyContentBox.find('img', {'class':'lazy-img'})
+            storyImage = soup.select_one('div.story_container__story_image img')
             if storyImage:
                 coverurl = storyImage['data-fullsize']
                 # try setting from data-fullsize, if fails, try using data-src
-                if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
-                    coverurl = storyImage['data-src']
+                cover_set = self.setCoverImage(self.url,coverurl)[0]
+                if not cover_set or cover_set.startswith("failedtoload"):
+                    coverurl = storyImage['src']
                     self.setCoverImage(self.url,coverurl)
 
                 coverSource = storyImage.parent.find('a', {'class':'source'})
@@ -284,16 +298,26 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         descriptionMeta = soup.find('meta', {'property':'og:description'})
         self.story.setMetadata("short_description", stripHTML(descriptionMeta['content']))
 
-        #groups
+        # groups.
+        # If there are more than X groups, there's a 'Show all' button
+        # that calls for a JSON containing HTML with the full list.
+        # But it doesn't work reliably with FlareSolverr.
+        groupList = None
         groupButton = soup.find('button', {'data-click':'showAll'})
         if groupButton != None and groupButton.find('i', {'class':'fa-search-plus'}):
-            groupResponse = self.get_request("https://www.fimfiction.net/ajax/stories/%s/groups" % (self.story.getMetadata("storyId")))
-            groupData = json.loads(groupResponse)
-            groupList = self.make_soup(groupData["content"])
-        else:
+            try:
+                groupResponse = self.get_request("https://www.fimfiction.net/ajax/stories/%s/groups" % (self.story.getMetadata("storyId")))
+                groupData = json.loads(groupResponse)
+                groupList = self.make_soup(groupData["content"])
+            except Exception as e:
+                logger.warning("Collecting 'groups' (AKA 'Featured In') from JSON failed:%s"%e)
+                logger.warning("Only 'groups' initially shown on the page will be collected.")
+                logger.warning("This is a known issue with JSON and FlareSolverr.  See #1122")
+
+        if not groupList:
             groupList = soup.find('ul', {'id':'story-groups-list'})
 
-        if not (groupList == None):
+        if groupList:
             for groupContent in groupList.find_all('a'):
                 self.story.addToList("groupsUrl", 'https://'+self.host+groupContent["href"])
                 groupName = groupContent.find('span', {"class":"group-name"})
@@ -304,7 +328,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
         #sequels
         for header in soup.find_all('h1', {'class':'header-stories'}):
-            # I don't know why using text=re.compile with find() wouldn't work, but it didn't.
+            # I don't know why using string=re.compile with find() wouldn't work, but it didn't.
             if header.text.startswith('Sequels'):
                 sequelContainer = header.parent
                 for sequel in sequelContainer.find_all('a', {'class':'story_link'}):
@@ -384,3 +408,33 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # data = self.get_request(url)
         if self.getConfig("is_adult"):
             self.set_adult_cookie()
+
+    def get_urls_from_page(self,url,normalize):
+        iterate = self.getConfig('scrape_bookshelf', default=False)
+        if not re.search(r'fimfiction\.net/bookshelf/(?P<listid>.+?)/',url) or iterate == 'legacy':
+            return super().get_urls_from_page(url,normalize)
+
+        self.before_get_urls_from_page(url,normalize)
+
+        final_urls = list()
+        while True:
+            data = self.get_request(url,usecache=True)
+            soup = self.make_soup(data)
+            paginator = soup.select_one('div.paginator-container > div.page_list > ul').find_all('li')
+            logger.debug("Paginator: " + str(len(paginator)))
+            stories_container = soup.select_one('div.content > div.two-columns > div.left').find_all('article', recursive=False)
+            x = 0
+            logger.debug("Container "+str(len(stories_container)))
+            for story_raw in stories_container:
+                x += 1
+                story_url = story_raw.select_one('div.story_content_box > header.title > div > a.story_name').get('href')
+                url_story = ('https://' + self.getSiteDomain() + story_url)
+                #logger.debug(url_story)
+                final_urls.append(url_story)
+            logger.debug("Discovered %s new stories."%str(x))
+
+            next_button = paginator[-1].select_one('a')
+            logger.debug("Next button: " + next_button.get_text())
+            if next_button.get_text() or not iterate:
+                return {'urllist': final_urls}
+            url = ('https://' + self.getSiteDomain() + next_button.get('href'))

fanficfare/adapters/adapter_fireflyfansnet.py

@@ -93,6 +93,9 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
 
         a = soup.find('a', href=re.compile(r"profileshow.aspx\?u="))
         self.story.setMetadata('authorId', a['href'].split('=')[1])
+        if not self.story.getMetadata('authorId'):
+            logger.warning("Site authorUrl missing authorId, using SiteMissingAuthorId")
+            self.story.setMetadata('authorId', 'SiteMissingAuthorId')
         self.story.setMetadata('authorUrl', 'http://' +
                                self.host + '/' + a['href'])
         self.story.setMetadata('author', a.string)
@@ -102,7 +105,6 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
         # to download them one at a time yourself. I'm also setting the status to
         # complete
         self.add_chapter(self.story.getMetadata('title'), self.url)
-        self.story.setMetadata('numChapters', 1)
         self.story.setMetadata('status', 'Completed')
 
         ## some stories do not have a summary listed, so I'm setting it here.

fanficfare/adapters/adapter_forumquestionablequestingcom.py

@@ -1,6 +1,6 @@
 #  -*- coding: utf-8 -*-
 
-# Copyright 2018 FanFicFare team
+# Copyright 2024 FanFicFare team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,15 +18,15 @@
 from __future__ import absolute_import
 import re
 
-from .base_xenforoforum_adapter import BaseXenForoForumAdapter
+from .base_xenforo2forum_adapter import BaseXenForo2ForumAdapter
 
 def getClass():
     return QuestionablequestingComAdapter
 
-class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
+class QuestionablequestingComAdapter(BaseXenForo2ForumAdapter):
 
     def __init__(self, config, url):
-        BaseXenForoForumAdapter.__init__(self, config, url)
+        BaseXenForo2ForumAdapter.__init__(self, config, url)
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','qq')

fanficfare/adapters/adapter_hlfictionnet.py

@@ -1,216 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return HLFictionNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HLFictionNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','hlf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'hlfiction.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','https://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-
-        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select')
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                # just in case there's tags, like <i> in chapter titles.
-                self.add_chapter(chapter,'https://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])
-        else:
-            self.add_chapter(self.story.getMetadata('title'),url)
-
-
-        asoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
-
-        for list in asoup.findAll('div', {'class' : re.compile('listbox')}):
-            a = list.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while 'classification' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'https://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_iketernalnet.py

@@ -1,262 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return IkEternalNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class IkEternalNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ike')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ik-eternal.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self.post_request(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=1"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self.get_request(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                data = self.get_request(url)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        asoup = soup.find('div', {'class': 'listbox'})
-        for a in asoup.findAll('p'):
-            a.name='br'
-        labels = asoup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_imagineeficcom.py

@@ -161,7 +161,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
 
@@ -178,7 +178,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
 
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
             label = labelspan.string
@@ -199,22 +199,22 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -238,7 +238,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
 
             seriessoup = self.make_soup(self.get_request(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
                 # skip 'report this' and 'TOC' links

fanficfare/adapters/adapter_inkbunnynet.py

@@ -125,7 +125,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
                 soup = self.make_soup(self.get_request(url,usecache=False))
 
         # removing all of the scripts
-        for tag in soup.findAll('script'):
+        for tag in soup.find_all('script'):
             tag.extract()
 
 
@@ -134,7 +134,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('title', stripHTML(title))
 
         # Get Author
-        authortag = soup.find('table',{'class':'pooltable'}).find('a',href=re.compile(r'/gallery/'))
+        authortag = soup.find('table',{'class':'pooltable'}).find('a',href=re.compile(r'/gallery/|/scraps/'))
         author = authortag['href'].split('/')[-1] # no separate ID
         self.story.setMetadata('author', author)
         self.story.setMetadata('authorId', author)
@@ -149,7 +149,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
         if not self.getConfig('keep_summary_html'):
             synopsis = stripHTML(synopsis)
 
-        self.setDescription(url, stripHTML(synopsis))
+        self.setDescription(url, synopsis)
 
         #Getting Keywords/Genres
         keywords = bookdetails.find('div', {'id':'kw_scroll'}).find_next_siblings('div')[0].div.div.find_all('a')
@@ -157,10 +157,11 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
             self.story.addToList('genre', stripHTML(kword))
 
         # Getting the Category
+        category = bookdetails.findChildren('div', recursive=False)[2].find('span', string='Type:').parent
+        category.find('span').decompose()
+        self.story.setMetadata('category', stripHTML(category))
         for div in bookdetails.find_all('div'):
-            if 'Details' == stripHTML(div).strip():
-                self.story.setMetadata('category', div.find_next_siblings('div')[0].span.next_sibling.strip())
-            elif 'Rating:' == stripHTML(div).strip()[:7]:
+            if 'Rating:' == stripHTML(div)[:7]:
                 rating = div.span.next_sibling.strip()
                 self.story.setMetadata('rating', rating)
                 break
@@ -178,7 +179,14 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
         if get_cover:
             cover_img = soup.find('img', {'id':'magicbox'})
             if cover_img:
+                # image content is treated like a normal image submission
                 self.setCoverImage(url, cover_img['src'])
+            else:
+                # image content is present, but secondary to text file
+                cover_div = soup.find('div', {'class': 'content magicboxParent'})
+                cover_img = cover_div.find('img', {'class':'shadowedimage'}) if cover_div else None
+                if cover_img:
+                    self.setCoverImage(url, cover_img['src'])
 
         ## Save for use below
         self.soup = soup
@@ -192,3 +200,11 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToDownload("Error downloading Chapter: %s No text block found -- non-story URL?" % url)
 
         return self.utf8FromSoup(url, story)
+
+    def before_get_urls_from_page(self,url,normalize):
+        # To display the links to stories that are not available to guests.
+        if self.getConfig("username") and self.getConfig("always_login"):
+            # performLogin extracts token from the soup
+            soup = self.make_soup(self.get_request(url))
+
+            self.performLogin(url, soup)

fanficfare/adapters/adapter_kakuyomujp.py

@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import absolute_import
+import logging, time
+logger = logging.getLogger(__name__)
+import re, json
+
+from .. import exceptions as exceptions
+
+# py2 vs py3 transition
+from ..six.moves import http_cookiejar as cl
+
+from .base_adapter import BaseSiteAdapter, makeDate
+
+def getClass():
+    return KakuyomuJpAdapter
+
+genres = {
+    'FANTASY': '異世界ファンタジー',
+    'ACTION': '現代ファンタジー',
+    'SF': 'SF',
+    'LOVE_STORY': '恋愛',
+    'ROMANCE': 'ラブコメ',
+    'DRAMA': '現代ドラマ',
+    'HORROR': 'ホラー',
+    'MYSTERY': 'ミステリー',
+    'NONFICTION': 'エッセイ・ノンフィクション',
+    'HISTORY': '歴史・時代・伝奇',
+    'CRITICISM': '創作論・評論',
+    'OTHERS': '詩・童話・その他',
+    'FAN_FICTION': '二次創作',
+}
+
+class KakuyomuJpAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.story.setMetadata('siteabbrev', 'kakuyomu')
+        self.story.setMetadata('language', 'Japanese')
+
+        self.storyId = self.path.split('/')[-1]
+        self.story.setMetadata('storyId', self.storyId)
+
+    @staticmethod
+    def getSiteDomain():
+        return 'kakuyomu.jp'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return ("https://kakuyomu.jp/works/12341234123412341234")
+
+    def getSiteURLPattern(self):
+        return r"^https?://kakuyomu\.jp/works/[0-9]+$"
+
+    def extractChapterUrlsAndMetadata(self):
+        data = self.get_request(self.url)
+
+        # Page could not be found
+        if 'お探しのページは見つかりませんでした' in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        soup = self.make_soup(data)
+        info = json.loads(soup.find(id='__NEXT_DATA__').contents[0])['props']['pageProps']['__APOLLO_STATE__']
+
+        workKey = 'Work:%s' % self.storyId
+
+        # Title
+        self.story.setMetadata('title', info[workKey]['title'])
+
+        # Author
+        authorKey = info[workKey]['author']['__ref']
+        self.story.setMetadata('authorId', authorKey.split(':')[1])
+        self.story.setMetadata('authorUrl', 'https://kakuyomu.jp/users/%s' % info[authorKey]['name'])
+        self.story.setMetadata('author', info[authorKey]['activityName'])
+
+        # Description
+        self.setDescription(self.url, info[workKey]['introduction'])
+        self.story.setMetadata('catchphrase', info[workKey]['catchphrase'])
+
+        # Date Published and Updated
+        # 2024-01-01T03:00:12Z
+        self.story.setMetadata('datePublished',
+                               makeDate(info[workKey]['publishedAt'], '%Y-%m-%dT%H:%M:%SZ'))
+        self.story.setMetadata('dateUpdated',
+                               makeDate(info[workKey]['editedAt'], '%Y-%m-%dT%H:%M:%SZ'))
+
+        # Character count
+        self.story.setMetadata('numWords', info[workKey]['totalCharacterCount'])
+
+        # Status
+        completed = info[workKey]['serialStatus'] == 'COMPLETED'
+        self.story.setMetadata('status', 'Completed' if completed else 'In-Progress')
+
+        # Warnings
+        rating = 'G'
+        if info[workKey]['isCruel']:
+            rating = 'R15'
+            self.story.addToList('warnings', '残酷描写有り')
+        if info[workKey]['isViolent']:
+            rating = 'R15'
+            self.story.addToList('warnings', '暴力描写有り')
+        if info[workKey]['isSexual']:
+            rating = 'R15'
+            self.story.addToList('warnings', '性描写有り')
+
+        # Tags
+        for tag in info[workKey]['tagLabels']:
+            if re.match(r'[RrＲ].?[1１][5５]', tag) is None:
+                self.story.addToList('freeformtags', tag)
+            else:
+                rating = 'R15'
+
+        # Rating
+        self.story.setMetadata('rating', rating)
+
+        # Genre
+        self.story.setMetadata('genre', genres[info[workKey]['genre']])
+
+        if info[workKey]['genre'] == 'FAN_FICTION':
+            fandomKey = info[workKey]['fanFictionSource']['__ref']
+            self.story.addToList('fandoms', info[fandomKey]['title'])
+
+        # Ratings, Comments, Etc.
+        self.story.setMetadata('reviews', info[workKey]['reviewCount'])
+        self.story.setMetadata('points', info[workKey]['totalReviewPoint'])
+        self.story.setMetadata('comments', info[workKey]['totalPublicEpisodeCommentCount'])
+        self.story.setMetadata('views', info[workKey]['totalReadCount'])
+        self.story.setMetadata('follows', info[workKey]['totalFollowers'])
+        self.story.setMetadata('collections', len(info[workKey]['publicWorkCollections']))
+        self.story.setMetadata('events', info[workKey]['totalWorkContestCount'] + info[workKey]['totalUserEventCount'])
+        self.story.setMetadata('published', info[workKey]['hasPublication'])
+
+        # visitorWorkFollowing
+        # workReviewByVisitor
+
+        # Chapters, Episodes
+
+        # TOC nodes are in a list
+        # each have a list of named episodes
+        # each can have a named chapter
+        # named chapters can be at depth 1 or 2
+        # episodes might be empty (premium subscription)
+
+        prependSectionTitles = self.getConfig('prepend_section_titles', 'firstepisode')
+
+        numEpisodes = 0
+        titles = []
+        nestingLevel = 0
+        newSection = False
+        for tocNodeRef in info[workKey]['tableOfContentsV2']:
+            tocNode = info[tocNodeRef['__ref']]
+
+            if tocNode['chapter'] is not None:
+                chapter = info[tocNode['chapter']['__ref']]
+                while chapter['level'] <= nestingLevel:
+                    titles.pop()
+                    nestingLevel -= 1
+                titles.append(chapter['title'])
+                nestingLevel = chapter['level']
+                newSection = True
+            else:
+                titles = []
+                nestingLevel = 0
+                newSection = False
+
+            for episodeRef in tocNode['episodeUnions']:
+                if not episodeRef['__ref'].startswith('EmptyEpisode'):
+                    numEpisodes += 1
+                    episode = info[episodeRef['__ref']]
+                    epUrl = 'https://kakuyomu.jp/works/' + self.storyId + '/episodes/' + episode['id']
+                    epTitle = episode['title']
+
+                    if ((len(titles) > 0) and
+                        ((newSection and prependSectionTitles == 'firstepisode') or
+                         prependSectionTitles == 'true')):
+                        titles.append(epTitle)
+                        # bracket with ZWSP to mark presence of section titles
+                        epTitle = u'\u200b' + u'\u3000\u200b'.join(titles)
+                        titles.pop()
+
+                    self.add_chapter(epTitle, epUrl)
+                newSection = False
+
+        logger.debug("Story: <%s>", self.story)
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from <%s>' % url)
+
+        soup = self.make_soup(self.get_request(url))
+        soup = soup.find('div', {'class':'widget-episodeBody js-episode-body'})
+        if soup is None:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        soup.attrs = {'class':'episode-body'}
+
+        return self.utf8FromSoup(url, soup)
+

fanficfare/adapters/adapter_ksarchivecom.py

@@ -144,13 +144,13 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
 
         # Find authorid and URL from... author urls.
         pagetitle = soup.find('div',id='pagetitle')
-        for a in pagetitle.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")):
+        for a in pagetitle.find_all('a', href=re.compile(r"viewuser.php\?uid=\d+")):
             self.story.addToList('authorId',a['href'].split('=')[1])
             self.story.addToList('authorUrl','https://'+self.host+'/'+a['href'])
             self.story.addToList('author',stripHTML(a))
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
 
@@ -166,7 +166,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 return ""
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
             label = stripHTML(labelspan)
@@ -193,7 +193,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 catstext = [stripHTML(cat) for cat in cats]
                 for cat in catstext:
                     # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
@@ -204,7 +204,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
             if 'Characters' in label:
                 self.story.addToList('characters','Kirk')
                 self.story.addToList('characters','Spock')
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 charstext = [stripHTML(char) for char in chars]
                 for char in charstext:
                     self.story.addToList('characters',stripHTML(char))
@@ -213,7 +213,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
                 genrestext = [stripHTML(genre) for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
@@ -223,7 +223,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
             ## has 'Story Type', which is much more what most sites
             ## call genre.
             if 'Story Type' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
                 genrestext = [stripHTML(genre) for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
@@ -233,21 +233,21 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 warningstext = [stripHTML(warning) for warning in warnings]
                 self.warning = ', '.join(warningstext)
                 for warning in warningstext:
                     self.story.addToList('warnings',stripHTML(warning))
 
             if 'Universe' in label:
-                universes = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
+                universes = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
                 universestext = [stripHTML(universe) for universe in universes]
                 self.universe = ', '.join(universestext)
                 for universe in universestext:
                     self.story.addToList('universe',stripHTML(universe))
 
             if 'Crossover Fandom' in label:
-                crossoverfandoms = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX
+                crossoverfandoms = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX
                 crossoverfandomstext = [stripHTML(crossoverfandom) for crossoverfandom in crossoverfandoms]
                 self.crossoverfandom = ', '.join(crossoverfandomstext)
                 for crossoverfandom in crossoverfandomstext:
@@ -274,7 +274,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
             series_url = 'https://'+self.host+'/'+a['href']
 
             seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
                 # skip 'report this' and 'TOC' links

fanficfare/adapters/adapter_libraryofmoriacom.py

@@ -17,6 +17,7 @@
 
 # Software: eFiction
 from __future__ import absolute_import
+from ..htmlcleanup import stripHTML
 from .base_efiction_adapter import BaseEfictionAdapter
 
 class LibraryOfMoriaComAdapter(BaseEfictionAdapter):
@@ -37,5 +38,19 @@ class LibraryOfMoriaComAdapter(BaseEfictionAdapter):
     def getDateFormat(self):
         return "%B %d, %Y"
 
+    def getRatingFromTOC(self):
+        # In many eFiction sites, the Rating is not included in
+        # print page, but is on the TOC page.  At least one site's rating
+        # (libraryofmoriacom) differs enough to be problematic.
+        toc = self.url + "&index=1"
+        soup = self.make_soup(self.get_request(toc))
+        for label in soup.select('div.listbox b'):
+            if 'Rated:' in label or 'Rating:' in stripHTML(label):
+                rating = stripHTML(label.next_sibling)
+                if rating.endswith(' ['):
+                    rating = rating[:-2]
+                self.story.setMetadata('rating',rating)
+                break
+
 def getClass():
     return LibraryOfMoriaComAdapter

fanficfare/adapters/adapter_literotica.py

@@ -19,6 +19,7 @@ from __future__ import absolute_import
 import logging
 logger = logging.getLogger(__name__)
 import re
+import json
 
 from bs4.element import Comment
 from ..htmlcleanup import stripHTML
@@ -37,7 +38,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
-        logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
+        #logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','litero')
@@ -47,16 +48,15 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         # where first chapter doesn't have '-ch-'.
         # Now just rely on extractChapterUrlsAndMetadata to reset
         # storyId to first chapter link.
-        storyId = self.parsedUrl.path.split('/',)[2]
 
         ## DON'T normalize to www.literotica.com--keep for language,
         ## which will be set in _setURL(url).  Also, multi-chapter
         ## have been keeping the language when 'normalizing' to first
         ## chapter.
         url = re.sub(r"^(https?://)"+LANG_RE+r"(\.i)?",
-                     r"\1\2",
+                     r"https://\2",
                      url)
-        url = url.replace('/beta/s/','/s/') # to allow beta site URLs.
+        url = url.replace('/beta/','/') # to allow beta site URLs.
 
         ## strip ?page=...
         url = re.sub(r"\?page=.*$", "", url)
@@ -66,7 +66,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
+        self.dateformat = "%m/%d/%Y"
 
     @staticmethod
     def getSiteDomain():
@@ -78,10 +78,12 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
+        return "https://www.literotica.com/s/story-title https://www.literotica.com/series/se/9999999 https://www.literotica.com/s/story-title https://www.literotica.com/i/image-or-comic-title https://www.literotica.com/p/poem-title https://portuguese.literotica.com/s/story-title https://german.literotica.com/s/story-title"
 
     def getSiteURLPattern(self):
-        return r"https?://"+LANG_RE+r"(\.i)?\.literotica\.com/(beta/)?s/([a-zA-Z0-9_-]+)"
+        # also https://www.literotica.com/series/se/80075773
+        # /s/ for story, /i/ for image/comic, /p/ for poem
+        return r"https?://"+LANG_RE+r"(\.i)?\.literotica\.com/((beta/)?[sip]/([a-zA-Z0-9_-]+)|series/se/(?P<storyseriesid>[a-zA-Z0-9_-]+))"
 
     def _setURL(self,url):
         # logger.debug("set URL:%s"%url)
@@ -90,260 +92,337 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         lang = m.group('lang')
         if lang not in ('www','other'):
             self.story.setMetadata('language',lang.capitalize())
+        # reset storyId
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[-1])
         # logger.debug("language:%s"%self.story.getMetadata('language'))
 
-    def getCategories(self, soup):
-        if self.getConfig("use_meta_keywords"):
-            categories = soup.find("meta", {"name":"keywords"})['content'].split(',')
-            categories = [c for c in categories if not self.story.getMetadata('title') in c]
-            if self.story.getMetadata('author') in categories:
-                categories.remove(self.story.getMetadata('author'))
-            # logger.debug("Meta = %s" % categories)
-            for category in categories:
-    #            logger.debug("\tCategory=%s" % category)
-#                 self.story.addToList('category', category.title())
-                self.story.addToList('eroticatags', category.title())
+    ## apply clean_chapter_titles
+    def add_chapter(self,chapter_title,url,othermeta={}):
+        if self.getConfig("clean_chapter_titles"):
+            storytitle = self.story.getMetadataRaw('title').lower()
+            chapter_name_type = None
+            # strip trailing ch or pt before doing the chapter clean.
+            # doesn't remove from story title metadata
+            storytitle = re.sub(r'^(.*?)( (ch|pt))?$',r'\1',storytitle)
+            if chapter_title.lower().startswith(storytitle):
+                chapter = chapter_title[len(storytitle):].strip()
+                # logger.debug('\tChapter: "%s"' % chapter)
+                if chapter == '':
+                    chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
+                    # Sometimes the first chapter does not have type of chapter
+                    if self.num_chapters() == 0:
+                        # logger.debug('\tChapter: first chapter without chapter type')
+                        chapter_name_type = None
+                else:
+                    separater_char = chapter[0]
+                    # logger.debug('\tseparater_char: "%s"' % separater_char)
+                    chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
+                    # logger.debug('\tChapter: "%s"' % chapter)
+                    if chapter.lower().startswith('ch.'):
+                        chapter = chapter[len('ch.'):].strip()
+                        try:
+                            chapter_title = 'Chapter %d' % int(chapter)
+                        except:
+                            chapter_title = 'Chapter %s' % chapter
+                        chapter_name_type = 'Chapter' if chapter_name_type is None else chapter_name_type
+                        # logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
+                    elif chapter.lower().startswith('pt.'):
+                        chapter = chapter[len('pt.'):].strip()
+                        try:
+                            chapter_title = 'Part %d' % int(chapter)
+                        except:
+                            chapter_title = 'Part %s' % chapter
+                        chapter_name_type = 'Part' if chapter_name_type is None else chapter_name_type
+                        # logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
+                    elif separater_char in [":", "-"]:
+                        chapter_title = chapter
+                        # logger.debug('\tChapter: taking chapter text as whole')
+        super(LiteroticaSiteAdapter, self).add_chapter(chapter_title,url,othermeta)
 
     def extractChapterUrlsAndMetadata(self):
         """
-        NOTE: Some stories can have versions,
-              e.g. /my-story-ch-05-version-10
-        NOTE: If two stories share the same title, a running index is added,
-              e.g.: /my-story-ch-02-1
-        Strategy:
-            * Go to author's page, search for the current story link,
-            * If it's in a tr.root-story => One-part story
-                * , get metadata and be done
-            * If it's in a tr.sl => Chapter in series
-                * Search up from there until we find a tr.ser-ttl (this is the
-                story)
-                * Gather metadata
-                * Search down from there for all tr.sl until the next
-                tr.ser-ttl, foreach
-                    * Chapter link is there
+        In April 2024, site introduced significant changes, including
+        adding a 'Story Series' page and link to it in each chapter.
+        But not all stories, one-shots don't have 'Story Series'.
+
+        literotica has 'Story Series' & 'Story'.  FFF calls them 'Story' & 'Chapters'
+        See https://github.com/JimmXinu/FanFicFare/issues/1058#issuecomment-2078490037
+
+        So /series/se/ will be the story URL for multi chapters but
+        keep individual 'chapter' URL for one-shots.
         """
+        logger.debug("Chapter/Story URL: <%s> " % self.url)
 
         if not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
 
-        # logger.debug("Chapter/Story URL: <%s> " % self.url)
-
-        (data1,rurl) = self.get_request_redirected(self.url)
+        (data,rurl) = self.get_request_redirected(self.url)
+        # logger.debug(data)
         ## for language domains
         self._setURL(rurl)
         logger.debug("set opened url:%s"%self.url)
-        soup1 = self.make_soup(data1)
-        #strip comments from soup
-        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, Comment))]
+        soup = self.make_soup(data)
 
-        if "This submission is awaiting moderator's approval" in data1:
+        if "This submission is awaiting moderator's approval" in data:
             raise exceptions.StoryDoesNotExist("This submission is awaiting moderator's approval. %s"%self.url)
 
+        ## 2025Feb - domains other than www now use different HTML.
+        ## Need to look for two different versions of basically
+        ## everything.
+
+        ## not series URL, assumed to be a chapter.  Look for Story
+        ## Info block of post-beta page.  I don't think it should happen?
+        if '/series/se' not in self.url:
+            #logger.debug(data)
+            ## looking for /series/se URL to indicate this is a
+            ## chapter.
+            if not soup.select_one('div.page__aside') and not soup.select_one('div.sidebar') and not soup.select_one('div[class^="_sidebar_"]'):
+                raise exceptions.FailedToDownload("Missing Story Info block, Beta turned off?")
+
+            storyseriestag = soup.select_one('a.bn_av')
+            if not storyseriestag:
+                storyseriestag = soup.select_one('a[class^="_files__link_"]')
+            # logger.debug("Story Series Tag:%s"%storyseriestag)
+
+            if storyseriestag:
+                self._setURL(storyseriestag['href'])
+                data = self.get_request(storyseriestag['href'])
+                # logger.debug(data)
+                soup = self.make_soup(data)
+                # logger.debug(soup)
+            else:
+                logger.debug("One-shot")
+
+        isSingleStory = '/series/se' not in self.url
+
+        if not isSingleStory:
+            # Normilize the url?
+            state = re.findall(r"prefix\=\"/series/\",state='(.+?)'</script>", data)
+            json_state = json.loads(state[0].replace("\\'","'").replace("\\\\","\\"))
+            url_series_id = unicode(re.match(self.getSiteURLPattern(),self.url).group('storyseriesid'))
+            json_series_id = unicode(json_state['series']['data']['id'])
+            if json_series_id != url_series_id:
+                res = re.sub(url_series_id, json_series_id, unicode(self.url))
+                logger.debug("Normalized url: %s"%res)
+                self._setURL(res)
+
+        ## common between one-shots and multi-chapters
+        # title
+        self.story.setMetadata('title', stripHTML(soup.select_one('h1')))
+        # logger.debug(self.story.getMetadata('title'))
+
         # author
-        authora = soup1.find("a", class_="y_eU")
+        ## XXX This is still the author URL like:
+        ## https://www.literotica.com/stories/memberpage.php?uid=999999&page=submissions
+        ## because that's what's on the page.  It redirects to the /authors/ page.
+        ## Only way I know right now to get the /authors/ is to make
+        ## the req and look at the redirect.
+        ## Should change to /authors/ if/when it starts appearing.
+        ## Assuming it's in the same place.
+        authora = soup.find("a", class_="y_eU")
+        if not authora:
+            authora = soup.select_one('a[class^="_author__title"]')
         authorurl = authora['href']
-        # logger.debug(authora)
-        # logger.debug(authorurl)
-        self.story.setMetadata('authorId', urlparse.parse_qs(authorurl.split('?')[1])['uid'][0])
         if authorurl.startswith('//'):
             authorurl = self.parsedUrl.scheme+':'+authorurl
+        # logger.debug(authora)
+        # logger.debug(authorurl)
+        self.story.setMetadata('author', stripHTML(authora))
         self.story.setMetadata('authorUrl', authorurl)
-        self.story.setMetadata('author', authora.text)
+        if '?' in authorurl:
+            self.story.setMetadata('authorId', urlparse.parse_qs(authorurl.split('?')[1])['uid'][0])
+        elif '/authors/' in authorurl:
+            self.story.setMetadata('authorId', authorurl.split('/')[-1])
+        else: # if all else fails
+            self.story.setMetadata('authorId', stripHTML(authora))
 
-        # get the author page
-        dataAuth = self.get_request(authorurl)
-        soupAuth = self.make_soup(dataAuth)
-        #strip comments from soup
-        [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, Comment))]
-#       logger.debug(soupAuth)
+        if soup.select('div#tabpanel-tags'):
+            # logger.debug("tags1")
+            self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div#tabpanel-tags a.av_as') ])
+        if soup.select('div[class^="_widget__tags_"]'):
+            # logger.debug("tags2")
+            self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div[class^="_widget__tags_"] a[class^="_tag_item_"]') ])
+        # logger.debug(self.story.getList('eroticatags'))
 
-        ## Find link to url in author's page
-        ## site has started using //domain.name/asdf urls remove https?: from front
-        ## site has started putting https back on again.
-        ## site is now using language specific german.lit... etc on author pages.
-        ## site is now back to using www.lit... etc on author pages.
-        search_url_re = r"https?://"+LANG_RE+r"(\.i)?\." + re.escape(self.getSiteDomain()) + self.url[self.url.index('/s/'):]+r"$"
-        logger.debug(search_url_re)
-        storyLink = soupAuth.find('a', href=re.compile(search_url_re))
-#         storyLink = soupAuth.find('a', href=re.compile(r'.*literotica.com/s/'+re.escape(self.story.getMetadata('storyId')) ))
-#         storyLink = soupAuth.find('a', href=re.compile(r'(https?:)?'+re.escape(self.url[self.url.index(':')+1:]).replace(r'www',r'[^\.]+') ))
-#         storyLink = soupAuth.find('a', href=self.url)#[self.url.index(':')+1:])
+        ## look first for 'Series Introduction', then Info panel short desc
+        ## series can have either, so put in common code.
+        desc = []
+        introtag = soup.select_one('div.bp_rh')
+        descdiv = soup.select_one('div#tabpanel-info div.bn_B') or \
+                  soup.select_one('div[class^="_tab__pane_"] div[class^="_widget__info_"]')
+        if introtag and stripHTML(introtag):
+            # make sure there's something in the tag.
+            # logger.debug("intro %s"%introtag)
+            desc.append(unicode(introtag))
+        elif descdiv and stripHTML(descdiv):
+            # make sure there's something in the tag.
+            # logger.debug("desc %s"%descdiv)
+            desc.append(unicode(descdiv))
+        if not desc or self.getConfig("include_chapter_descriptions_in_summary"):
+            ## Only for backward compatibility with 'stories' that
+            ## don't have an intro or short desc.
+            descriptions = []
+            for i, chapterdesctag in enumerate(soup.select('p.br_rk')):
+                # remove category link, but only temporarily
+                a = chapterdesctag.a.extract()
+                descriptions.append("%d. %s" % (i + 1, stripHTML(chapterdesctag)))
+                # now put it back--it's used below
+                chapterdesctag.append(a)
+            desc.append(unicode("<p>"+"</p>\n<p>".join(descriptions)+"</p>"))
 
-        if storyLink is not None:
-            # pull the published date from the author page
-            # default values from single link.  Updated below if multiple chapter.
-            # logger.debug("Found story on the author page.")
-            date = storyLink.parent.parent.findAll('td')[-1].text
-            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
-
-        if storyLink is not None:
-            urlTr = storyLink.parent.parent
-            if "sl" in urlTr['class']:
-                isSingleStory = False
-            else:
-                isSingleStory = True
-        else:
-            raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (self.url, authorurl))
+        self.setDescription(self.url,u''.join(desc))
 
         if isSingleStory:
-            self.story.setMetadata('title', storyLink.text.strip('/'))
-            # logger.debug('Title: "%s"' % storyLink.text.strip('/'))
-            self.setDescription(authorurl, urlTr.findAll("td")[1].text)
-            self.story.addToList('category', urlTr.findAll("td")[2].text)
-#             self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
-            date = urlTr.findAll('td')[-1].text
-            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
-            self.add_chapter(storyLink.text, self.url)
-            averrating = stripHTML(storyLink.parent)
-            ## title (0.00)
-            averrating = averrating[averrating.rfind('(')+1:averrating.rfind(')')]
-            try:
-                self.story.setMetadata('averrating', float(averrating))
-            except:
-                pass
-#             self.story.setMetadata('averrating',averrating)
-        # parse out the list of chapters
+            ## one-shots don't *display* date info, but they have it
+            ## hidden in <script>
+            ## shows _date_approve "date_approve":"01/31/2024"
+
+            ## multichap also have "date_approve", but they have
+            ## several and they're more than just the story chapters.
+            date = re.search(r'"date_approve":"(\d\d/\d\d/\d\d\d\d)"',data)
+            if not date:
+                date = re.search(r'date_approve:"(\d\d/\d\d/\d\d\d\d)"',data)
+            if date:
+                dateval = makeDate(date.group(1), self.dateformat)
+                self.story.setMetadata('datePublished', dateval)
+                self.story.setMetadata('dateUpdated', dateval)
+
+            ## one-shots don't have same json data to get aver_rating
+            ## from below. This kludge matches the data_approve
+            rateall = re.search(r'rate_all:([\d\.]+)',data)
+            if rateall:
+                self.story.setMetadata('averrating', '%4.2f' % float(rateall.group(1)))
+
+            ## one-shots assumed completed.
+            self.story.setMetadata('status','Completed')
+
+            # Add the category from the breadcumb.
+            breadcrumbs = soup.find('div', id='BreadCrumbComponent')
+            if not breadcrumbs:
+                breadcrumbs = soup.select_one('ul[class^="_breadcrumbs_list_"]')
+            if not breadcrumbs:
+                # _breadcrumbs_18u7l_1
+                breadcrumbs = soup.select_one('nav[class^="_breadcrumbs_"]')
+            self.story.addToList('category', breadcrumbs.find_all('a')[1].string)
+
+            ## one-shot chapter
+            self.add_chapter(self.story.getMetadata('title'), self.url)
+
         else:
-            seriesTr = urlTr.previousSibling
-            while 'ser-ttl' not in seriesTr['class']:
-                seriesTr = seriesTr.previousSibling
-            m = re.match(r"^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
-            self.story.setMetadata('title', m.group('title'))
-            seriesTitle = m.group('title')
+            ## Multi-chapter stories.  AKA multi-part 'Story Series'.
+            bn_antags = soup.select('div#tabpanel-info p.bn_an')
+            # logger.debug(bn_antags)
+            if bn_antags and not self.getConfig("dates_from_chapters"):
+                ## Use dates from series metadata unless dates_from_chapters is enabled
+                dates = []
+                for datetag in bn_antags[:2]:
+                    datetxt = stripHTML(datetag)
+                    # remove 'Started:' 'Updated:'
+                    # Assume can't use 'Started:' 'Updated:' (vs [0] or [1]) because of lang localization
+                    datetxt = datetxt[datetxt.index(':')+1:]
+                    dates.append(datetxt)
+                # logger.debug(dates)
+                self.story.setMetadata('datePublished', makeDate(dates[0], self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(dates[1], self.dateformat))
 
-            ## Walk the chapters
-            chapterTr = seriesTr.nextSibling
-            dates = []
-            descriptions = []
-            ratings = []
-            chapters = []
-            chapter_name_type = None
-            while chapterTr is not None and 'sl' in chapterTr['class']:
-                description = "%d. %s" % (len(descriptions)+1,stripHTML(chapterTr.findAll("td")[1]))
-                description = stripHTML(chapterTr.findAll("td")[1])
-                chapterLink = chapterTr.find("td", "fc").find("a")
-                if self.getConfig('chapter_categories_use_all'):
-                    self.story.addToList('category', chapterTr.findAll("td")[2].text)
-                # self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
-                pub_date = makeDate(chapterTr.findAll('td')[-1].text, self.dateformat)
-                dates.append(pub_date)
-                chapterTr = chapterTr.nextSibling
+            ## bn_antags[2] contains "The author has completed this series." or "The author is still actively writing this series."
+            ## I won't be surprised if this breaks later because of lang localization
+            if "completed" in stripHTML(bn_antags[-1]):
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')
 
-                chapter_title = chapterLink.text
-                if self.getConfig("clean_chapter_titles"):
-                    # logger.debug('\tChapter Name: "%s"' % chapterLink.text)
-                    if chapterLink.text.lower().startswith(seriesTitle.lower()):
-                        chapter = chapterLink.text[len(seriesTitle):].strip()
-                        # logger.debug('\tChapter: "%s"' % chapter)
-                        if chapter == '':
-                            chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
-                            # Sometimes the first chapter does not have type of chapter
-                            if self.num_chapters() == 0:
-                                logger.debug('\tChapter: first chapter without chapter type')
-                                chapter_name_type = None
-                        else:
-                            separater_char = chapter[0]
-                            # logger.debug('\tseparater_char: "%s"' % separater_char)
-                            chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
-                            # logger.debug('\tChapter: "%s"' % chapter)
-                            if chapter.lower().startswith('ch.'):
-                                chapter = chapter[len('ch.'):].strip()
-                                try:
-                                    chapter_title = 'Chapter %d' % int(chapter)
-                                except:
-                                    chapter_title = 'Chapter %s' % chapter
-                                chapter_name_type = 'Chapter' if chapter_name_type is None else chapter_name_type
-                                logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
-                            elif chapter.lower().startswith('pt.'):
-                                chapter = chapter[len('pt.'):]
-                                try:
-                                    chapter_title = 'Part %d' % int(chapter)
-                                except:
-                                    chapter_title = 'Part %s' % chapter
-                                chapter_name_type = 'Part' if chapter_name_type is None else chapter_name_type
-                                logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
-                            elif separater_char in [":", "-"]:
-                                chapter_title = chapter
-                                logger.debug('\tChapter: taking chapter text as whole')
+            ## category from chapter list
+            self.story.extendList('category',[ stripHTML(t) for t in soup.select('a.br_rl') ])
 
-                # pages include full URLs.
-                chapurl = chapterLink['href']
-                if chapurl.startswith('//'):
-                    chapurl = self.parsedUrl.scheme + ':' + chapurl
+            for chapteratag in soup.select('a.br_rj'):
+                chapter_title = stripHTML(chapteratag)
+                # logger.debug('\tChapter: "%s"' % chapteratag)
+                # /series/se does include full URLs current.
+                chapurl = chapteratag['href']
                 # logger.debug("Chapter URL: " + chapurl)
-                # logger.debug("Chapter Title: " + chapter_title)
-                # logger.debug("Chapter description: " + description)
-                chapters.append((chapter_title, chapurl, description, pub_date))
-#                 self.add_chapter(chapter_title, chapurl)
-                numrating = stripHTML(chapterLink.parent)
-                ## title (0.00)
-                numrating = numrating[numrating.rfind('(')+1:numrating.rfind(')')]
-                try:
-                    ratings.append(float(numrating))
-                except:
-                    pass
+                self.add_chapter(chapter_title, chapurl)
 
-            if self.getConfig("clean_chapter_titles") \
-                and chapter_name_type is not None \
-                and not chapters[0][0].startswith(chapter_name_type):
-                logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
-                logger.debug('\tChapter: first chapter="%s"' % chapters[0][0])
-                logger.debug('\tChapter: first chapter number="%s"' % chapters[0][0][len('Chapter'):])
-                chapters[0] = ("%s %s" % (chapter_name_type, chapters[0][0][len('Chapter'):].strip()),
-                               chapters[0][1],
-                               chapters[0][2],
-                               chapters[0][3]
-                               )
+            # <img src="https://uploads.literotica.com/series/cover/813-1695143444-desktop-x1.jpg" alt="Series cover">
+            coverimg = soup.select_one('img[alt="Series cover"]')
+            if coverimg:
+                self.setCoverImage(self.url,coverimg['src'])
 
-            chapters = sorted(chapters, key=lambda chapter: chapter[3])
-            for i, chapter in enumerate(chapters):
-                self.add_chapter(chapter[0], chapter[1])
-                descriptions.append("%d. %s" % (i + 1, chapter[2]))
-            ## Set the oldest date as publication date, the newest as update date
-            dates.sort()
-            self.story.setMetadata('datePublished', dates[0])
-            self.story.setMetadata('dateUpdated', dates[-1])
-            self.story.setMetadata('datePublished', chapters[0][3])
-            self.story.setMetadata('dateUpdated', chapters[-1][3])
-            ## Set description to joint chapter descriptions
-            self.setDescription(authorurl,"<p>"+"</p>\n<p>".join(descriptions)+"</p>")
+        #### Attempting averrating from JS metadata.
+        #### also alternate chapters from json
+        try:
+            state_start="state='"
+            state_end="'</script>"
+            i = data.index(state_start)
+            if i:
+                state = data[i+len(state_start):data.index(state_end,i)].replace("\\'","'").replace("\\\\","\\")
+                if state:
+                    # logger.debug(state)
+                    json_state = json.loads(state)
+                    # logger.debug(json.dumps(json_state, sort_keys=True,indent=2, separators=(',', ':')))
+                    all_rates = []
+                    if 'series' in json_state:
+                        all_rates = [ float(x['rate_all']) for x in json_state['series']['works'] ]
 
-            if len(ratings) > 0:
-                self.story.setMetadata('averrating','%4.2f' % (sum(ratings) / float(len(ratings))))
+                        ## Extract dates from chapter approval dates if dates_from_chapters is enabled
+                        if self.getConfig("dates_from_chapters"):
+                            date_approvals = []
+                            for work in json_state['series']['works']:
+                                if 'date_approve' in work:
+                                    try:
+                                        date_approvals.append(makeDate(work['date_approve'], self.dateformat))
+                                    except:
+                                        pass
+                            if date_approvals:
+                                # Oldest date is published, newest is updated
+                                date_approvals.sort()
+                                self.story.setMetadata('datePublished', date_approvals[0])
+                                self.story.setMetadata('dateUpdated', date_approvals[-1])
+                    if all_rates:
+                        self.story.setMetadata('averrating', '%4.2f' % (sum(all_rates) / float(len(all_rates))))
 
-        # normalize on first chapter URL.
-        self._setURL(self.get_chapter(0,'url'))
+                    ## alternate chapters from JSON
+                    if self.num_chapters() < 1:
+                        logger.debug("Getting Chapters from series JSON")
+                        seriesid = json_state.get('series',{}).get('data',{}).get('id',None)
+                        if seriesid:
+                            logger.info("Fetching chapter data from JSON")
+                            logger.debug(seriesid)
+                            series_json = json.loads(self.get_request('https://literotica.com/api/3/series/%s/works'%seriesid))
+                            # logger.debug(json.dumps(series_json, sort_keys=True,indent=2, separators=(',', ':')))
+                            for chap in series_json:
+                                self.add_chapter(chap['title'], 'https://www.literotica.com/s/'+chap['url'])
 
-        # reset storyId to first chapter.
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+                                ## Collect tags from series/story page if tags_from_chapters is enabled
+                                if self.getConfig("tags_from_chapters"):
+                                    self.story.extendList('eroticatags', [ unicode(t['tag']).title() for t in chap['tags'] ])
 
 
-        # Add the category from the breadcumb. This might duplicate a category already added.
-        self.story.addToList('category', soup1.find('div', id='BreadCrumbComponent').findAll('a')[1].string)
-        self.getCategories(soup1)
+        except Exception as e:
+            logger.warning("Processing JSON failed. (%s)"%e)
 
+        ## Features removed because not supportable by new site form:
+        ## averrating metadata entry
+        ## order_chapters_by_date option
+        ## use_meta_keywords option
         return
 
-
     def getPageText(self, raw_page, url):
-        # logger.debug('Getting page text')
-#         logger.debug(soup)
+        logger.debug('Getting page text')
         raw_page = raw_page.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
-#         logger.debug("\tChapter text: %s" % raw_page)
+        # logger.debug("\tChapter text: %s" % raw_page)
         page_soup = self.make_soup(raw_page)
-        [comment.extract() for comment in page_soup.findAll(text=lambda text:isinstance(text, Comment))]
-        story2 = page_soup.find('div', 'aa_ht').div
-#         logger.debug('getPageText - story2: %s' % story2)
-
-        fullhtml = unicode(story2)
-#         logger.debug(fullhtml)
-        # Strip some starting and ending tags,
-        fullhtml = re.sub(r'^<div.*?>', r'', fullhtml)
-        fullhtml = re.sub(r'</div>$', r'', fullhtml)
-        fullhtml = re.sub(r'<p></p>$', r'', fullhtml)
-#         logger.debug('getPageText - fullhtml: %s' % fullhtml)
+        [comment.extract() for comment in page_soup.find_all(string=lambda text:isinstance(text, Comment))]
+        fullhtml = ""
+        for aa_ht_div in page_soup.find_all('div', 'aa_ht') + page_soup.select('div[class^="_article__content_"]'):
+            if aa_ht_div.div:
+                html = unicode(aa_ht_div.div)
+                # Strip some starting and ending tags,
+                html = re.sub(r'^<div.*?>', r'', html)
+                html = re.sub(r'</div>$', r'', html)
+                html = re.sub(r'<p></p>$', r'', html)
+                fullhtml = fullhtml + html
+        # logger.debug('getPageText - fullhtml: %s' % fullhtml)
         return fullhtml
 
     def getChapterText(self, url):
@@ -353,9 +432,15 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         raw_page = self.get_request(url)
         page_soup = self.make_soup(raw_page)
         pages = page_soup.find('div',class_='l_bH')
+        if not pages:
+            pages = page_soup.select_one('div._pagination_h0sum_1')
+        if not pages:
+            pages = page_soup.select_one('div.clearfix.panel._pagination_1400x_1')
+        if not pages:
+            pages = page_soup.select_one('div[class^="panel clearfix _pagination_"]')
+        # logger.debug(pages)
 
         fullhtml = ""
-        self.getCategories(page_soup)
         chapter_description = ''
         if self.getConfig("description_in_chapter"):
             chapter_description = page_soup.find("meta", {"name" : "description"})['content']
@@ -366,7 +451,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             ## look for highest numbered page, they're not all listed
             ## when there are many.
 
-            last_page_link = pages.find_all('a', class_='l_bJ')[-1]
+            last_page_links = pages.find_all('a', class_='l_bJ')
+            if not last_page_links:
+                last_page_links = pages.select('a[class^="_pagination__item_"]')
+            last_page_link = last_page_links[-1]
             last_page_no = int(urlparse.parse_qs(last_page_link['href'].split('?')[1])['page'][0])
             # logger.debug(last_page_no)
             for page_no in range(2, last_page_no+1):
@@ -375,7 +463,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
                 raw_page = self.get_request(page_url)
                 fullhtml += self.getPageText(raw_page, url)
 
-#         logger.debug(fullhtml)
+        #logger.debug(fullhtml)
         page_soup = self.make_soup(fullhtml)
         fullhtml = self.utf8FromSoup(url, self.make_soup(fullhtml))
         fullhtml = chapter_description + fullhtml
@@ -383,6 +471,123 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
         return fullhtml
 
+    def get_urls_from_page(self,url,normalize):
+        from ..geturls import get_urls_from_html
+
+        ## hook for logins, etc.
+        self.before_get_urls_from_page(url,normalize)
+
+        # this way it uses User-Agent or other special settings.
+        data = self.get_request(url,usecache=False)
+        soup = self.make_soup(data)
+
+        page_urls = get_urls_from_html(soup, url, configuration=self.configuration, normalize=normalize)
+
+        if not self.getConfig("fetch_stories_from_api",True):
+            logger.debug('fetch_stories_from_api Not enabled')
+            return {'urllist': page_urls}
+
+        user_story_list = re.search(r'literotica\.com/authors/.+?/lists\?listid=(?P<list_id>\d+)', url)
+        fav_authors = re.search(r'literotica\.com/authors/.+?/favorites', url)
+        written = re.search(r'literotica.com/authors/.+?/works/', url)
+        logger.debug((bool(user_story_list), bool(fav_authors), bool(written)))
+
+        # If the url is not supported
+        if not user_story_list and not fav_authors and not written:
+            logger.debug('No supported link. %s', url)
+            return {'urllist':page_urls}
+
+        # Grabbing the main list where chapters are contained.
+        if user_story_list:
+            js_story_list = re.search(r';\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]\);\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]=\{success:!\d,current_page:(?P<current_page>\d+?),last_page:(?P<last_page>\d+?),total:\d+?,per_page:\d+,(has_series:!\d)?data:\$R\[\d+?\]=\[\$R\[\d+?\]=(?P<data>.+)\}\]\}\);', data) # }] } } });  \$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]\);\$R\[\d+?]\(\$R\[\d+?\],\$R\[\d+?\]=\{sliders:
+            logger.debug('user_story_list ID [%s]'%user_story_list.group('list_id'))
+        else:
+            js_story_list = re.search(r'\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]={current_page:(?P<current_page>\d+?),last_page:(?P<last_page>\d+?),total:\d+?,per_page:\d+,(has_series:!\d,)?data:\$R\[\d+\]=\[\$R\[\d+\]=\{(?!aim)(?P<data>.+)\}\);_\$HY\.r\[', data)
+
+        # In case the regex becomes outdated
+        if not js_story_list:
+            logger.debug('Failed to grab data from the js.')
+            return {'urllist':page_urls}
+
+        user = None
+        script_tags = soup.find_all('script')
+        for script in script_tags:
+            if not script.string:
+                continue
+            # Getting author from the js.
+            user = re.search(r'_\$HY\.r\[\"AuthorQuery\[\\\"(?P<author>.+?)\\\"\]\"\]', script.string)
+            if user != None:
+                logger.debug("User: [%s]"%user.group('author'))
+                break
+        else:
+            logger.debug('Failed to get a username')
+            return {'urllist': page_urls}
+
+        # Extract the current (should be 1) and last page numbers from the js.
+        logger.debug("Pages %s/%s"%(js_story_list.group('current_page'), js_story_list.group('last_page')))
+
+        urls = []
+        # Necessary to format a proper link as there were no visible data specifying what kind of link that should be.
+        cat_to_link = {'adult-comics': 'i', 'erotic-art': 'i', 'illustrated-poetry': 'p', 'erotic-audio-poetry': 'p', 'erotic-poetry': 'p', 'non-erotic-poetry': 'p'}
+        stories_found = re.findall(r"category_info:\$R\[.*?type:\".+?\",pageUrl:\"(.+?)\"}.+?,type:\"(.+?)\",url:\"(.+?)\",", js_story_list.group('data'))
+        for story in stories_found:
+            story_category, story_type, story_url = story
+            urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(story_category, 's'), story_url))
+
+        # Removes the duplicates
+        seen = set()
+        urls = [x for x in (page_urls + urls) if not (x in seen or seen.add(x))]
+        logger.debug("Found [%s] stories so far."%len(urls))
+
+        # Sometimes the rest of the stories are burried in the js so no fetching in necessery.
+        if js_story_list.group('last_page') == js_story_list.group('current_page'):
+            return {'urllist': urls}
+
+        user = urlparse.quote(user.group(1))
+        logger.debug("Escaped user: [%s]"%user)
+
+        if written:
+            category = re.search(r"_\$HY\.r\[\"AuthorSeriesAndWorksQuery\[\\\".+?\\\",\\\"\D+?\\\",\\\"(?P<type>\D+?)\\\"\]\"\]=\$R\[\d+?\]=\$R\[\d+?\]\(\$R\[\d+?\]=\{", data)
+        elif fav_authors:
+            category = re.search(r"_\$HY\.r\[\"AuthorFavoriteWorksQuery\[\\\".+?\\\",\\\"(?P<type>\D+?)\\\",\d\]\"\]=\$R\[\d+?\]=\$R\[\d+?\]\(\$R\[\d+?\]={", data)
+
+        if not user_story_list and not category:
+            logger.debug("Type of works not found")
+            return {'urllist': urls}
+
+        last_page = int(js_story_list.group('last_page'))
+        current_page = int(js_story_list.group('current_page')) + 1
+        # Fetching the remaining urls from api. Can't trust the number given about the pages left from a website. Sometimes even the api returns outdated number of pages.
+        while current_page <= last_page:
+            i = len(urls)
+            logger.debug("Pages %s/%s"%(current_page, int(last_page)))
+            if fav_authors:
+                jsn = self.get_request('https://literotica.com/api/3/users/{}/favorite/works?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22type%22%3A%22{}%22%2C%22withSeriesDetails%22%3Atrue%7D'.format(user, current_page, category.group('type')))
+            elif user_story_list:
+                jsn = self.get_request('https://literotica.com/api/3/users/{}/list/{}?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22withSeriesDetails%22%3Atrue%7D'.format(user, user_story_list.group('list_id'), current_page))
+            else:
+                jsn = self.get_request('https://literotica.com/api/3/users/{}/series_and_works?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22sort%22%3A%22date%22%2C%22type%22%3A%22{}%22%2C%22listType%22%3A%22expanded%22%7D'.format(user, current_page, category.group('type')))
+
+            urls_data = json.loads(jsn)
+            last_page = urls_data["last_page"]
+            current_page = int(urls_data["current_page"]) + 1
+            for story in urls_data['data']:
+                #logger.debug('parts' in story)
+                if story['url'] and story.get('work_count') == None:
+                    urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(story["category_info"]["pageUrl"], 's'), str(story['url'])))
+                    continue
+                # Most of the time series has no url specified and contains all of the story links belonging to the series
+                urls.append('https://www.literotica.com/series/se/%s'%str(story['id']))
+                for series_story in story['parts']:
+                    urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(series_story["category_info"]["pageUrl"], 's'), str(series_story['url'])))
+            logger.debug("Found [%s] stories."%(len(urls) - i))
+
+        # Again removing duplicates.
+        seen = set()
+        urls = [x for x in urls if not (x in seen or seen.add(x))]
+
+        logger.debug("Found total of [%s] stories"%len(urls))
+        return {'urllist':urls}
 
 def getClass():
     return LiteroticaSiteAdapter

fanficfare/adapters/adapter_looselugscom.py

@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2018 FanFicFare team
-# Copyright 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-##################################################################################
-### Rewritten by: GComyn on November, 06, 2016
-### Original was adapter_fannation.py
-##################################################################################
-from __future__ import absolute_import
-from .base_efiction_adapter import BaseEfictionAdapter
-
-class LooseLugsComAdapter(BaseEfictionAdapter):
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.looselugs.com'
-
-    @classmethod
-    def getSiteAbbrev(self):
-        return 'looselugs'
-
-    @classmethod
-    def getDateFormat(self):
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        return "%B %d, %Y"
-
-##################################################################################
-### The Efiction Base Adapter uses the Bulk story to retrieve the metadata, but
-### on this site, the Rating is not present in the Bulk page... 
-### so it is not retrieved.
-##################################################################################
-        
-def getClass():
-    return LooseLugsComAdapter

fanficfare/adapters/adapter_lotrgficcom.py

@@ -1,347 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-##############################################################################
-###  Adapted by GComyn
-###  Completed on November, 22, 2016
-##############################################################################
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-class LOTRgficComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.story.setMetadata('siteabbrev','lotrgfic')
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.lotrgfic.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&warning=3"
-        else:
-            addurl=""
-
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
-            raise exceptions.AdultCheckRequired(self.url)
-        elif "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-
-        ### Main Content for the Table Of Contents page.
-        div = soup.find('div',{'id':'maincontent'})
-
-        divfooter = div.find('div',{'id':'footer'})
-        if divfooter != None:
-            divfooter.extract()
-
-        ## Title
-        a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = div.find('a', href=re.compile(r"viewuser.php"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in div.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
-
-
-        ### Metadata is contained
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        ### This site has the metadata formatted all over the place,
-        ### so we have to do some very cludgy programming to get it.
-        ### If someone can do it better, please do so, and let us know.
-        ## I'm going to leave this section in, so we can get those
-        ## elements that are "formatted correctly".
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## the summary is not encased in a span label... so we can't do anything here.
-                ## I'm going to leave it here just in case.
-                ## Everything until the next span class='label'
-                svalue = ''
-                while value and 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
-                svalue=svalue[:svalue.find('<span class="label">')]
-                self.setDescription(url,svalue)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Places' in label:
-                places = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                placestext = [place.string for place in places]
-                self.warning = ', '.join(placestext)
-                for place in placestext:
-                    self.story.addToList('places',place.string)
-
-            if 'Times' in label:
-                times = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                timestext = [time.string for time in times]
-                self.warning = ', '.join(timestext)
-                for time in timestext:
-                    self.story.addToList('times',time.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
-
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'https://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-        ## Now we are going to cludge together the rest of the metadata
-        metad = soup.findAll('p',{'class':'smaller'})
-        ## Categories don't have a proper label, but do use links, so...
-        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        catstext = [cat.string for cat in cats]
-        for cat in catstext:
-            if cat != None:
-                self.story.addToList('category',cat.string)
-
-        ## Characters don't have a proper label, but do use links, so...
-        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        charstext = [char.string for char in chars]
-        for char in charstext:
-            if char != None:
-                self.story.addToList('characters',char.string)
-
-        ### Rating is not enclosed in a label, only in a p tag classed 'smaller' so...
-        ratng = metad[0].find('strong').get_text().replace('Rated','').strip()
-        self.story.setMetadata('rating', ratng)
-
-        ## No we try to get the summary... it's not within it's own
-        ## dedicated tag, so we have to split some hairs..
-        ## This may not work every time... but I tested it with 6 stories...
-        mdata = metad[0]
-        while '<hr/>' not in unicode(mdata.nextSibling):
-            mdata = mdata.nextSibling
-        self.setDescription(url,mdata.previousSibling.previousSibling.get_text())
-
-        ### the rest of the metadata are not in tags at all... so we have to be really cludgy.
-        ## we don't need the rest of them, so we get rid of all but the last one
-        metad = metad[-1]
-        ## we also don't need any of the links in here, so we'll get rid of them as well.
-        links = metad.findAll('a')
-        for link in links:
-            link.extract()
-        ## and we've already done the labels, so let's remove them
-        labels = metad.findAll('span',{'class':'label'})
-        for label in labels:
-            label.extract()
-        ## now we should only have text and <br>'s... somthing like this:
-            #<p class="smaller">Categories:
-            #<br/>
-            #Characters: , , ,
-            #<br/>
-            # , <br/> <br/> <br/> None<br/>
-            #Challenges: None
-            #<br/>
-            #Series: None
-            #<br/>
-            #Chapters: 1    |    Word count: 200    |    Read Count: 767
-            #<br/>
-            #Completed: Yes    |    Updated: 04/27/13    |    Published: 04/27/13
-            #<br/>
-            #</p>
-        ## we'll have to remove the non-breaking spaces to get this to work.
-        metad = unicode(metad).replace(u"\xa0",'').replace('\n','')
-        for txt in metad.split('<br/>'):
-            if 'Challenges:' in txt:
-                txt = txt.replace('Challenges:','').strip()
-                self.story.setMetadata('challenges', txt)
-            elif 'Series:' in txt:
-                txt = txt.replace('Series:','').strip()
-                self.story.setMetadata('challenges', txt)
-            elif 'Chapters:' in txt:
-                for txt2 in txt.split('|'):
-                    txt2 = txt2.replace('\n','').strip()
-                    if 'Word count:' in txt2:
-                        txt2 = txt2.replace('Word count:','').strip()
-                        self.story.setMetadata('numWords', value)
-                    elif 'Read Count:' in txt2:
-                        txt2= txt2.replace('Read Count:','').strip()
-                        self.story.setMetadata('readings', value)
-            elif 'Completed:' in txt:
-                for txt2 in txt.split('|'):
-                    txt2 = txt2.strip()
-                    if 'Completed:' in txt2:
-                        if 'Yes' in txt2:
-                            self.story.setMetadata('status', 'Completed')
-                        else:
-                            self.story.setMetadata('status', 'In-Progress')
-                    elif 'Updated:' in txt2:
-                        txt2= txt2.replace('Updated:','').strip()
-                        self.story.setMetadata('dateUpdated', makeDate(txt2.strip(), "%b/%d/%y"))
-                    elif 'Published:' in txt2:
-                        txt2= txt2.replace('Published:','').strip()
-                        self.story.setMetadata('datePublished', makeDate(txt2.strip(), "%b/%d/%y"))
-
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self.get_request(url)
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-
-        soup = self.make_soup(data)
-
-        span = soup.find('div', {'id' : 'maincontent'})
-
-        # Everything is encased in the maincontent section, so we have
-        # to remove as much as we can systematically
-        tables = span.findAll('table')
-        for table in tables:
-            table.extract()
-
-        headings = span.findAll('h3')
-        for heading in headings:
-            heading.extract()
-
-        links = span.findAll('a')
-        for link in links:
-            link.extract()
-
-        forms = span.findAll('form')
-        for form in forms:
-            form.extract()
-
-        divs = span.findAll('div')
-        for div in divs:
-            div.extract()
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return LOTRgficComAdapter

fanficfare/adapters/adapter_lumossycophanthexcom.py

@@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         self.story.setMetadata('rating', rating)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
 
@@ -134,7 +134,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
 
         # <span class="label">Rated:</span> NC-17<br /> etc
 
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
 
         value = labels[0].previousSibling
         svalue = ""
@@ -154,22 +154,22 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('numWords', value.split(' -')[0])
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -194,7 +194,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/'+a['href']
 
             seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):

fanficfare/adapters/adapter_masseffect2in.py

@@ -162,7 +162,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
                     self.story.extendList('authorId', [authorId])
                     self.story.extendList('authorUrl', [authorUrl])
 
-                if not self.story.getMetadata('rating'):
+                if not self.story.getMetadataRaw('rating'):
                     ratingTitle = chapter.getRatingTitle()
                     if ratingTitle:
                         self.story.setMetadata('rating', ratingTitle)
@@ -204,7 +204,6 @@ class MassEffect2InAdapter(BaseSiteAdapter):
         self.story.setMetadata('datePublished', datePublished)
         self.story.setMetadata('dateUpdated', dateUpdated)
         self.story.setMetadata('numWords', unicode(wordCount))
-        self.story.setMetadata('numChapters', len(chapters))
 
         # Site-specific metadata.
         self.story.setMetadata('language', self.SITE_LANGUAGE)
@@ -678,7 +677,7 @@ class Chapter(object):
 
     def _excludeEditorSignature(self, root):
         """Exclude editor signature from within `root' element."""
-        for textNode in root.findAll(text=True):
+        for stringNode in root.find_all(string=True):
             if re.match(self.SIGNED_PATTERN, textNode.string):
                 editorLink = textNode.findNext('a')
                 if editorLink:

fanficfare/adapters/adapter_mcstoriescom.py

@@ -64,7 +64,9 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
         return "https://mcstories.com/StoryTitle/ https://mcstories.com/StoryTitle/index.html https://mcstories.com/StoryTitle/StoryTitle1.html"
 
     def getSiteURLPattern(self):
-        return r"https?://(www\.)?mcstories\.com/([a-zA-Z0-9_-]+)/"
+        ## Note that this uses a regular expression *negative*
+        ## lookahead--story URLs *can't* have /Titles/ /Authors/ etc.
+        return r"https?://(www\.)?mcstories\.com(?!/(Titles|Authors|Tags|ReadersPicks)/)/[a-zA-Z0-9_-]+/"
 
     def extractChapterUrlsAndMetadata(self):
         """
@@ -83,7 +85,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
         data1 = self.get_request(self.url)
         soup1 = self.make_soup(data1)
         #strip comments from soup
-        [comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
+        [comment.extract() for comment in soup1.find_all(string=lambda text:isinstance(text, Comment))]
 
         if 'Page Not Found.' in data1:
             raise exceptions.StoryDoesNotExist(self.url)
@@ -93,12 +95,13 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('title', title.text)
 
         # Author
-        author = soup1.find('h3', class_='byline').a
-        authorurl = urlparse.urljoin(self.url, author['href'])
-        self.story.setMetadata('author', author.text)
-        self.story.setMetadata('authorUrl', authorurl)
-        authorid = os.path.splitext(os.path.basename(authorurl))[0]
-        self.story.setMetadata('authorId', authorid)
+        # byline = soup1.find('h3', class_='byline')
+        for author in soup1.select('h3.byline a'):
+            authorurl = urlparse.urljoin(self.url, author['href'])
+            self.story.addToList('author', author.text)
+            self.story.addToList('authorUrl', authorurl)
+            authorid = os.path.splitext(os.path.basename(authorurl))[0]
+            self.story.addToList('authorId', authorid)
 
         # Description
         synopsis = soup1.find('section', class_='synopsis')
@@ -160,7 +163,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
         soup1 = self.make_soup(data1)
 
         #strip comments from soup
-        [comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
+        [comment.extract() for comment in soup1.find_all(string=lambda text:isinstance(text, Comment))]
 
         # get story text
         story1 = soup1.find('article', id='mcstories')

fanficfare/adapters/adapter_mediaminerorg.py

@@ -68,7 +68,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y %H:%M"
+        self.dateformat = "%m.%d.%Y"
 
     @staticmethod
     def getSiteDomain():
@@ -89,12 +89,16 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         ## even newer urls
         ## https://www.mediaminer.org/fanfic/s/gundam-wing-fan-fiction/the-preventer-operatives/171000
         ## https://www.mediaminer.org/fanfic/c/gundam-wing-fan-fiction/the-preventer-operatives/171000/608822
+        ## email urls:
+        ## https://www.mediaminer.org/fanfic/view_ch.php/161297/626395?utm_source=add_chapter&utm_medium=email
+        ## author page urls:
+        ## https://www.mediaminer.org/fanfic/view_st.php?id=145608&submit=View
         return r"https?://"+re.escape(self.getSiteDomain())+r"/fanfic/"+\
             r"((s/(?P<cattitle4>[^/]+)/(?P<urltitle4>[^/]+)/(?P<id4>\d+))|"+\
             r"((c/(?P<cattitle5>[^/]+)/(?P<urltitle5>[^/]+)/(?P<id5>\d+))/\d+)|"+\
             r"(s/(?P<urltitle1>[^/]+)/(?P<id1>\d+))|"+\
             r"((c/(?P<urltitle2>[^/]+)/[^/]+/(?P<id2>\d+))/\d+)|"+\
-            r"(view_st\.php/(?P<id3>\d+)))"
+            r"(view_(st|ch)\.php(/|\?id=)(?P<id3>\d+)))"
 
     # Override stripURLParameters so the id parameter won't get stripped
     @classmethod
@@ -142,21 +146,24 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         for (atag,aurl,name) in [ (x,x['href'],stripHTML(x)) for x in chap_p.find_all('a') ]:
             self.add_chapter(name,'https://'+self.host+aurl)
 
-
         # category
         # <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
-        for a in soup.findAll('a',href=re.compile(r"^/fanfic/a/")):
+        for a in soup.find_all('a',href=re.compile(r"^/fanfic/a/")):
             self.story.addToList('category',a.string)
 
         # genre
         # <a href="/fanfic/src.php/g/567">Ranma 1/2</a>
-        for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")):
+        for a in soup.find_all('a',href=re.compile(r"^/fanfic/src.php/g/")):
             self.story.addToList('genre',a.string)
 
-        metastr = stripHTML(soup.find("div",{"class":"post-meta"}))
+        metasoup = soup.find("div",{"class":"post-meta"})
+        metastr = stripHTML(metasoup)
+        metahtml = unicode(metasoup)
+        
+        self.setDescription(url, metahtml[metahtml.index('</a><br/>')+9:metahtml.index('<br/><b>')])
 
         # Latest Revision: February 07, 2015 15:21 PST
-        m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",metastr)
+        m = re.match(r".*?(?:Latest Revision|Uploaded On): ?(\d\d\.\d\d\.\d\d\d\d) ?",metastr)
         if m:
             self.story.setMetadata('dateUpdated', makeDate(m.group(1), self.dateformat))
             # site doesn't give date published on index page.
@@ -164,19 +171,20 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
             # self.story.setMetadata('datePublished',
             #                        self.story.getMetadataRaw('dateUpdated'))
 
-        # Words: 123456
-        m = re.match(r".*?\| Words: (\d+) \|",metastr)
+        # Words:123 or 23.1K or 1.0M
+        m = re.match(r".*?\| ?Words: ?([\.\d]+)(K|M|) ?\|",metastr)
         if m:
-            self.story.setMetadata('numWords', m.group(1))
-
-        # Summary: ....
-        m = re.match(r".*?Summary: (.*)$",metastr)
-        if m:
-            self.setDescription(url, m.group(1))
-            #self.story.setMetadata('description', m.group(1))
+            if not m.group(2):
+                word_factor = 1
+            elif m.group(2) == 'K':
+                word_factor = 1000
+            elif m.group(2) == 'M':
+                word_factor = 1000000
+            num_words = int(float(m.group(1))*word_factor)
+            self.story.setMetadata('numWords', num_words)
 
         # completed
-        m = re.match(r".*?Status: Completed.*?",metastr)
+        m = re.match(r".*?Status: ?Completed.*?",metastr)
         if m:
             self.story.setMetadata('status','Completed')
         else:
@@ -194,7 +202,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # print("data:%s"%data)
         headerstr = stripHTML(soup.find('div',{'class':'post-meta'}))
 
-        m = re.match(r".*?Uploaded On: ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",headerstr)
+        m = re.match(r".*?Uploaded On: ?(\d\d\.\d\d\.\d\d\d\d)",headerstr)
         if m:
             date = makeDate(m.group(1), self.dateformat)
             if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):

fanficfare/adapters/adapter_merengohu.py

@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2022 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-from .base_efiction_adapter import BaseEfictionAdapter
-
-class MerengoHuAdapter(BaseEfictionAdapter):
-
-    @classmethod
-    def getProtocol(self):
-        return "https"
-
-    @staticmethod
-    def getSiteDomain():
-        return 'merengo.hu'
-
-    @classmethod
-    def getSiteAbbrev(self):
-        return 'merengo'
-
-    @classmethod
-    def getDateFormat(self):
-        return "%Y.%m.%d"
-
-    @classmethod
-    def getBacktoIndex(self):
-        return 'Vissza az indexhez'
-
-    def extractChapterUrlsAndMetadata(self):
-        ## merengo.hu has a custom 18 consent click through
-        self.get_request(self.getUrlForPhp('tizennyolc.php')+'?consent=true')
-
-        ## Call super of extractChapterUrlsAndMetadata().
-        ## base_efiction leaves the soup in self.html.
-        return super(MerengoHuAdapter, self).extractChapterUrlsAndMetadata()
-
-def getClass():
-    return MerengoHuAdapter
-

fanficfare/adapters/adapter_merlinficdtwinscouk.py

@@ -1,272 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-import re
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-# py2 vs py3 transition
-from ..six import text_type as unicode
-
-from .base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return MerlinFicDtwinsCoUk
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class MerlinFicDtwinsCoUk(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','mrfd')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'merlinfic.dtwins.co.uk'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self.post_request(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        data = self.get_request(url)
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self.get_request(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                data = self.get_request(url)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        soup = self.make_soup(data)
-        # print data
-
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while 'label' not in defaultGetattr(value,'class'):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            seriessoup = self.make_soup(self.get_request(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self.get_request(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_midnightwhispers.py

@@ -154,7 +154,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
+        for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
             self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
 
@@ -170,7 +170,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
                 return ""
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
+        labels = soup.find_all('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
             label = labelspan.string
@@ -191,13 +191,13 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
                 catstext = [cat.string for cat in cats]
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
                 charstext = [char.string for char in chars]
                 for char in charstext:
                     self.story.addToList('characters',char.string)
@@ -206,7 +206,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 genrestext = [genre.string for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
@@ -216,7 +216,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 warningstext = [warning.string for warning in warnings]
                 self.warning = ', '.join(warningstext)
                 for warning in warningstext:
@@ -243,7 +243,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
             series_url = 'https://'+self.host+'/'+a['href']
 
             seriessoup = self.make_soup(self.get_request(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
                 # skip 'report this' and 'TOC' links

fanficfare/adapters/adapter_novelfull.py

@@ -40,7 +40,7 @@ class NovelFullSiteAdapter(BaseSiteAdapter):
 
         self.story.setMetadata("title", soup.select_one("h3.title").text)
 
-        for author in soup.find("h3", text="Author:").fetchNextSiblings(
+        for author in soup.find("h3", string="Author:").fetchNextSiblings(
             "a", href=re.compile("/author/")
         ):
             self.story.addToList("authorId", author.text)
@@ -91,7 +91,7 @@ class NovelFullSiteAdapter(BaseSiteAdapter):
         content = soup.find(id="chapter-content")
 
         # Remove chapter header if present
-        chapter_header = content.find(["p", "h3"], text=re.compile(r"Chapter \d+:"))
+        chapter_header = content.find(["p", "h3"], string=re.compile(r"Chapter \d+:"))
 
         if chapter_header:
             chapter_header.decompose()

fanficfare/adapters/adapter_novelonlinefullcom.py

@@ -189,13 +189,13 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter):
                 "Error downloading Chapter: %s!  Missing required element!" % url)
 
         # Some comments we will get is invalid. Remove them all.
-        [comment.extract() for comment in story.find_all(text=lambda text:isinstance(text, Comment))]
+        [comment.extract() for comment in story.find_all(string=lambda text:isinstance(text, Comment))]
 
         # We don't need links. They have a bad css and they are not working most of times.
         [a.extract() for a in story.find_all('a')]
 
         # Some tags have non-standard tag name.
-        for tag in story.findAll(recursive=True):
+        for tag in story.find_all(recursive=True):
             if tag.name not in HTML_TAGS:
                 tag.name = 'span'
 

fanficfare/adapters/adapter_noveltrovecom.py

@@ -1,149 +0,0 @@
-#  -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-####################################################################################################
-## Adapted by GComyn on April 22, 2017
-####################################################################################################
-
-from __future__ import absolute_import
-import logging
-import re
-import sys  # ## used for debug purposes
-
-# py2 vs py3 transition
-
-from .base_adapter import BaseSiteAdapter, makeDate
-
-from .. import exceptions as exceptions
-from ..htmlcleanup import stripHTML
-
-logger = logging.getLogger(__name__)
-
-####################################################################################################
-def getClass():
-    return NovelTroveComSiteAdapter
-
-
-####################################################################################################
-class NovelTroveComSiteAdapter(BaseSiteAdapter):
-    ''' This is a site with 1 story per page, so no multiple chapter stories
-        The date is listed (on the newer stories) as a month and a year, so I'll be adding that
-         to the summary, instead of trying to transform it to a date. '''
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult = False
-
-        # get storyId from url
-        # https://noveltrove.com/story/983/put-that-big-cock-in-me
-        self.story.setMetadata('storyId', self.parsedUrl.path.split('/')[2] + '_' + self.parsedUrl.path.split('/')[3])
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ntcom')
-
-        # This is a 1 story/page site, so we will initialize the variable to keep the soup
-        self.html = ''
-        self.endindex = []
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b. '%y"
-
-####################################################################################################
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'noveltrove.com'
-
-####################################################################################################
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://"+cls.getSiteDomain()+"/story/12345/astoryname"
-
-####################################################################################################
-    def getSiteURLPattern(self):
-        return r"https://"+re.escape(self.getSiteDomain())+r"/story/([0-9])+/*(?P<id>[^/]+)"
-
-####################################################################################################
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def doExtractChapterUrlsAndMetadata(self, get_cover=True):
-
-        url = self.url
-
-        data = self.get_request(url)
-
-        soup = self.make_soup(data)
-
-        # Now go hunting for all the meta data we can get
-        metablock = soup.find('div', {'class': 'title-infos'})
-
-        ## Getting Title
-        title = stripHTML(metablock.find('h1'))
-        self.story.setMetadata('title', title)
-
-        ## Getting author
-        author = metablock.find('a', {'class':'author'})
-        self.story.setMetadata('authorId',author['href'].split('/')[1])
-        self.story.setMetadata('authorUrl','https://'+self.host+author['href'])
-        self.story.setMetadata('author',author.string)
-
-        ## Get the categories
-        for tag in metablock.find_all('a', {'class':'story-category'}):
-            self.story.addToList('category',stripHTML(tag))
-
-        ## There is no summary for these stories, so I'm going to take the first
-        ## 250 characters.
-        synopsis = ''
-        pcount = 0
-        for para in soup.find('div', {'class':'body'}).find_all('p'):
-            synopsis += para.get_text() + ' '
-            pcount += 1
-            if pcount > 10:
-                break
-
-        synopsis = synopsis.strip()[:250] + '...'
-
-        self.setDescription(url, synopsis)
-
-        ## Since this is a 1 story/page site, the published and updated dates are the same.
-        dateposted = stripHTML(metablock.find('div', {'class':'date'}))
-        self.story.setMetadata('datePublished', makeDate(dateposted, self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(dateposted, self.dateformat))
-
-        ## This is a 1 story/page site, so we'll keep the soup for the getChapterText function
-        ## the chapterUrl and numChapters need to be set as well
-        self.html = soup
-        self.add_chapter(self.story.getMetadata('title'), url)
-        self.story.setMetadata('status', 'Completed')
-
-        ## Getting the non-standard title page entries
-        copyrt = soup.find('div', {'class':'copyright'}).get_text()
-        self.story.setMetadata('copyright', copyrt)
-
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-        logger.debug('Using data that we got from: %s' % url)
-
-        soup = self.html
-        story = soup.find('div', {'class':'body'})
-
-        if story == None:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,story)

fanficfare/adapters/adapter_novelupdatescc.py

@@ -1,48 +0,0 @@
-#  -*- coding: utf-8 -*-
-
-# Copyright 2020 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-from __future__ import absolute_import
-import logging
-logger = logging.getLogger(__name__)
-
-from .adapter_wuxiaworldco import WuxiaWorldCoSiteAdapter
-
-def getClass():
-    return NovelUpdatesCcSiteAdapter
-
-class NovelUpdatesCcSiteAdapter(WuxiaWorldCoSiteAdapter):
-    DATE_FORMAT = '%Y-%m-%d %H:%M'
-
-    def __init__(self, config, url):
-        WuxiaWorldCoSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev', 'nucc')
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.novelupdates.cc'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.novelupdates.cc','m.novelupdates.cc']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return 'https://%s/story-name' % cls.getSiteDomain()
-
-    def getSiteURLPattern(self):
-        return r'https?://(www|m)\.novelupdates\.cc/(?P<id>[^/]+)(/)?'

fanficfare/adapters/adapter_occlumencysycophanthexcom.py

@@ -137,14 +137,14 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
 
         try:
             # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
+            a = soup.find_all('option')[1]['value']
             self.story.setMetadata('storyId',a.split('=',)[1])
             url = 'http://'+self.host+'/'+a
             soup = self.make_soup(self.get_request(url))
         except:
             pass
 
-        for info in asoup.findAll('table', {'class' : 'border'}):
+        for info in asoup.find_all('table', {'class' : 'border'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
                 self.story.setMetadata('title',stripHTML(a))
@@ -152,7 +152,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
 
 
         # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        chapters=soup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
         if len(chapters) == 0:
             self.add_chapter(self.story.getMetadata('title'),url)
         else:
@@ -171,7 +171,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
             except:
                 return ""
 
-        cats = info.findAll('a',href=re.compile('categories.php'))
+        cats = info.find_all('a',href=re.compile('categories.php'))
         for cat in cats:
             self.story.addToList('category',cat.string)
 
@@ -188,7 +188,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         self.setDescription(url,svalue)
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = info.findAll('b')
+        labels = info.find_all('b')
         for labelspan in labels:
             value = labelspan.nextSibling
             label = stripHTML(labelspan)