mirror of
https://github.com/JimmXinu/FanFicFare.git
synced 2026-05-09 05:21:13 +02:00
Compare commits
No commits in common. "main" and "v4.37.0" have entirely different histories.
182 changed files with 18976 additions and 22625 deletions
|
|
@ -53,7 +53,7 @@ Test versions are available at:
|
|||
- The [test plugin] is posted at MobileRead.
|
||||
- The test version of CLI for pip install is uploaded to the testpypi repository and can be installed with:
|
||||
```
|
||||
pip install --extra-index-url https://test.pypi.org/simple/ --upgrade FanFicFare
|
||||
pip install --extra-index-url https://testpypi.python.org/pypi --upgrade FanFicFare
|
||||
```
|
||||
|
||||
### Other Releases
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ except NameError:
|
|||
from calibre.customize import InterfaceActionBase
|
||||
|
||||
# pulled out from FanFicFareBase for saving in prefs.py
|
||||
__version__ = (4, 57, 7)
|
||||
__version__ = (4, 37, 0)
|
||||
|
||||
## Apparently the name for this class doesn't matter--it was still
|
||||
## 'demo' for the first few versions.
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>, 2018, Jim Miller'
|
||||
|
|
@ -21,9 +22,7 @@ from calibre.gui2.actions import menu_action_unique_name
|
|||
from calibre.gui2.keyboard import ShortcutConfig
|
||||
from calibre.utils.config import config_dir
|
||||
from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
|
||||
|
||||
import fanficfare.six as six
|
||||
from six import text_type as unicode
|
||||
from fanficfare.six import text_type as unicode
|
||||
|
||||
# Global definition of our plugin name. Used for common functions that require this.
|
||||
plugin_name = None
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2021, Jim Miller'
|
||||
|
|
@ -23,8 +24,7 @@ from PyQt5.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QLabel,
|
|||
from calibre.gui2 import dynamic, info_dialog
|
||||
from calibre.gui2.complete2 import EditWithComplete
|
||||
from calibre.gui2.dialogs.confirm_delete import confirm
|
||||
import fanficfare.six as six
|
||||
from six import text_type as unicode
|
||||
from fanficfare.six import text_type as unicode
|
||||
|
||||
try:
|
||||
from calibre.ebooks.covers import generate_cover as cal_generate_cover
|
||||
|
|
@ -371,7 +371,6 @@ class ConfigWidget(QWidget):
|
|||
prefs['suppresstitlesort'] = self.std_columns_tab.suppresstitlesort.isChecked()
|
||||
prefs['authorcase'] = self.std_columns_tab.authorcase.isChecked()
|
||||
prefs['titlecase'] = self.std_columns_tab.titlecase.isChecked()
|
||||
prefs['seriescase'] = self.std_columns_tab.seriescase.isChecked()
|
||||
prefs['setanthologyseries'] = self.std_columns_tab.setanthologyseries.isChecked()
|
||||
|
||||
prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
|
||||
|
|
@ -417,10 +416,6 @@ class ConfigWidget(QWidget):
|
|||
prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
|
||||
prefs['update_existing_only_from_email'] = self.imap_tab.update_existing_only_from_email.isChecked()
|
||||
prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
|
||||
|
||||
prefs['site_split_jobs'] = self.other_tab.site_split_jobs.isChecked()
|
||||
prefs['reconsolidate_jobs'] = self.other_tab.reconsolidate_jobs.isChecked()
|
||||
|
||||
prefs.save_to_db()
|
||||
self.plugin_action.set_popup_mode()
|
||||
|
||||
|
|
@ -761,7 +756,6 @@ class BasicTab(QWidget):
|
|||
tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
|
||||
rejectreasons=rejecturllist.get_reject_reasons(),
|
||||
reasonslabel=_('Add this reason to all URLs added:'),
|
||||
accept_storyurls=True,
|
||||
save_size_name='fff:Add Reject List')
|
||||
d.exec_()
|
||||
if d.result() == d.Accepted:
|
||||
|
|
@ -1100,7 +1094,7 @@ class CalibreCoverTab(QWidget):
|
|||
self.plugin_gen_cover = QRadioButton(_('Plugin %(gc)s')%no_trans,self)
|
||||
self.plugin_gen_cover.setToolTip(_("Use the %(gc)s plugin to create covers.<br>"
|
||||
"Requires that you have the the %(gc)s plugin installed.<br>"
|
||||
"Additional settings are below.")%no_trans)
|
||||
"Additional settings are below."%no_trans))
|
||||
self.gencov_rdgrp.addButton(self.plugin_gen_cover)
|
||||
# always, new only, when no cover from site, inject yes/no...
|
||||
self.plugin_gen_cover.setChecked(prefs['plugin_gen_cover'])
|
||||
|
|
@ -1280,31 +1274,6 @@ class OtherTab(QWidget):
|
|||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
||||
groupbox = QGroupBox()
|
||||
self.l.addWidget(groupbox)
|
||||
|
||||
groupl = QVBoxLayout()
|
||||
groupbox.setLayout(groupl)
|
||||
|
||||
label = QLabel("<h3>"+
|
||||
_("Background Job Settings")+
|
||||
"</h3>"
|
||||
)
|
||||
label.setWordWrap(True)
|
||||
groupl.addWidget(label)
|
||||
|
||||
self.site_split_jobs = QCheckBox(_('Split downloads into separate background jobs by site'),self)
|
||||
self.site_split_jobs.setToolTip(_("Launches a separate background Job for each site in the list of stories to download/update. Otherwise, there will be only one background job."))
|
||||
self.site_split_jobs.setChecked(prefs['site_split_jobs'])
|
||||
groupl.addWidget(self.site_split_jobs)
|
||||
|
||||
self.reconsolidate_jobs = QCheckBox(_('Reconsolidate split downloads before updating library'),self)
|
||||
self.reconsolidate_jobs.setToolTip(_("Hold all downloads/updates launched together until they all finish. Otherwise, there will be a 'Proceed to update' dialog for each site."))
|
||||
self.reconsolidate_jobs.setChecked(prefs['reconsolidate_jobs'])
|
||||
groupl.addWidget(self.reconsolidate_jobs)
|
||||
|
||||
self.l.addSpacing(5)
|
||||
|
||||
label = QLabel(_("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFicFare confirmation dialogs back again."))
|
||||
label.setWordWrap(True)
|
||||
self.l.addWidget(label)
|
||||
|
|
@ -1638,11 +1607,6 @@ class StandardColumnsTab(QWidget):
|
|||
self.setanthologyseries.setChecked(prefs['setanthologyseries'])
|
||||
row.append(self.setanthologyseries)
|
||||
|
||||
self.seriescase = QCheckBox(_('Fix Series Case?'),self)
|
||||
self.seriescase.setToolTip(_("If checked, Calibre's routine for correcting the capitalization of title will be applied.")
|
||||
+"\n"+_("This effects Calibre metadata only, not FanFicFare metadata in title page."))
|
||||
self.seriescase.setChecked(prefs['seriescase'])
|
||||
row.append(self.seriescase)
|
||||
grid = QGridLayout()
|
||||
for rownum, row in enumerate(rows):
|
||||
for colnum, col in enumerate(row):
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ from calibre.gui2 import gprefs
|
|||
show_download_options = 'fff:add new/update dialogs:show_download_options'
|
||||
from calibre.gui2.dialogs.confirm_delete import confirm
|
||||
from calibre.gui2.complete2 import EditWithComplete
|
||||
from fanficfare.exceptions import NotGoingToDownload
|
||||
from fanficfare.six import text_type as unicode, ensure_text
|
||||
|
||||
# pulls in translation files for _() strings
|
||||
|
|
@ -156,6 +155,15 @@ class RejectUrlEntry:
|
|||
|
||||
return retval
|
||||
|
||||
class NotGoingToDownload(Exception):
|
||||
def __init__(self,error,icon='dialog_error.png',showerror=True):
|
||||
self.error=error
|
||||
self.icon=icon
|
||||
self.showerror=showerror
|
||||
|
||||
def __str__(self):
|
||||
return self.error
|
||||
|
||||
class DroppableQTextEdit(QTextEdit):
|
||||
def __init__(self,parent):
|
||||
QTextEdit.__init__(self,parent)
|
||||
|
|
@ -181,32 +189,12 @@ class DroppableQTextEdit(QTextEdit):
|
|||
else:
|
||||
return QTextEdit.insertFromMimeData(self, mime_data)
|
||||
|
||||
class HotKeyedSizePersistedDialog(SizePersistedDialog):
|
||||
|
||||
def __init__(self, gui, save_size_name):
|
||||
super(HotKeyedSizePersistedDialog,self).__init__(gui, save_size_name)
|
||||
self.keys=dict()
|
||||
|
||||
def addCtrlKeyPress(self,key,func):
|
||||
# print("addKeyPress: key(0x%x)"%key)
|
||||
# print("control: 0x%x"%QtCore.Qt.ControlModifier)
|
||||
self.keys[key]=func
|
||||
|
||||
def keyPressEvent(self, event):
|
||||
# print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
|
||||
if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
|
||||
func = self.keys[event.key()]
|
||||
return func()
|
||||
else:
|
||||
return super(HotKeyedSizePersistedDialog,self).keyPressEvent(event)
|
||||
|
||||
class AddNewDialog(HotKeyedSizePersistedDialog):
|
||||
class AddNewDialog(SizePersistedDialog):
|
||||
|
||||
go_signal = pyqtSignal(object, object, object, object)
|
||||
|
||||
def __init__(self, gui, prefs, icon):
|
||||
super(AddNewDialog,self).__init__(gui, 'fff:add new dialog')
|
||||
|
||||
SizePersistedDialog.__init__(self, gui, 'fff:add new dialog')
|
||||
self.prefs = prefs
|
||||
|
||||
self.setMinimumWidth(300)
|
||||
|
|
@ -345,9 +333,6 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.button_box.rejected.connect(self.reject)
|
||||
self.l.addWidget(self.button_box)
|
||||
|
||||
self.addCtrlKeyPress(QtCore.Qt.Key_Return,self.ok_clicked)
|
||||
self.addCtrlKeyPress(QtCore.Qt.Key_Enter,self.ok_clicked) # num pad
|
||||
|
||||
def click_show_download_options(self,x):
|
||||
self.gbf.setVisible(x)
|
||||
gprefs[show_download_options] = x
|
||||
|
|
@ -490,15 +475,14 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.collision.setCurrentIndex(i)
|
||||
|
||||
def get_fff_options(self):
|
||||
retval = dict(self.extraoptions)
|
||||
retval.update( {
|
||||
'fileform': unicode(self.fileform.currentText()),
|
||||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': False, # self.bgmeta.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation'],
|
||||
'do_wordcount':self.prefs['do_wordcount'],
|
||||
} )
|
||||
retval = {
|
||||
'fileform': unicode(self.fileform.currentText()),
|
||||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': False, # self.bgmeta.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation'],
|
||||
'do_wordcount':self.prefs['do_wordcount'],
|
||||
}
|
||||
|
||||
if self.merge:
|
||||
retval['fileform']=='epub'
|
||||
|
|
@ -513,6 +497,7 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
def get_urlstext(self):
|
||||
return unicode(self.url.toPlainText())
|
||||
|
||||
|
||||
class FakeLineEdit():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
|
@ -634,48 +619,6 @@ class UserPassDialog(QDialog):
|
|||
self.status=False
|
||||
self.hide()
|
||||
|
||||
class TOTPDialog(QDialog):
|
||||
'''
|
||||
Need to collect Timebased One Time Password(TOTP) for some sites.
|
||||
'''
|
||||
def __init__(self, gui, site, exception=None):
|
||||
QDialog.__init__(self, gui)
|
||||
self.status=False
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
||||
grid = QGridLayout()
|
||||
self.l.addLayout(grid)
|
||||
|
||||
self.setWindowTitle(_('Time-based One Time Password(TOTP)'))
|
||||
grid.addWidget(QLabel(_("Site requires a Time-based One Time Password(TOTP) for this url:\n%s")%exception.url),0,0,1,2)
|
||||
|
||||
grid.addWidget(QLabel(_("TOTP:")),2,0)
|
||||
self.totp = QLineEdit(self)
|
||||
grid.addWidget(self.totp,2,1)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
self.l.addLayout(horz)
|
||||
|
||||
self.ok_button = QPushButton(_('OK'), self)
|
||||
self.ok_button.clicked.connect(self.ok)
|
||||
horz.addWidget(self.ok_button)
|
||||
|
||||
self.cancel_button = QPushButton(_('Cancel'), self)
|
||||
self.cancel_button.clicked.connect(self.cancel)
|
||||
horz.addWidget(self.cancel_button)
|
||||
|
||||
self.resize(self.sizeHint())
|
||||
|
||||
def ok(self):
|
||||
self.status=True
|
||||
self.hide()
|
||||
|
||||
def cancel(self):
|
||||
self.status=False
|
||||
self.hide()
|
||||
|
||||
def LoopProgressDialog(gui,
|
||||
book_list,
|
||||
foreach_function,
|
||||
|
|
@ -1320,7 +1263,6 @@ class EditTextDialog(SizePersistedDialog):
|
|||
icon=None, title=None, label=None, tooltip=None,
|
||||
read_only=False,
|
||||
rejectreasons=[],reasonslabel=None,
|
||||
accept_storyurls=False,
|
||||
save_size_name='fff:edit text dialog',
|
||||
):
|
||||
SizePersistedDialog.__init__(self, parent, save_size_name)
|
||||
|
|
@ -1334,10 +1276,7 @@ class EditTextDialog(SizePersistedDialog):
|
|||
self.setWindowIcon(icon)
|
||||
self.l.addWidget(self.label)
|
||||
|
||||
if accept_storyurls:
|
||||
self.textedit = DroppableQTextEdit(self)
|
||||
else:
|
||||
self.textedit = QTextEdit(self)
|
||||
self.textedit = QTextEdit(self)
|
||||
self.textedit.setLineWrapMode(QTextEditNoWrap)
|
||||
self.textedit.setReadOnly(read_only)
|
||||
self.textedit.setText(text)
|
||||
|
|
@ -1392,7 +1331,7 @@ class QTextEditPlainPaste(QTextEdit):
|
|||
else:
|
||||
QTextEdit.insertFromMimeData(self, mimeData)
|
||||
|
||||
class IniTextDialog(HotKeyedSizePersistedDialog):
|
||||
class IniTextDialog(SizePersistedDialog):
|
||||
|
||||
def __init__(self, parent, text,
|
||||
icon=None, title=None, label=None,
|
||||
|
|
@ -1400,7 +1339,9 @@ class IniTextDialog(HotKeyedSizePersistedDialog):
|
|||
read_only=False,
|
||||
save_size_name='fff:ini text dialog',
|
||||
):
|
||||
super(IniTextDialog,self).__init__(parent, save_size_name)
|
||||
SizePersistedDialog.__init__(self, parent, save_size_name)
|
||||
|
||||
self.keys=dict()
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
|
@ -1501,6 +1442,19 @@ class IniTextDialog(HotKeyedSizePersistedDialog):
|
|||
# print("call parent accept")
|
||||
return SizePersistedDialog.accept(self)
|
||||
|
||||
def addCtrlKeyPress(self,key,func):
|
||||
# print("addKeyPress: key(0x%x)"%key)
|
||||
# print("control: 0x%x"%QtCore.Qt.ControlModifier)
|
||||
self.keys[key]=func
|
||||
|
||||
def keyPressEvent(self, event):
|
||||
# print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
|
||||
if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
|
||||
func = self.keys[event.key()]
|
||||
return func()
|
||||
else:
|
||||
return SizePersistedDialog.keyPressEvent(self, event)
|
||||
|
||||
def get_plain_text(self):
|
||||
return unicode(self.textedit.toPlainText())
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,29 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
from six.moves import range
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2021, Jim Miller'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import fanficfare.six as six
|
||||
from fanficfare.six import ensure_text, string_types, text_type as unicode
|
||||
|
||||
# import cProfile
|
||||
|
||||
# def do_cprofile(func):
|
||||
# def profiled_func(*args, **kwargs):
|
||||
# profile = cProfile.Profile()
|
||||
# try:
|
||||
# profile.enable()
|
||||
# result = func(*args, **kwargs)
|
||||
# profile.disable()
|
||||
# return result
|
||||
# finally:
|
||||
# profile.print_stats()
|
||||
# return profiled_func
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -17,7 +32,6 @@ import os
|
|||
import re
|
||||
import sys
|
||||
import threading
|
||||
import copy
|
||||
from io import BytesIO
|
||||
from functools import partial
|
||||
from datetime import datetime, time
|
||||
|
|
@ -25,7 +39,7 @@ from string import Template
|
|||
import traceback
|
||||
from collections import defaultdict
|
||||
|
||||
from PyQt5.Qt import (QApplication, QMenu, QTimer, QToolButton, pyqtSignal, QEventLoop)
|
||||
from PyQt5.Qt import (QApplication, QMenu, QTimer, QToolButton, pyqtSignal)
|
||||
|
||||
from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
|
||||
from calibre.ebooks.metadata import MetaInformation
|
||||
|
|
@ -64,14 +78,12 @@ from fanficfare import adapters, exceptions
|
|||
|
||||
from fanficfare.epubutils import (
|
||||
get_dcsource, get_dcsource_chaptercount, get_story_url_from_epub_html,
|
||||
get_story_url_from_zip_html, reset_orig_chapters_epub, get_cover_img)
|
||||
get_story_url_from_zip_html, reset_orig_chapters_epub, get_cover_data)
|
||||
|
||||
from fanficfare.geturls import (
|
||||
get_urls_from_page, get_urls_from_text,get_urls_from_imap,
|
||||
get_urls_from_mime)
|
||||
|
||||
from fanficfare.fff_profile import do_cprofile
|
||||
|
||||
from calibre_plugins.fanficfare_plugin.fff_util import (
|
||||
get_fff_adapter, get_fff_config, get_fff_personalini,
|
||||
get_common_elements)
|
||||
|
|
@ -97,10 +109,9 @@ from calibre_plugins.fanficfare_plugin.prefs import (
|
|||
from calibre_plugins.fanficfare_plugin.dialogs import (
|
||||
AddNewDialog, UpdateExistingDialog,
|
||||
LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
|
||||
RejectListDialog, EmailPassDialog, TOTPDialog,
|
||||
RejectListDialog, EmailPassDialog,
|
||||
save_collisions, question_dialog_all,
|
||||
RejectUrlEntry, IniTextDialog,
|
||||
EditTextDialog)
|
||||
NotGoingToDownload, RejectUrlEntry, IniTextDialog)
|
||||
|
||||
# because calibre immediately transforms html into zip and don't want
|
||||
# to have an 'if html'. db.has_format is cool with the case mismatch,
|
||||
|
|
@ -183,7 +194,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
self.menu.aboutToShow.connect(self.about_to_show_menu)
|
||||
|
||||
self.imap_pass = None
|
||||
self.download_job_manager = DownloadJobManager()
|
||||
|
||||
def initialization_complete(self):
|
||||
# otherwise configured hot keys won't work until the menu's
|
||||
|
|
@ -194,6 +204,20 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
prefs,
|
||||
self.qaction.icon())
|
||||
|
||||
## Kludgey, yes, but with the real configuration inside the
|
||||
## library now, how else would a user be able to change this
|
||||
## setting if it's crashing calibre?
|
||||
def check_macmenuhack(self):
|
||||
try:
|
||||
return self.macmenuhack
|
||||
except:
|
||||
file_path = os.path.join(calibre_config_dir,
|
||||
*("plugins/fanficfare_macmenuhack.txt".split('/')))
|
||||
file_path = os.path.abspath(file_path)
|
||||
logger.debug("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
|
||||
self.macmenuhack = os.access(file_path, os.F_OK)
|
||||
return self.macmenuhack
|
||||
|
||||
accepts_drops = True
|
||||
|
||||
def accept_enter_event(self, event, mime_data):
|
||||
|
|
@ -418,38 +442,30 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
self.reject_list_action = self.create_menu_item_ex(self.menu, _('Reject Selected Books'),
|
||||
unique_name='Reject Selected Books', image='rotate-right.png',
|
||||
triggered=self.reject_list_urls)
|
||||
# self.menu.addSeparator()
|
||||
|
||||
self.add_reject_urls_action = self.create_menu_item_ex(self.menu, _('Add Reject URLs'),
|
||||
image='rotate-right.png',
|
||||
unique_name='Add Reject URLs',
|
||||
shortcut_name=_('Add Reject URLs'),
|
||||
triggered=self.add_reject_urls)
|
||||
# print("platform.system():%s"%platform.system())
|
||||
# print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
|
||||
if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
|
||||
self.menu.addSeparator()
|
||||
self.editpersonalini_action = self.create_menu_item_ex(self.menu, _('Edit personal.ini'),
|
||||
image= 'config.png',
|
||||
unique_name='Edit personal.ini',
|
||||
shortcut_name=_('Edit personal.ini'),
|
||||
triggered=self.editpersonalini)
|
||||
|
||||
self.edit_reject_urls_action = self.create_menu_item_ex(self.menu, _('Edit Reject URLs'),
|
||||
image='rotate-right.png',
|
||||
unique_name='Edit Reject URLs',
|
||||
shortcut_name=_('Edit Reject URLs'),
|
||||
triggered=self.edit_reject_urls)
|
||||
self.config_action = self.create_menu_item_ex(self.menu, _('&Configure FanFicFare'),
|
||||
image= 'config.png',
|
||||
unique_name='Configure FanFicFare',
|
||||
shortcut_name=_('Configure FanFicFare'),
|
||||
triggered=do_user_config)
|
||||
|
||||
self.menu.addSeparator()
|
||||
self.about_action = self.create_menu_item_ex(self.menu, _('About FanFicFare'),
|
||||
image= 'images/icon.png',
|
||||
unique_name='About FanFicFare',
|
||||
shortcut_name=_('About FanFicFare'),
|
||||
triggered=self.about)
|
||||
|
||||
self.editpersonalini_action = self.create_menu_item_ex(self.menu, _('Edit personal.ini'),
|
||||
image= 'config.png',
|
||||
unique_name='Edit personal.ini',
|
||||
shortcut_name=_('Edit personal.ini'),
|
||||
triggered=self.editpersonalini)
|
||||
|
||||
self.config_action = self.create_menu_item_ex(self.menu, _('&Configure FanFicFare'),
|
||||
image= 'config.png',
|
||||
unique_name='Configure FanFicFare',
|
||||
shortcut_name=_('Configure FanFicFare'),
|
||||
triggered=do_user_config)
|
||||
|
||||
self.about_action = self.create_menu_item_ex(self.menu, _('About FanFicFare'),
|
||||
image= 'images/icon.png',
|
||||
unique_name='About FanFicFare',
|
||||
shortcut_name=_('About FanFicFare'),
|
||||
triggered=self.about)
|
||||
self.gui.keyboard.finalize()
|
||||
|
||||
def about(self,checked):
|
||||
|
|
@ -485,35 +501,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
prefs['personal.ini'] = get_resources('plugin-example.ini')
|
||||
prefs.save_to_db()
|
||||
|
||||
def add_reject_urls(self):
|
||||
d = EditTextDialog(self.gui,
|
||||
"http://example.com/story.php?sid=5,"+_("Reason why I rejected it")+"\nhttp://example.com/story.php?sid=6,"+_("Title by Author")+" - "+_("Reason why I rejected it"),
|
||||
# icon=self.windowIcon(),
|
||||
title=_("FanFicFare"),
|
||||
label=_("Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored."),
|
||||
tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
|
||||
rejectreasons=rejecturllist.get_reject_reasons(),
|
||||
reasonslabel=_('Add this reason to all URLs added:'),
|
||||
accept_storyurls=True,
|
||||
save_size_name='fff:Add Reject List')
|
||||
d.exec_()
|
||||
if d.result() == d.Accepted:
|
||||
rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
|
||||
|
||||
def edit_reject_urls(self):
|
||||
with busy_cursor():
|
||||
d = RejectListDialog(self.gui,
|
||||
rejecturllist.get_list(),
|
||||
rejectreasons=rejecturllist.get_reject_reasons(),
|
||||
header=_("Edit Reject URLs List"),
|
||||
show_delete=False,
|
||||
show_all_reasons=False)
|
||||
d.exec_()
|
||||
if d.result() != d.Accepted:
|
||||
return
|
||||
with busy_cursor():
|
||||
rejecturllist.add(d.get_reject_list(),clear=True)
|
||||
|
||||
def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
|
||||
shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
|
||||
unique_name=None):
|
||||
|
|
@ -553,11 +540,11 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
def update_lists(self,checked,add=True):
|
||||
if prefs['addtolists'] or prefs['addtoreadlists']:
|
||||
if not self.is_library_view():
|
||||
self.do_status_message(_('Cannot Update Reading Lists from Device View'), 3000)
|
||||
self.gui.status_bar.show_message(_('Cannot Update Reading Lists from Device View'), 3000)
|
||||
return
|
||||
|
||||
if len(self.gui.library_view.get_selected_ids()) == 0:
|
||||
self.do_status_message(_('No Selected Books to Update Reading Lists'), 3000)
|
||||
self.gui.status_bar.show_message(_('No Selected Books to Update Reading Lists'), 3000)
|
||||
return
|
||||
|
||||
self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
|
||||
|
|
@ -600,36 +587,14 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
if prefs['imapsessionpass']:
|
||||
self.imap_pass = imap_pass
|
||||
|
||||
try:
|
||||
with busy_cursor():
|
||||
self.do_status_message(_('Fetching Story URLs from Email...'),1000)
|
||||
url_list = get_urls_from_imap(prefs['imapserver'],
|
||||
prefs['imapuser'],
|
||||
imap_pass,
|
||||
prefs['imapfolder'],
|
||||
prefs['imapmarkread'],)
|
||||
except Exception as e:
|
||||
det_msg = "".join(traceback.format_exception(*sys.exc_info()))
|
||||
logger.error("Error Fetching Email:\n%s"%det_msg)
|
||||
error_dialog(self.gui,
|
||||
_("Error Fetching Email"),
|
||||
"<p><b>"+
|
||||
_("An error has occurred while FanFicFare was fetching email.")+
|
||||
"</b></p><p>"+
|
||||
_("If it was an authentication error, be aware:")+
|
||||
"<ul><li>"+
|
||||
_("Microsoft mail servers such as live.com, outlook.com and hotmail.com no longer allow third party apps like FanFicFare.")+
|
||||
"</li><li>"+
|
||||
_("Gmail requires Two Factor Authentication and <a href='https://support.google.com/accounts/answer/185833#app-passwords'>Google App Passwords</a>.")+
|
||||
"</li><li>"+
|
||||
_("Usernames or passwords in personal.ini that contain the percent sign(%) must have it escaped as two percent signs(%%).")+
|
||||
"</li></ul>"
|
||||
,
|
||||
det_msg=det_msg,
|
||||
show=True)
|
||||
return
|
||||
|
||||
with busy_cursor():
|
||||
self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
|
||||
url_list = get_urls_from_imap(prefs['imapserver'],
|
||||
prefs['imapuser'],
|
||||
imap_pass,
|
||||
prefs['imapfolder'],
|
||||
prefs['imapmarkread'],)
|
||||
|
||||
## reject will now be redundant with reject check inside
|
||||
## prep_downloads because of change-able story URLs.
|
||||
## Keeping here to because it's the far more common case.
|
||||
|
|
@ -646,7 +611,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
notupdate_list = set([x for x in url_list if not self.do_id_search(adapters.getNormalStoryURL(x))])
|
||||
url_list = url_list - notupdate_list
|
||||
|
||||
self.do_status_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
|
||||
self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
|
||||
|
||||
if prefs['download_from_email_immediately']:
|
||||
## do imap fetch w/o GUI elements
|
||||
|
|
@ -662,7 +627,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
'add_tag':prefs['imaptags'],
|
||||
},"\n".join(url_list))
|
||||
else:
|
||||
self.do_status_message(_('Finished Fetching Story URLs from Email.'),3000)
|
||||
self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
|
||||
|
||||
else:
|
||||
if url_list:
|
||||
|
|
@ -719,12 +684,12 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
return
|
||||
|
||||
with busy_cursor():
|
||||
self.do_status_message(_('Fetching Story URLs from Page...'))
|
||||
self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
|
||||
|
||||
frompage = self.get_urls_from_page(url)
|
||||
url_list = frompage.get('urllist',[])
|
||||
|
||||
self.do_status_message(_('Finished Fetching Story URLs from Page.'),3000)
|
||||
self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
|
||||
|
||||
if url_list:
|
||||
# make a copy before adding to avoid changing passed param
|
||||
|
|
@ -749,7 +714,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
def list_story_urls(self,checked):
|
||||
'''Get list of URLs from existing books.'''
|
||||
if not self.gui.current_view().selectionModel().selectedRows() :
|
||||
self.do_status_message(_('No Selected Books to Get URLs From'),
|
||||
self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
|
||||
3000)
|
||||
return
|
||||
|
||||
|
|
@ -796,12 +761,12 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
def unnew_books(self,checked):
|
||||
'''Get list of URLs from existing books.'''
|
||||
if not self.is_library_view():
|
||||
self.do_status_message(_('Can only UnNew books in library'),
|
||||
self.gui.status_bar.show_message(_('Can only UnNew books in library'),
|
||||
3000)
|
||||
return
|
||||
|
||||
if not self.gui.current_view().selectionModel().selectedRows() :
|
||||
self.do_status_message(_('No Selected Books to Get URLs From'),
|
||||
self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
|
||||
3000)
|
||||
return
|
||||
|
||||
|
|
@ -862,7 +827,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
|
||||
if changed_ids:
|
||||
logger.debug(_('Starting auto conversion of %d books.')%(len(changed_ids)))
|
||||
self.do_status_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
|
||||
self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
|
||||
self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
|
||||
|
||||
def reject_list_urls(self,checked):
|
||||
|
|
@ -877,7 +842,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
book_list = [ self.make_book_from_device_row(x) for x in rows ]
|
||||
|
||||
if len(book_list) == 0 :
|
||||
self.do_status_message(_('No Selected Books have URLs to Reject'), 3000)
|
||||
self.gui.status_bar.show_message(_('No Selected Books have URLs to Reject'), 3000)
|
||||
return
|
||||
|
||||
# Progbar because fetching urls from device epubs can be slow.
|
||||
|
|
@ -953,15 +918,15 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
def update_anthology(self,checked,extraoptions={}):
|
||||
self.check_valid_collision(extraoptions)
|
||||
if not self.get_epubmerge_plugin():
|
||||
self.do_status_message(_('Cannot Make Anthologys without %s')%'EpubMerge 1.3.1+', 3000)
|
||||
self.gui.status_bar.show_message(_('Cannot Make Anthologys without %s')%'EpubMerge 1.3.1+', 3000)
|
||||
return
|
||||
|
||||
if not self.is_library_view():
|
||||
self.do_status_message(_('Cannot Update Books from Device View'), 3000)
|
||||
self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
|
||||
return
|
||||
|
||||
if len(self.gui.library_view.get_selected_ids()) != 1:
|
||||
self.do_status_message(_('Can only update 1 anthology at a time'), 3000)
|
||||
self.gui.status_bar.show_message(_('Can only update 1 anthology at a time'), 3000)
|
||||
return
|
||||
|
||||
db = self.gui.current_db
|
||||
|
|
@ -971,13 +936,13 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
try:
|
||||
with busy_cursor():
|
||||
self.do_status_message(_('Fetching Story URLs for Series...'))
|
||||
self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
|
||||
book_id = self.gui.library_view.get_selected_ids()[0]
|
||||
mergebook = self.make_book_id_only(book_id)
|
||||
self.populate_book_from_calibre_id(mergebook, db)
|
||||
|
||||
if not db.has_format(book_id,'EPUB',index_is_id=True):
|
||||
self.do_status_message(_('Can only Update Epub Anthologies'), 3000)
|
||||
self.gui.status_bar.show_message(_('Can only Update Epub Anthologies'), 3000)
|
||||
return
|
||||
|
||||
tdir = PersistentTemporaryDirectory(prefix='fff_anthology_')
|
||||
|
|
@ -1008,7 +973,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
url_list_text = "\n".join(url_list)
|
||||
|
||||
self.do_status_message(_('Finished Fetching Story URLs for Series.'),3000)
|
||||
self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
|
||||
except NotAnthologyException:
|
||||
# using an exception purely to get outside 'with busy_cursor:'
|
||||
info_dialog(self.gui, _("Cannot Update Anthology"),
|
||||
|
|
@ -1083,14 +1048,14 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
def update_dialog(self,checked,id_list=None,extraoptions={}):
|
||||
if not self.is_library_view():
|
||||
self.do_status_message(_('Cannot Update Books from Device View'), 3000)
|
||||
self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
|
||||
return
|
||||
|
||||
if not id_list:
|
||||
id_list = self.gui.library_view.get_selected_ids()
|
||||
|
||||
if len(id_list) == 0:
|
||||
self.do_status_message(_('No Selected Books to Update'), 3000)
|
||||
self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
|
||||
return
|
||||
|
||||
self.check_valid_collision(extraoptions)
|
||||
|
|
@ -1150,9 +1115,26 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
prefs
|
||||
|
||||
def do_id_search(self,url):
|
||||
## Aug2024 moved site specific search changes to adapters as
|
||||
## classmethod
|
||||
regexp = adapters.get_url_search(url)
|
||||
# older idents can be uri vs url and have | instead of : after
|
||||
# http, plus many sites are now switching to https.
|
||||
regexp = r'identifiers:"~ur(i|l):~^https?%s$"'%(re.sub(r'^https?','',re.escape(url)))
|
||||
# logger.debug(regexp)
|
||||
## Added Jan 2021, adapter_fanfictionnet is keeping title in
|
||||
## URL now, search with and without url title. 'URL changed'
|
||||
## check will still trigger if existing URL has a *different*
|
||||
## url title.
|
||||
## Changed Sep 2021, adapter_fanfictionnet is keeping title in
|
||||
## storyURL now, but if the story title changes, the Jan
|
||||
## solution wasn't finding the existing story.
|
||||
if r"\.fanfiction\.net" in regexp:
|
||||
regexp = re.sub(r"^(?P<keep>.*net/s/\d+/\d+/)(?P<urltitle>[^\$]*)?",
|
||||
r"\g<keep>(.*)",regexp)
|
||||
|
||||
## Jul 2024, added similar handling for scribblehub
|
||||
## https://www.scribblehub.com/series/862913/title
|
||||
if r"\.scribblehub\.com" in regexp:
|
||||
regexp = re.sub(r"^(?P<keep>.*com/series/\d+/\d+/)(?P<urltitle>[^\$]*)?",
|
||||
r"\g<keep>(.*)",regexp)
|
||||
# logger.debug(regexp)
|
||||
retval = self.gui.current_db.search_getting_ids(regexp,None,use_virtual_library=False)
|
||||
# logger.debug(retval)
|
||||
|
|
@ -1195,7 +1177,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
win_title=_("Downloading metadata for stories")
|
||||
status_prefix=_("Fetched metadata for")
|
||||
|
||||
self.do_status_message(status_bar, 3000)
|
||||
self.gui.status_bar.show_message(status_bar, 3000)
|
||||
LoopProgressDialog(self.gui,
|
||||
books,
|
||||
partial(self.prep_download_loop, options = options, merge=merge),
|
||||
|
|
@ -1204,7 +1186,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
win_title=win_title,
|
||||
status_prefix=status_prefix)
|
||||
else:
|
||||
self.do_status_message(_('No valid story URLs entered.'), 3000)
|
||||
self.gui.status_bar.show_message(_('No valid story URLs entered.'), 3000)
|
||||
# LoopProgressDialog calls prep_download_loop for each 'good' story,
|
||||
# prep_download_loop updates book object for each with metadata from site,
|
||||
# LoopProgressDialog calls start_download_job at the end which goes
|
||||
|
|
@ -1254,9 +1236,9 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
def get_story_metadata_only(self,adapter):
|
||||
url = adapter.url
|
||||
## 5 tries, should be enough if user/pass, totp & is_adult
|
||||
## needed, or a couple tries of one or the other
|
||||
for x in [0,1,2,3,4]:
|
||||
## three tries, that's enough if both user/pass & is_adult needed,
|
||||
## or a couple tries of one or the other
|
||||
for x in range(0,2):
|
||||
try:
|
||||
adapter.getStoryMetadataOnly(get_cover=False)
|
||||
except exceptions.FailedToLogin as f:
|
||||
|
|
@ -1267,13 +1249,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
adapter.username = userpass.user.text()
|
||||
adapter.password = userpass.passwd.text()
|
||||
|
||||
except exceptions.NeedTimedOneTimePassword as e:
|
||||
logger.warn("Login Failed, Need Username/Password.")
|
||||
totpdlg = TOTPDialog(self.gui,url,e)
|
||||
totpdlg.exec_() # exec_ will make it act modal
|
||||
if totpdlg.status:
|
||||
adapter.totp = totpdlg.totp.text()
|
||||
|
||||
except exceptions.AdultCheckRequired:
|
||||
if question_dialog_all(self.gui, _('Are You an Adult?'), '<p>'+
|
||||
_("%s requires that you be an adult. Please confirm you are an adult in your locale:")%url,
|
||||
|
|
@ -1285,7 +1260,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# let other exceptions percolate up.
|
||||
return adapter.getStoryMetadataOnly(get_cover=False)
|
||||
|
||||
@do_cprofile
|
||||
# @do_cprofile
|
||||
def prep_download_loop(self,book,
|
||||
options={'fileform':'epub',
|
||||
'collision':ADDNEW,
|
||||
|
|
@ -1319,16 +1294,9 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
if self.reject_url(merge,book):
|
||||
return
|
||||
|
||||
## Check existing for SKIP mode. Again, redundant with below
|
||||
## for when story URL changes, but also kept here to avoid
|
||||
## network hit.
|
||||
identicalbooks = self.do_id_search(url)
|
||||
if collision == SKIP and identicalbooks:
|
||||
raise exceptions.NotGoingToDownload(_("Skipping duplicate story."),"list_remove.png")
|
||||
|
||||
# Dialogs should prevent this case now.
|
||||
if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
|
||||
raise exceptions.NotGoingToDownload(_("Cannot update non-epub format."))
|
||||
raise NotGoingToDownload(_("Cannot update non-epub format."))
|
||||
|
||||
if not book['good']:
|
||||
# book has already been flagged bad for whatever reason.
|
||||
|
|
@ -1421,10 +1389,9 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# logger.debug("search seriesUrl:%s"%self.do_id_search(story.getMetadata('seriesUrl')))
|
||||
if not bgmeta:
|
||||
series = story.getMetadata('series')
|
||||
seriesUrl = story.getMetadata('seriesUrl')
|
||||
if not merge and series and seriesUrl and prefs['checkforseriesurlid']:
|
||||
if not merge and series and prefs['checkforseriesurlid']:
|
||||
# try to find *series anthology* by *seriesUrl* identifier url or uri first.
|
||||
identicalbooks = self.do_id_search(seriesUrl)
|
||||
identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
|
||||
# print("identicalbooks:%s"%identicalbooks)
|
||||
if len(identicalbooks) > 0 and \
|
||||
(prefs['auto_reject_seriesurlid'] or
|
||||
|
|
@ -1456,7 +1423,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
book['is_adult'] = adapter.is_adult
|
||||
book['username'] = adapter.username
|
||||
book['password'] = adapter.password
|
||||
book['totp'] = adapter.totp
|
||||
|
||||
book['icon'] = 'plus.png'
|
||||
book['status'] = _('Add')
|
||||
|
|
@ -1510,7 +1476,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# try to find by identifier url or uri first.
|
||||
identicalbooks = self.do_id_search(url)
|
||||
# logger.debug("identicalbooks:%s"%identicalbooks)
|
||||
mi = None
|
||||
if len(identicalbooks) < 1 and prefs['matchtitleauth']:
|
||||
# find dups
|
||||
mi = MetaInformation(book['title'],book['author'])
|
||||
|
|
@ -1522,38 +1487,10 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
logger.debug("existing found by identifier URL")
|
||||
|
||||
if collision == SKIP and identicalbooks:
|
||||
raise exceptions.NotGoingToDownload(_("Skipping duplicate story."),"list_remove.png")
|
||||
raise NotGoingToDownload(_("Skipping duplicate story."),"list_remove.png")
|
||||
|
||||
if len(identicalbooks) > 1:
|
||||
identicalbooks_msg = _("More than one identical book by Identifier URL or title/author(s)--can't tell which book to update/overwrite.")
|
||||
identicalwhy_msg = _('<b>%(url)s</b> is already in your library more than once.')%{'url':url}
|
||||
if mi:
|
||||
identicalwhy_msg = _('<b>%(title)s</b> by <b>%(author)s</b> is already in your library more than once with different source URLs.')%{'title':mi.title,'author':', '.join(mi.author)}
|
||||
if question_dialog_all(self.gui,
|
||||
_('Download as New Book?'),'''
|
||||
<h3>%s</h3>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
<p>%s</p>
|
||||
<p>%s</p>'''%(_('Download as New Book?'),
|
||||
identicalbooks_msg,
|
||||
identicalwhy_msg,
|
||||
_('Do you want to add a new book for this URL?'),
|
||||
_('New URL: <a href="%(newurl)s">%(newurl)s</a>')%{'newurl':book['url']},
|
||||
_("Click '<b>Yes</b>' to a new book with new URL."),
|
||||
_("Click '<b>No</b>' to skip URL.")),
|
||||
show_copy_button=False,
|
||||
question_name='download_new_dup',
|
||||
question_cache=self.question_cache):
|
||||
book_id = None
|
||||
mi = None
|
||||
book['calibre_id'] = None
|
||||
identicalbooks = []
|
||||
collision = book['collision'] = ADDNEW
|
||||
else:
|
||||
raise exceptions.NotGoingToDownload(identicalbooks_msg,"minusminus.png")
|
||||
raise NotGoingToDownload(_("More than one identical book by Identifier URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
|
||||
|
||||
## changed: add new book when CALIBREONLY if none found.
|
||||
if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
|
||||
|
|
@ -1640,11 +1577,11 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# returns int adjusted for start-end range.
|
||||
urlchaptercount = story.getChapterCount()
|
||||
if chaptercount == urlchaptercount and collision == UPDATE:
|
||||
raise exceptions.NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
|
||||
raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
|
||||
elif chaptercount > urlchaptercount and not (collision == UPDATEALWAYS and adapter.getConfig('force_update_epub_always')):
|
||||
raise exceptions.NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite or force_update_epub_always to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
|
||||
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite or force_update_epub_always to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
|
||||
elif chaptercount == 0:
|
||||
raise exceptions.NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
|
||||
raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
|
||||
|
||||
if collision == OVERWRITE and \
|
||||
db.has_format(book_id,formmapping[fileform],index_is_id=True):
|
||||
|
|
@ -1661,7 +1598,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# updated does have time, use full timestamps.
|
||||
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
|
||||
(lastupdated.time() != time.min and fileupdated > lastupdated):
|
||||
raise exceptions.NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png',showerror=False)
|
||||
raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png',showerror=False)
|
||||
|
||||
# For update, provide a tmp file copy of the existing epub so
|
||||
# it can't change underneath us. Now also overwrite for logpage preserve.
|
||||
|
|
@ -1766,7 +1703,12 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
calonly = False
|
||||
break
|
||||
if calonly:
|
||||
self._do_download_list_completed(book_list,options=options)
|
||||
class NotJob(object):
|
||||
def __init__(self,result):
|
||||
self.failed=False
|
||||
self.result=result
|
||||
notjob = NotJob(book_list)
|
||||
self.download_list_completed(notjob,options=options)
|
||||
return
|
||||
|
||||
self.do_mark_series_anthologies(options.get('mark_anthology_ids',set()))
|
||||
|
|
@ -1796,20 +1738,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
msgl)
|
||||
return
|
||||
|
||||
### *Don't* split anthology.
|
||||
if merge:
|
||||
self.dispatch_bg_job(_("Anthology"), book_list, copy.copy(options), merge)
|
||||
elif prefs['site_split_jobs']: ### YYY Split list into sites, one BG job per site
|
||||
sites_lists = defaultdict(list)
|
||||
[ sites_lists[x['site']].append(x) for x in book_list ]
|
||||
for site in sites_lists.keys():
|
||||
site_list = sites_lists[site]
|
||||
self.dispatch_bg_job(site, site_list, copy.copy(options), merge)
|
||||
else:
|
||||
self.dispatch_bg_job(None, book_list, copy.copy(options), merge)
|
||||
|
||||
def dispatch_bg_job(self, site, book_list, options, merge):
|
||||
options['site'] = site
|
||||
basic_cachefile = PersistentTemporaryFile(suffix='.basic_cache',
|
||||
dir=options['tdir'])
|
||||
options['basic_cache'].save_cache(basic_cachefile.name)
|
||||
|
|
@ -1829,30 +1757,18 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
# get libs from plugin zip.
|
||||
options['plugin_path'] = self.interface_action_base_plugin.plugin_path
|
||||
|
||||
args = ['calibre_plugins.fanficfare_plugin.jobs',
|
||||
'do_download_worker_single',
|
||||
(site, book_list, options, merge)]
|
||||
if site:
|
||||
desc = _('Download %s FanFiction Book(s) for %s') % (sum(1 for x in book_list if x['good']),site)
|
||||
else:
|
||||
desc = _('Download %s FanFiction Book(s)') % sum(1 for x in book_list if x['good'])
|
||||
|
||||
func = 'arbitrary_n'
|
||||
cpus = self.gui.job_manager.server.pool_size
|
||||
args = ['calibre_plugins.fanficfare_plugin.jobs', 'do_download_worker',
|
||||
(book_list, options, cpus, merge)]
|
||||
desc = _('Download %s FanFiction Book(s)') % sum(1 for x in book_list if x['good'])
|
||||
job = self.gui.job_manager.run_job(
|
||||
self.Dispatcher(partial(self.download_list_completed,
|
||||
options=options,merge=merge)),
|
||||
'arbitrary_n',
|
||||
args=args,
|
||||
self.Dispatcher(partial(self.download_list_completed,options=options,merge=merge)),
|
||||
func, args=args,
|
||||
description=desc)
|
||||
self.download_job_manager.get_batch(options['tdir']).add_job(site,job)
|
||||
job.tdir=options['tdir']
|
||||
job.site=site
|
||||
job.orig_book_list = book_list
|
||||
# set as part of job, otherwise *changing* reconsolidate_jobs
|
||||
# after launch could cause job results to be ignored.
|
||||
job.reconsolidate=prefs['reconsolidate_jobs'] # YYY batch update
|
||||
|
||||
self.gui.jobs_pointer.start()
|
||||
self.do_status_message(_('Starting %d FanFicFare Downloads')%len(book_list),3000)
|
||||
self.gui.status_bar.show_message(_('Starting %d FanFicFare Downloads')%len(book_list),3000)
|
||||
|
||||
def do_mark_series_anthologies(self,mark_anthology_ids):
|
||||
if prefs['mark_series_anthologies'] and mark_anthology_ids:
|
||||
|
|
@ -1881,7 +1797,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
else:
|
||||
return None
|
||||
|
||||
@do_cprofile
|
||||
def update_books_loop(self,book,db=None,
|
||||
options={'fileform':'epub',
|
||||
'collision':ADDNEW,
|
||||
|
|
@ -1998,14 +1913,9 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
self.gui.library_view.sort_by_named_field('marked', True)
|
||||
|
||||
logger.debug(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)))
|
||||
self.do_status_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
|
||||
batch = self.download_job_manager.get_batch(options['tdir'])
|
||||
batch.finish_job(options.get('site',None))
|
||||
if batch.all_done():
|
||||
remove_dir(options['tdir'])
|
||||
logger.debug("removed tdir(%s)"%options['tdir'])
|
||||
else:
|
||||
logger.debug("DIDN'T removed tdir(%s)"%options['tdir'])
|
||||
self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
|
||||
remove_dir(options['tdir'])
|
||||
logger.debug("removed tdir")
|
||||
|
||||
if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
|
||||
cp_plugin = self.gui.iactions['Count Pages']
|
||||
|
|
@ -2034,62 +1944,28 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
if prefs['autoconvert'] and all_not_calonly_ids:
|
||||
logger.debug(_('Starting auto conversion of %d books.')%(len(all_ids)))
|
||||
self.do_status_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
|
||||
self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
|
||||
self.gui.iactions['Convert Books'].auto_convert_auto_add(all_not_calonly_ids)
|
||||
|
||||
def download_list_completed(self, job, options={},merge=False):
|
||||
tdir = job.tdir
|
||||
site = job.site
|
||||
logger.debug("Batch Job:%s %s"%(tdir,site))
|
||||
batch = self.download_job_manager.get_batch(tdir)
|
||||
|
||||
if job.failed:
|
||||
# logger.debug(job.orig_book_list)
|
||||
## I don't *think* there would be any harm to modifying
|
||||
## the original book list, but I elect not to chance it.
|
||||
failedjobresult = copy.deepcopy(job.orig_book_list)
|
||||
for x in failedjobresult:
|
||||
if x['good']:
|
||||
## may have failed before reaching BG job.
|
||||
x['good'] = False
|
||||
x['status'] = _('Error')
|
||||
x['added'] = False
|
||||
x['reportorder'] = x['listorder']+10000000 # force to end.
|
||||
x['comment'] = _('Background Job Failed, see Calibre Jobs log.')
|
||||
x['showerror'] = True
|
||||
self.gui.job_exception(job, dialog_title=_('Background Job Failed to Download Stories for (%s)')%job.site)
|
||||
job.result = failedjobresult
|
||||
self.gui.job_exception(job, dialog_title='Failed to Download Stories')
|
||||
return
|
||||
|
||||
if job.reconsolidate: # YYY batch update
|
||||
logger.debug("batch.finish_job(%s)"%site)
|
||||
batch.finish_job(site)
|
||||
|
||||
showsite = None
|
||||
# set as part of job, otherwise *changing* reconsolidate_jobs
|
||||
# after launch could cause job results to be ignored.
|
||||
if job.reconsolidate: # YYY batch update
|
||||
if batch.all_done():
|
||||
book_list = batch.get_results()
|
||||
else:
|
||||
return
|
||||
else:
|
||||
showsite = site
|
||||
book_list = job.result
|
||||
|
||||
return self._do_download_list_completed(book_list, options, merge, showsite)
|
||||
|
||||
def _do_download_list_completed(self, book_list, options={},merge=False,showsite=None):
|
||||
self.previous = self.gui.library_view.currentIndex()
|
||||
db = self.gui.current_db
|
||||
|
||||
book_list = job.result
|
||||
good_list = [ x for x in book_list if x['good'] ]
|
||||
bad_list = [ x for x in book_list if not x['good'] ]
|
||||
chapter_error_list = [ x for x in book_list if 'chapter_error_count' in x ]
|
||||
|
||||
sort_func = lambda x : x.get('reportorder',x['listorder'])
|
||||
good_list = sorted(good_list,key=sort_func)
|
||||
bad_list = sorted(bad_list,key=sort_func)
|
||||
|
||||
try:
|
||||
good_list = sorted(good_list,key=lambda x : x['reportorder'])
|
||||
bad_list = sorted(bad_list,key=lambda x : x['reportorder'])
|
||||
except KeyError:
|
||||
good_list = sorted(good_list,key=lambda x : x['listorder'])
|
||||
bad_list = sorted(bad_list,key=lambda x : x['listorder'])
|
||||
#print("book_list:%s"%book_list)
|
||||
payload = (good_list, bad_list, options)
|
||||
|
||||
msgl = [ _('FanFicFare found <b>%s</b> good and <b>%s</b> bad updates.')%(len(good_list),len(bad_list)) ]
|
||||
|
|
@ -2130,8 +2006,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
|
||||
do_update_func = self.do_download_merge_update
|
||||
else:
|
||||
if showsite:
|
||||
msgl.append(_('Downloading from %s')%showsite)
|
||||
msgl.extend([
|
||||
_('See log for details.'),
|
||||
_('Proceed with updating your library?')])
|
||||
|
|
@ -2152,15 +2026,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
htmllog,
|
||||
msgl)
|
||||
|
||||
def do_status_message(self,message,timeout=0):
|
||||
self.gui.status_bar.show_message(message,timeout)
|
||||
try:
|
||||
QApplication.processEvents(QEventLoop.ProcessEventsFlag.ExcludeUserInputEvents)
|
||||
except:
|
||||
## older versions of qt don't have ExcludeUserInputEvents.
|
||||
## but they also don't need the processEvents() call
|
||||
pass
|
||||
|
||||
def do_proceed_question(self, update_func, payload, htmllog, msgl):
|
||||
msg = '<p>'+'</p>\n<p>'.join(msgl)+ '</p>\n'
|
||||
def proceed_func(*args, **kwargs):
|
||||
|
|
@ -2188,7 +2053,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
good_list = sorted(good_list,key=lambda x : x['listorder'])
|
||||
bad_list = sorted(bad_list,key=lambda x : x['listorder'])
|
||||
|
||||
self.do_status_message(_('Merging %s books.')%total_good)
|
||||
self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
|
||||
|
||||
existingbook = None
|
||||
if 'mergebook' in options:
|
||||
|
|
@ -2213,45 +2078,30 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
## start with None. If no subbook covers, don't force one
|
||||
## here. User can configure FFF to always create/polish a
|
||||
## cover if they want. This is about when we force it.
|
||||
coverimgpath = None
|
||||
coverpath = None
|
||||
coverimgtype = None
|
||||
had_cover = False
|
||||
|
||||
# epubmerge wants a path to cover img on disk
|
||||
def write_image(imgtype,imgdata):
|
||||
tmp = PersistentTemporaryFile(prefix='cover_',
|
||||
suffix='.'+imagetypes[imgtype],
|
||||
dir=options['tdir'])
|
||||
tmp.write(imgdata)
|
||||
tmp.flush()
|
||||
tmp.close()
|
||||
return tmp.name
|
||||
|
||||
## if prior epub had a cover, we should use it again.
|
||||
if mergebook['calibre_id'] and db.has_format(mergebook['calibre_id'],'EPUB',index_is_id=True):
|
||||
(covertype,coverdata) = get_cover_img(db.format(mergebook['calibre_id'],'EPUB',index_is_id=True,as_file=True))
|
||||
if coverdata:
|
||||
had_cover = True
|
||||
coverimgpath = write_image(covertype,coverdata)
|
||||
coverimgtype = covertype
|
||||
logger.debug("prior anthology cover found")
|
||||
|
||||
## look for covers inside the subbooks. Stop at the first
|
||||
## one, which will be used if there isn't a pre-existing
|
||||
## first, look for covers inside the subbooks. Stop at the
|
||||
## first one, which will be used if there isn't a pre-existing
|
||||
## calibre cover.
|
||||
if not coverimgpath:
|
||||
if not coverpath:
|
||||
for book in good_list:
|
||||
(covertype,coverdata) = get_cover_img(book['outfile'])
|
||||
coverdata = get_cover_data(book['outfile'])
|
||||
if coverdata: # found a cover.
|
||||
coverimgpath = write_image(covertype,coverdata)
|
||||
coverimgtype = covertype
|
||||
logger.debug('from subbook coverimgpath:%s'%coverimgpath)
|
||||
(coverimgtype,coverimgdata) = coverdata[4:6]
|
||||
# logger.debug('coverimgtype:%s [%s]'%(coverimgtype,imagetypes[coverimgtype]))
|
||||
tmpcover = PersistentTemporaryFile(suffix='.'+imagetypes[coverimgtype],
|
||||
dir=options['tdir'])
|
||||
tmpcover.write(coverimgdata)
|
||||
tmpcover.flush()
|
||||
tmpcover.close()
|
||||
coverpath = tmpcover.name
|
||||
break
|
||||
# logger.debug('coverpath:%s'%coverpath)
|
||||
|
||||
## if updating an existing book and there is at least one
|
||||
## subbook cover:
|
||||
if not had_cover and coverimgpath and mergebook['calibre_id']:
|
||||
logger.debug("anth cover: using cal cover")
|
||||
if coverpath and mergebook['calibre_id']:
|
||||
# Couldn't find a better way to get the cover path.
|
||||
calcoverpath = os.path.join(db.library_path,
|
||||
db.path(mergebook['calibre_id'], index_is_id=True),
|
||||
|
|
@ -2259,11 +2109,9 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
## if there's an existing cover, use it. Calibre will set
|
||||
## it for us during lots of different actions anyway.
|
||||
if os.path.exists(calcoverpath):
|
||||
coverimgpath = calcoverpath
|
||||
coverpath = calcoverpath
|
||||
|
||||
## Note that this cover will be replaced if 'inject
|
||||
## generated' cover is on
|
||||
logger.debug('coverimgpath:%s'%coverimgpath)
|
||||
# logger.debug('coverpath:%s'%coverpath)
|
||||
mrg_args = [tmp.name,
|
||||
[ x['outfile'] for x in good_list ],]
|
||||
mrg_kwargs = {
|
||||
|
|
@ -2271,7 +2119,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
'titleopt':mergebook['title'],
|
||||
'keepmetadatafiles':True,
|
||||
'source':mergebook['url'],
|
||||
'coverjpgpath':coverimgpath
|
||||
'coverjpgpath':coverpath
|
||||
}
|
||||
logger.debug('anthology_merge_keepsingletocs:%s'%
|
||||
mergebook['anthology_merge_keepsingletocs'])
|
||||
|
|
@ -2300,10 +2148,11 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
good_list = sorted(good_list,key=lambda x : x['listorder'])
|
||||
bad_list = sorted(bad_list,key=lambda x : x['listorder'])
|
||||
|
||||
self.do_status_message(_('FanFicFare Adding/Updating books.'))
|
||||
self.gui.status_bar.show_message(_('FanFicFare Adding/Updating books.'))
|
||||
errorcol_label = self.get_custom_col_label(prefs['errorcol'])
|
||||
lastcheckedcol_label = self.get_custom_col_label(prefs['lastcheckedcol'])
|
||||
|
||||
columns = self.gui.library_view.model().custom_columns
|
||||
if good_list or prefs['mark'] or (bad_list and errorcol_label) or lastcheckedcol_label:
|
||||
LoopProgressDialog(self.gui,
|
||||
good_list+bad_list,
|
||||
|
|
@ -2649,6 +2498,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
db.new_api.set_link_for_authors(author_id_to_link_map)
|
||||
|
||||
# set series link if found.
|
||||
logger.debug("has link_map:%s"%(hasattr(db.new_api,'set_link_map')))
|
||||
## new_api.set_link_map added in Calibre v6.15
|
||||
if hasattr(db.new_api,'set_link_map') and \
|
||||
prefs['set_series_url'] and \
|
||||
|
|
@ -2657,7 +2507,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
series = book['series']
|
||||
if '[' in series: # a few can have a series w/o number
|
||||
series = series[:series.rindex(' [')]
|
||||
logger.debug("Setting series link:%s"%book['all_metadata']['seriesUrl'])
|
||||
db.new_api.set_link_map('series',{series:
|
||||
book['all_metadata']['seriesUrl']})
|
||||
|
||||
|
|
@ -2797,7 +2646,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
addremovefunc(l,
|
||||
book_ids,
|
||||
display_warnings=False,
|
||||
refresh_screen=True)
|
||||
refresh_screen=False)
|
||||
else:
|
||||
if l != '':
|
||||
message="<p>"+_("You configured FanFicFare to automatically update Reading List '%s', but you don't have a list of that name?")%l+"</p>"
|
||||
|
|
@ -2816,7 +2665,7 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
#add_book_ids,
|
||||
book_ids,
|
||||
display_warnings=False,
|
||||
refresh_screen=True)
|
||||
refresh_screen=False)
|
||||
else:
|
||||
if l != '':
|
||||
message="<p>"+_("You configured FanFicFare to automatically update Reading List '%s', but you don't have a list of that name?")%l+"</p>"
|
||||
|
|
@ -2847,9 +2696,6 @@ class FanFicFarePlugin(InterfaceAction):
|
|||
mi.pubdate = book['pubdate']
|
||||
mi.timestamp = book['timestamp']
|
||||
mi.comments = book['comments']
|
||||
if prefs['seriescase']:
|
||||
from calibre.ebooks.metadata.sources.base import fixcase
|
||||
book['series'] = fixcase(book['series'])
|
||||
mi.series = book['series']
|
||||
return mi
|
||||
|
||||
|
|
@ -3201,7 +3047,6 @@ The previously downloaded book is still in the anthology, but FFF doesn't have t
|
|||
|
||||
if prefs['setanthologyseries'] and book['title'] == series:
|
||||
book['series'] = series+' [0]'
|
||||
book['all_metadata']['seriesUrl'] = options.get('anthology_url','')
|
||||
|
||||
# logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
|
||||
if configuration.getConfig('anthology_title_pattern'):
|
||||
|
|
@ -3222,9 +3067,7 @@ The previously downloaded book is still in the anthology, but FFF doesn't have t
|
|||
s = options.get('frompage',{}).get('status','')
|
||||
if s:
|
||||
book['all_metadata']['status'] = s
|
||||
## status into tags only if in include_subject_tags
|
||||
if 'status' in configuration.getConfigList('include_subject_tags'):
|
||||
book['tags'].append(s)
|
||||
book['tags'].append(s)
|
||||
book['tags'].extend(configuration.getConfigList('anthology_tags'))
|
||||
book['all_metadata']['anthology'] = "true"
|
||||
|
||||
|
|
@ -3262,53 +3105,9 @@ def pretty_book(d, indent=0, spacer=' '):
|
|||
# return '\n'.join([(pretty_book(v, indent, spacer)) for v in d])
|
||||
|
||||
if isinstance(d, dict):
|
||||
for k in ('password','username','totp'):
|
||||
for k in ('password','username'):
|
||||
if k in d and d[k]:
|
||||
d[k]=_('(was set, removed for security)')
|
||||
return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer))
|
||||
for k, v in d.items()])
|
||||
return "%s%s"%(kindent, d)
|
||||
|
||||
class DownloadBatch():
|
||||
def __init__(self,tdir=None):
|
||||
self.runningjobs = dict() # keyed by site
|
||||
self.jobsorder = []
|
||||
self.tdir = tdir
|
||||
|
||||
def add_job(self,site,job):
|
||||
self.runningjobs[site]=job
|
||||
self.jobsorder.append(job)
|
||||
|
||||
def finish_job(self,site):
|
||||
try:
|
||||
self.runningjobs.pop(site)
|
||||
except:
|
||||
pass
|
||||
|
||||
def all_done(self):
|
||||
return len(self.runningjobs) == 0
|
||||
|
||||
def get_results(self):
|
||||
retlist = []
|
||||
for j in self.jobsorder:
|
||||
## failed / no result
|
||||
try:
|
||||
iter(j.result)
|
||||
except TypeError:
|
||||
# not iterable abc.Iterable only in newer pythons
|
||||
pass
|
||||
else:
|
||||
retlist.extend(j.result)
|
||||
return retlist
|
||||
|
||||
class DownloadJobManager():
|
||||
def __init__(self):
|
||||
self.batches = {}
|
||||
|
||||
def get_batch(self,batch):
|
||||
if batch not in self.batches:
|
||||
self.batches[batch] = DownloadBatch()
|
||||
return self.batches[batch]
|
||||
|
||||
def remove_batch(self,batch):
|
||||
del self.batches[batch]
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2020, Jim Miller, 2011, Grant Drake <grant.drake@gmail.com>'
|
||||
|
|
@ -16,6 +17,8 @@ from io import StringIO
|
|||
from collections import defaultdict
|
||||
import sys
|
||||
|
||||
from calibre.utils.ipc.server import Empty, Server
|
||||
from calibre.utils.ipc.job import ParallelJob
|
||||
from calibre.utils.date import local_tz
|
||||
|
||||
# pulls in translation files for _() strings
|
||||
|
|
@ -30,11 +33,21 @@ except NameError:
|
|||
#
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
def do_download_worker_single(site,
|
||||
book_list,
|
||||
options,
|
||||
merge,
|
||||
notification=lambda x,y:x):
|
||||
def do_download_worker(book_list,
|
||||
options,
|
||||
cpus,
|
||||
merge=False,
|
||||
notification=lambda x,y:x):
|
||||
'''
|
||||
Coordinator job, to launch child jobs to do downloads.
|
||||
This is run as a worker job in the background to keep the UI more
|
||||
responsive and get around any memory leak issues as it will launch
|
||||
a child job for each book as a worker process
|
||||
'''
|
||||
## Now running one BG proc per site, which downloads for the same
|
||||
## site in serial.
|
||||
logger.info("CPUs:%s"%cpus)
|
||||
server = Server(pool_size=cpus)
|
||||
|
||||
logger.info(options['version'])
|
||||
|
||||
|
|
@ -43,87 +56,142 @@ def do_download_worker_single(site,
|
|||
from calibre.debug import print_basic_debug_info
|
||||
print_basic_debug_info(sys.stderr)
|
||||
|
||||
sites_lists = defaultdict(list)
|
||||
[ sites_lists[x['site']].append(x) for x in book_list if x['good'] ]
|
||||
|
||||
totals = {}
|
||||
# can't do direct assignment in list comprehension? I'm sure it
|
||||
# makes sense to some pythonista.
|
||||
# [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
|
||||
[ totals.update({x['url']:0.0}) for x in book_list if x['good'] ]
|
||||
# logger.debug(sites_lists.keys())
|
||||
|
||||
# Queue all the jobs
|
||||
jobs_running = 0
|
||||
for site in sites_lists.keys():
|
||||
site_list = sites_lists[site]
|
||||
logger.info(_("Launch background process for site %s:")%site + "\n" +
|
||||
"\n".join([ x['url'] for x in site_list ]))
|
||||
# logger.debug([ x['url'] for x in site_list])
|
||||
args = ['calibre_plugins.fanficfare_plugin.jobs',
|
||||
'do_download_site',
|
||||
(site,site_list,options,merge)]
|
||||
job = ParallelJob('arbitrary_n',
|
||||
"site:(%s)"%site,
|
||||
done=None,
|
||||
args=args)
|
||||
job._site_list = site_list
|
||||
job._processed = False
|
||||
server.add_job(job)
|
||||
jobs_running += 1
|
||||
|
||||
# This server is an arbitrary_n job, so there is a notifier available.
|
||||
# Set the % complete to a small number to avoid the 'unavailable' indicator
|
||||
notification(0.01, _('Downloading FanFiction Stories'))
|
||||
from calibre_plugins.fanficfare_plugin import FanFicFareBase
|
||||
fffbase = FanFicFareBase(options['plugin_path'])
|
||||
with fffbase: # so the sys.path was modified while loading the
|
||||
# plug impl.
|
||||
from fanficfare.fff_profile import do_cprofile
|
||||
|
||||
## extra function just so I can easily use the same
|
||||
## @do_cprofile decorator
|
||||
@do_cprofile
|
||||
def profiled_func():
|
||||
count = 0
|
||||
totals = {}
|
||||
# can't do direct assignment in list comprehension? I'm sure it
|
||||
# makes sense to some pythonista.
|
||||
# [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
|
||||
[ totals.update({x['url']:0.0}) for x in book_list if x['good'] ]
|
||||
# logger.debug(sites_lists.keys())
|
||||
|
||||
def do_indiv_notif(percent,msg):
|
||||
# dequeue the job results as they arrive, saving the results
|
||||
count = 0
|
||||
while True:
|
||||
job = server.changed_jobs_queue.get()
|
||||
# logger.debug("job get job._processed:%s"%job._processed)
|
||||
# A job can 'change' when it is not finished, for example if it
|
||||
# produces a notification.
|
||||
msg = None
|
||||
try:
|
||||
## msg = book['url']
|
||||
(percent,msg) = job.notifications.get_nowait()
|
||||
# logger.debug("%s<-%s"%(percent,msg))
|
||||
if percent == 10.0: # Only when signaling d/l done.
|
||||
count += 1
|
||||
totals[msg] = 1.0/len(totals)
|
||||
# logger.info("Finished: %s"%msg)
|
||||
else:
|
||||
totals[msg] = percent/len(totals)
|
||||
notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
|
||||
notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
|
||||
except Empty:
|
||||
pass
|
||||
# without update, is_finished will never be set. however, we
|
||||
# do want to get all the notifications for status so we don't
|
||||
# miss the 'done' ones.
|
||||
job.update(consume_notifications=False)
|
||||
|
||||
do_list = []
|
||||
done_list = []
|
||||
logger.info("\n\n"+_("Downloading FanFiction Stories")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
|
||||
## pass failures from metadata through bg job so all results are
|
||||
## together.
|
||||
# if not job._processed:
|
||||
# sleep(0.5)
|
||||
## Can have a race condition where job.is_finished before
|
||||
## notifications for all downloads have been processed.
|
||||
## Or even after the job has been finished.
|
||||
# logger.debug("job.is_finished(%s) or job._processed(%s)"%(job.is_finished, job._processed))
|
||||
if not job.is_finished:
|
||||
continue
|
||||
|
||||
## only process each job once. We can get more than one loop
|
||||
## after job.is_finished.
|
||||
if not job._processed:
|
||||
# sleep(1)
|
||||
# A job really finished. Get the information.
|
||||
|
||||
## This is where bg proc details end up in GUI log.
|
||||
## job.details is the whole debug log for each proc.
|
||||
logger.info("\n\n" + ("="*80) + " " + job.details.replace('\r',''))
|
||||
# logger.debug("Finished background process for site %s:\n%s"%(job._site_list[0]['site'],"\n".join([ x['url'] for x in job._site_list ])))
|
||||
for b in job._site_list:
|
||||
book_list.remove(b)
|
||||
book_list.extend(job.result)
|
||||
job._processed = True
|
||||
jobs_running -= 1
|
||||
|
||||
## Can't use individual count--I've seen stories all reported
|
||||
## finished before results of all jobs processed.
|
||||
if jobs_running == 0:
|
||||
book_list = sorted(book_list,key=lambda x : x['listorder'])
|
||||
logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
|
||||
|
||||
good_lists = defaultdict(list)
|
||||
bad_lists = defaultdict(list)
|
||||
for book in book_list:
|
||||
if book['good']:
|
||||
do_list.append(book)
|
||||
good_lists[book['status']].append(book)
|
||||
else:
|
||||
done_list.append(book)
|
||||
for book in do_list:
|
||||
# logger.info("%s"%book['url'])
|
||||
done_list.append(do_download_for_worker(book,options,merge,do_indiv_notif))
|
||||
count += 1
|
||||
return finish_download(done_list)
|
||||
return profiled_func()
|
||||
bad_lists[book['status']].append(book)
|
||||
|
||||
def finish_download(donelist):
|
||||
book_list = sorted(donelist,key=lambda x : x['listorder'])
|
||||
logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
|
||||
order = [_('Add'),
|
||||
_('Update'),
|
||||
_('Meta'),
|
||||
_('Different URL'),
|
||||
_('Rejected'),
|
||||
_('Skipped'),
|
||||
_('Bad'),
|
||||
_('Error'),
|
||||
]
|
||||
j = 0
|
||||
for d in [ good_lists, bad_lists ]:
|
||||
for status in order:
|
||||
if d[status]:
|
||||
l = d[status]
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in l])))
|
||||
for book in l:
|
||||
book['reportorder'] = j
|
||||
j += 1
|
||||
del d[status]
|
||||
# just in case a status is added but doesn't appear in order.
|
||||
for status in d.keys():
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in d[status]])))
|
||||
break
|
||||
|
||||
good_lists = defaultdict(list)
|
||||
bad_lists = defaultdict(list)
|
||||
for book in book_list:
|
||||
if book['good']:
|
||||
good_lists[book['status']].append(book)
|
||||
else:
|
||||
bad_lists[book['status']].append(book)
|
||||
|
||||
order = [_('Add'),
|
||||
_('Update'),
|
||||
_('Meta'),
|
||||
_('Different URL'),
|
||||
_('Rejected'),
|
||||
_('Skipped'),
|
||||
_('Bad'),
|
||||
_('Error'),
|
||||
]
|
||||
stnum = 0
|
||||
for d in [ good_lists, bad_lists ]:
|
||||
for status in order:
|
||||
stnum += 1
|
||||
if d[status]:
|
||||
l = d[status]
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in l])))
|
||||
for book in l:
|
||||
# Add prior listorder to 10000 * status num for
|
||||
# ordering of accumulated results with multiple bg
|
||||
# jobs
|
||||
book['reportorder'] = stnum*10000 + book['listorder']
|
||||
del d[status]
|
||||
# just in case a status is added but doesn't appear in order.
|
||||
for status in d.keys():
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in d[status]])))
|
||||
server.close()
|
||||
|
||||
# return the book list as the job result
|
||||
return book_list
|
||||
|
||||
def do_download_site(site,book_list,options,merge,notification=lambda x,y:x):
|
||||
# logger.info(_("Started job for %s")%site)
|
||||
retval = []
|
||||
for book in book_list:
|
||||
# logger.info("%s"%book['url'])
|
||||
retval.append(do_download_for_worker(book,options,merge,notification))
|
||||
notification(10.0,book['url'])
|
||||
return retval
|
||||
|
||||
def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
||||
'''
|
||||
Child job, to download story when run as a worker job
|
||||
|
|
@ -133,13 +201,13 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
fffbase = FanFicFareBase(options['plugin_path'])
|
||||
with fffbase: # so the sys.path was modified while loading the
|
||||
# plug impl.
|
||||
from calibre_plugins.fanficfare_plugin.dialogs import NotGoingToDownload
|
||||
from calibre_plugins.fanficfare_plugin.prefs import (
|
||||
SAVE_YES, SAVE_YES_UNLESS_SITE, OVERWRITE, OVERWRITEALWAYS, UPDATE,
|
||||
UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
|
||||
from calibre_plugins.fanficfare_plugin.wordcount import get_word_count
|
||||
from fanficfare import adapters, writers
|
||||
from fanficfare.epubutils import get_update_data
|
||||
from fanficfare.exceptions import NotGoingToDownload
|
||||
from fanficfare.six import text_type as unicode
|
||||
|
||||
from calibre_plugins.fanficfare_plugin.fff_util import get_fff_config
|
||||
|
|
@ -168,7 +236,6 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
adapter.is_adult = book['is_adult']
|
||||
adapter.username = book['username']
|
||||
adapter.password = book['password']
|
||||
adapter.totp = book['totp']
|
||||
adapter.setChaptersRange(book['begin'],book['end'])
|
||||
|
||||
## each site download job starts with a new copy of the
|
||||
|
|
@ -359,7 +426,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
data = {'smarten_punctuation':True}
|
||||
opts = ALL_OPTS.copy()
|
||||
opts.update(data)
|
||||
O = namedtuple('Options', ' '.join(ALL_OPTS.keys()))
|
||||
O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
|
||||
opts = O(**opts)
|
||||
|
||||
log = Log(level=Log.DEBUG)
|
||||
|
|
@ -392,8 +459,7 @@ def inject_cal_cols(book,story,configuration):
|
|||
if 'calibre_columns' in book:
|
||||
injectini = ['[injected]']
|
||||
extra_valid = []
|
||||
for k in book['calibre_columns'].keys():
|
||||
v = book['calibre_columns'][k]
|
||||
for k, v in six.iteritems(book['calibre_columns']):
|
||||
story.setMetadata(k,v['val'])
|
||||
injectini.append('%s_label:%s'%(k,v['label']))
|
||||
extra_valid.append(k)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -126,7 +126,6 @@ default_prefs['suppressauthorsort'] = False
|
|||
default_prefs['suppresstitlesort'] = False
|
||||
default_prefs['authorcase'] = False
|
||||
default_prefs['titlecase'] = False
|
||||
default_prefs['seriescase'] = False
|
||||
default_prefs['setanthologyseries'] = False
|
||||
default_prefs['mark'] = False
|
||||
default_prefs['mark_success'] = True
|
||||
|
|
@ -198,11 +197,6 @@ default_prefs['auto_reject_from_email'] = False
|
|||
default_prefs['update_existing_only_from_email'] = False
|
||||
default_prefs['download_from_email_immediately'] = False
|
||||
|
||||
|
||||
#default_prefs['single_proc_jobs'] = True # setting and code removed
|
||||
default_prefs['site_split_jobs'] = True
|
||||
default_prefs['reconsolidate_jobs'] = True
|
||||
|
||||
def set_library_config(library_config,db,setting=PREFS_KEY_SETTINGS):
|
||||
db.prefs.set_namespaced(PREFS_NAMESPACE,
|
||||
setting,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -30,12 +30,8 @@ from .. import configurable as configurable
|
|||
|
||||
## must import each adapter here.
|
||||
|
||||
from . import base_adapter
|
||||
from . import base_efiction_adapter
|
||||
from . import adapter_test1
|
||||
from . import adapter_test2
|
||||
from . import adapter_test3
|
||||
from . import adapter_test4
|
||||
from . import adapter_fanfictionnet
|
||||
from . import adapter_fictionalleyarchiveorg
|
||||
from . import adapter_fictionpresscom
|
||||
|
|
@ -65,10 +61,11 @@ from . import adapter_dokugacom
|
|||
from . import adapter_storiesofardacom
|
||||
from . import adapter_ncisfictioncom
|
||||
from . import adapter_fanfiktionde
|
||||
from . import adapter_ponyfictionarchivenet
|
||||
from . import adapter_themasquenet
|
||||
from . import adapter_pretendercentrecom
|
||||
from . import adapter_darksolaceorg
|
||||
from . import adapter_storyroomcom
|
||||
from . import adapter_finestoriescom
|
||||
from . import adapter_dracoandginnycom
|
||||
from . import adapter_wolverineandroguecom
|
||||
from . import adapter_thehookupzonenet
|
||||
|
|
@ -106,10 +103,12 @@ from . import adapter_fireflyfansnet
|
|||
from . import adapter_trekfanfictionnet
|
||||
from . import adapter_wwwutopiastoriescom
|
||||
from . import adapter_sinfuldreamscomunicornfic
|
||||
from . import adapter_sinfuldreamscomwhisperedmuse
|
||||
from . import adapter_sinfuldreamscomwickedtemptation
|
||||
from . import adapter_asianfanficscom
|
||||
from . import adapter_mttjustoncenet
|
||||
from . import adapter_narutoficorg
|
||||
from . import adapter_starskyhutcharchivenet
|
||||
from . import adapter_thedelphicexpansecom
|
||||
from . import adapter_wwwaneroticstorycom
|
||||
from . import adapter_lcfanficcom
|
||||
|
|
@ -118,8 +117,11 @@ from . import adapter_alternatehistorycom
|
|||
from . import adapter_wattpadcom
|
||||
from . import adapter_novelonlinefullcom
|
||||
from . import adapter_wwwnovelallcom
|
||||
from . import adapter_wuxiaworldxyz
|
||||
from . import adapter_novelupdatescc
|
||||
from . import adapter_hentaifoundrycom
|
||||
from . import adapter_mugglenetfanfictioncom
|
||||
from . import adapter_swiorgru
|
||||
from . import adapter_fanficsme
|
||||
from . import adapter_fanfictalkcom
|
||||
from . import adapter_scifistoriescom
|
||||
|
|
@ -127,6 +129,7 @@ from . import adapter_chireadscom
|
|||
from . import adapter_scribblehubcom
|
||||
from . import adapter_fictionlive
|
||||
from . import adapter_thesietchcom
|
||||
from . import adapter_fastnovelsnet
|
||||
from . import adapter_squidgeworldorg
|
||||
from . import adapter_novelfull
|
||||
from . import adapter_psychficcom
|
||||
|
|
@ -138,9 +141,6 @@ from . import adapter_kakuyomujp
|
|||
from . import adapter_fanfictionsfr
|
||||
from . import adapter_touchfluffytail
|
||||
from . import adapter_spiritfanfictioncom
|
||||
from . import adapter_superlove
|
||||
from . import adapter_cfaa
|
||||
from . import adapter_althistorycom
|
||||
|
||||
## This bit of complexity allows adapters to be added by just adding
|
||||
## importing. It eliminates the long if/else clauses we used to need
|
||||
|
|
@ -224,21 +224,6 @@ def get_section_url(url):
|
|||
## return unchanged in that case.
|
||||
return url
|
||||
|
||||
def get_url_search(url):
|
||||
'''
|
||||
For adapters that have story URLs that can change. This is
|
||||
used for searching the Calibre library by identifiers:url for
|
||||
sites (generally) that contain author or title that can
|
||||
change, but also have a unique identifier that doesn't.
|
||||
|
||||
returns a string containing a regexp, not a compiled re object.
|
||||
'''
|
||||
cls = _get_class_for(url)[0]
|
||||
if not cls:
|
||||
## still apply common processing.
|
||||
cls = base_adapter.BaseSiteAdapter
|
||||
return cls.get_url_search(url)
|
||||
|
||||
def getAdapter(config,url,anyurl=False):
|
||||
|
||||
#logger.debug("trying url:"+url)
|
||||
|
|
|
|||
|
|
@ -22,9 +22,9 @@ logger = logging.getLogger(__name__)
|
|||
from .base_otw_adapter import BaseOTWAdapter
|
||||
|
||||
def getClass():
|
||||
return AdastrafanficComAdapter
|
||||
return SquidgeWorldOrgAdapter
|
||||
|
||||
class AdastrafanficComAdapter(BaseOTWAdapter):
|
||||
class SquidgeWorldOrgAdapter(BaseOTWAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseOTWAdapter.__init__(self, config, url)
|
||||
|
|
|
|||
|
|
@ -68,7 +68,9 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%B %d, %Y"
|
||||
self.dateformat = "%Y-%m-%d"
|
||||
|
||||
|
||||
|
||||
## Added because adult-fanfiction.org does send you to
|
||||
## www.adult-fanfiction.org when you go to it and it also moves
|
||||
|
|
@ -137,45 +139,91 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
def getSiteURLPattern(self):
|
||||
return r'https?://(anime|anime2|bleach|books|buffy|cartoon|celeb|comics|ff|games|hp|inu|lotr|manga|movies|naruto|ne|original|tv|xmen|ygo|yuyu)\.adult-fanfiction\.org/story\.php\?no=\d+$'
|
||||
|
||||
##This is not working right now, so I'm commenting it out, but leaving it for future testing
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
#def needToLoginCheck(self, data):
|
||||
##This adapter will always require a login
|
||||
# return True
|
||||
|
||||
# <form name="login" method="post" action="">
|
||||
# <div class="top">E-mail: <span id="sprytextfield1">
|
||||
# <input name="email" type="text" id="email" size="20" maxlength="255" />
|
||||
# <span class="textfieldRequiredMsg">Email is required.</span><span class="textfieldInvalidFormatMsg">Invalid E-mail.</span></span></div>
|
||||
# <div class="top">Password: <span id="sprytextfield2">
|
||||
# <input name="pass1" type="password" id="pass1" size="20" maxlength="32" />
|
||||
# <span class="textfieldRequiredMsg">password is required.</span><span class="textfieldMinCharsMsg">Minimum 8 characters8.</span><span class="textfieldMaxCharsMsg">Exceeded 32 characters.</span></span></div>
|
||||
# <div class="top"><br /> <input name="loginsubmittop" type="hidden" id="loginsubmit" value="TRUE" />
|
||||
# <input type="submit" value="Login" />
|
||||
# </div>
|
||||
# </form>
|
||||
|
||||
|
||||
##This is not working right now, so I'm commenting it out, but leaving it for future testing
|
||||
#def performLogin(self, url, soup):
|
||||
# params = {}
|
||||
|
||||
# if self.password:
|
||||
# params['email'] = self.username
|
||||
# params['pass1'] = self.password
|
||||
# else:
|
||||
# params['email'] = self.getConfig("username")
|
||||
# params['pass1'] = self.getConfig("password")
|
||||
# params['submit'] = 'Login'
|
||||
|
||||
# # copy all hidden input tags to pick up appropriate tokens.
|
||||
# for tag in soup.findAll('input',{'type':'hidden'}):
|
||||
# params[tag['name']] = tag['value']
|
||||
|
||||
# logger.debug("Will now login to URL {0} as {1} with password: {2}".format(url, params['email'],params['pass1']))
|
||||
|
||||
# d = self.post_request(url, params, usecache=False)
|
||||
# d = self.post_request(url, params, usecache=False)
|
||||
# soup = self.make_soup(d)
|
||||
|
||||
#if not (soup.find('form', {'name' : 'login'}) == None):
|
||||
# logger.info("Failed to login to URL %s as %s" % (url, params['email']))
|
||||
# raise exceptions.FailedToLogin(url,params['email'])
|
||||
# return False
|
||||
#else:
|
||||
# return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def doExtractChapterUrlsAndMetadata(self, get_cover=True):
|
||||
|
||||
## You need to have your is_adult set to true to get this story
|
||||
if not (self.is_adult or self.getConfig("is_adult")):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
else:
|
||||
d = self.post_request('https://www.adult-fanfiction.org/globals/ajax/age-verify.php', {"verify":"1"})
|
||||
if "Age verified successfully" not in d:
|
||||
raise exceptions.FailedToDownload("Failed to Verify Age: {0}".format(d))
|
||||
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
# logger.debug(data)
|
||||
|
||||
if "The dragons running the back end of the site can not seem to find the story you are looking for." in data:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: The dragons running the back end of the site can not seem to find the story you are looking for.".format(self.zone, self.getBaseDomain()))
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
##This is not working right now, so I'm commenting it out, but leaving it for future testing
|
||||
#self.performLogin(url, soup)
|
||||
|
||||
|
||||
## Title
|
||||
## Some of the titles have a backslash on the story page, but not on the Author's page
|
||||
## So I am removing it from the title, so it can be found on the Author's page further in the code.
|
||||
## Also, some titles may have extra spaces ' ', and the search on the Author's page removes them,
|
||||
## so I have to here as well. I used multiple replaces to make sure, since I did the same below.
|
||||
h1 = soup.find('h1')
|
||||
# logger.debug("Title:%s"%h1)
|
||||
self.story.setMetadata('title',stripHTML(h1).replace('\\','').replace(' ',' ').replace(' ',' ').replace(' ',' ').strip())
|
||||
a = soup.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a).replace('\\','').replace(' ',' ').replace(' ',' ').replace(' ',' ').strip())
|
||||
|
||||
# Find the chapters from first list only
|
||||
chapters = soup.select_one('select.chapter-select').select('option')
|
||||
for chapter in chapters:
|
||||
self.add_chapter(chapter,self.url+'&chapter='+chapter['value'])
|
||||
# Find the chapters:
|
||||
chapters = soup.find('ul',{'class':'dropdown-content'})
|
||||
for i, chapter in enumerate(chapters.findAll('a')):
|
||||
self.add_chapter(chapter,self.url+'&chapter='+unicode(i+1))
|
||||
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"profile.php\?id=\d+"))
|
||||
a = soup.find('a', href=re.compile(r"profile.php\?no=\d+"))
|
||||
if a == None:
|
||||
# I know that the original author of fanficfare wants to always have metadata,
|
||||
# but I posit that if the story is there, even if we can't get the metadata from the
|
||||
|
|
@ -184,56 +232,140 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('authorUrl','https://www.adult-fanfiction.org')
|
||||
self.story.setMetadata('author','Unknown')
|
||||
logger.warning('There was no author found for the story... Metadata will not be retreived.')
|
||||
self.setDescription(url,'>>>>>>>>>> No Summary Given, Unknown Author <<<<<<<<<<')
|
||||
self.setDescription(url,'>>>>>>>>>> No Summary Given <<<<<<<<<<')
|
||||
else:
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl',a['href'])
|
||||
self.story.setMetadata('author',stripHTML(a))
|
||||
|
||||
## The story page does not give much Metadata, so we go to
|
||||
## the Author's page. Except it's actually a sub-req for
|
||||
## list of author's stories for that subdomain
|
||||
author_Url = 'https://members.{0}/load-user-stories.php?subdomain={1}&uid={2}'.format(
|
||||
self.getBaseDomain(),
|
||||
self.zone,
|
||||
self.story.getMetadata('authorId'))
|
||||
##The story page does not give much Metadata, so we go to the Author's page
|
||||
|
||||
logger.debug('Getting the load-user-stories page: {0}'.format(author_Url))
|
||||
##Get the first Author page to see if there are multiple pages.
|
||||
##AFF doesn't care if the page number is larger than the actual pages,
|
||||
##it will continue to show the last page even if the variable is larger than the actual page
|
||||
author_Url = '{0}&view=story&zone={1}&page=1'.format(self.story.getMetadata('authorUrl'), self.zone)
|
||||
#author_Url = self.story.getMetadata('authorUrl')+'&view=story&zone='+self.zone+'&page=1'
|
||||
|
||||
##I'm resetting the author page to the zone for this story
|
||||
self.story.setMetadata('authorUrl',author_Url)
|
||||
|
||||
logger.debug('Getting the author page: {0}'.format(author_Url))
|
||||
adata = self.get_request(author_Url)
|
||||
|
||||
none_found = "No stories found in this category."
|
||||
if none_found in adata:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: {2}".format(self.zone, self.getBaseDomain(), none_found))
|
||||
if "The member you are looking for does not exist." in adata:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: The member you are looking for does not exist.".format(self.zone, self.getBaseDomain()))
|
||||
#raise exceptions.StoryDoesNotExist(self.zone+'.'+self.getBaseDomain() +" says: The member you are looking for does not exist.")
|
||||
|
||||
asoup = self.make_soup(adata)
|
||||
# logger.debug(asoup)
|
||||
|
||||
story_card = asoup.select_one('div.story-card:has(a[href="{0}"])'.format(url))
|
||||
# logger.debug(story_card)
|
||||
##Getting the number of author pages
|
||||
pages = 0
|
||||
pagination=asoup.find('ul',{'class' : 'pagination'})
|
||||
if pagination:
|
||||
pages = pagination.findAll('li')[-1].find('a')
|
||||
if not pages == None:
|
||||
pages = pages['href'].split('=')[-1]
|
||||
else:
|
||||
pages = 0
|
||||
|
||||
## Category
|
||||
## I've only seen one category per story so far, but just in case:
|
||||
for cat in story_card.select('div.story-card-category'):
|
||||
# remove Category:, old code suggests Located: is also
|
||||
# possible, so removing by <strong>
|
||||
cat.find("strong").decompose()
|
||||
self.story.addToList('category',stripHTML(cat))
|
||||
storya = None
|
||||
##If there is only 1 page of stories, check it to get the Metadata,
|
||||
if pages == 0:
|
||||
a = asoup.findAll('li')
|
||||
for lc2 in a:
|
||||
if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
|
||||
storya = lc2
|
||||
break
|
||||
## otherwise go through the pages
|
||||
else:
|
||||
page=1
|
||||
i=0
|
||||
while i == 0:
|
||||
##We already have the first page, so if this is the first time through, skip getting the page
|
||||
if page != 1:
|
||||
author_Url = '{0}&view=story&zone={1}&page={2}'.format(self.story.getMetadata('authorUrl'), self.zone, unicode(page))
|
||||
logger.debug('Getting the author page: {0}'.format(author_Url))
|
||||
adata = self.get_request(author_Url)
|
||||
##This will probably never be needed, since AFF doesn't seem to care what number you put as
|
||||
## the page number, it will default to the last page, even if you use 1000, for an author
|
||||
## that only hase 5 pages of stories, but I'm keeping it in to appease Saint Justin Case (just in case).
|
||||
if "The member you are looking for does not exist." in adata:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: The member you are looking for does not exist.".format(self.zone, self.getBaseDomain()))
|
||||
# we look for the li element that has the story here
|
||||
asoup = self.make_soup(adata)
|
||||
|
||||
self.setDescription(url,story_card.select_one('div.story-card-description'))
|
||||
a = asoup.findAll('li')
|
||||
for lc2 in a:
|
||||
if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
|
||||
i=1
|
||||
storya = lc2
|
||||
break
|
||||
page = page + 1
|
||||
if page > int(pages):
|
||||
break
|
||||
|
||||
for tag in story_card.select('span.story-tag'):
|
||||
self.story.addToList('eroticatags',stripHTML(tag))
|
||||
|
||||
## created/updates share formatting
|
||||
for meta in story_card.select('div.story-card-meta-item span:last-child'):
|
||||
meta = stripHTML(meta)
|
||||
if 'Created: ' in meta:
|
||||
meta = meta.replace('Created: ','')
|
||||
self.story.setMetadata('datePublished', makeDate(meta, self.dateformat))
|
||||
|
||||
if 'Updated: ' in meta:
|
||||
meta = meta.replace('Updated: ','')
|
||||
self.story.setMetadata('dateUpdated', makeDate(meta, self.dateformat))
|
||||
##Split the Metadata up into a list
|
||||
##We have to change the soup type to a string, then remove the newlines, and double spaces,
|
||||
##then changes the <br/> to '-:-', which seperates the different elemeents.
|
||||
##Then we strip the HTML elements from the string.
|
||||
##There is also a double <br/>, so we have to fix that, then remove the leading and trailing '-:-'.
|
||||
##They are always in the same order.
|
||||
## EDIT 09/26/2016: Had some trouble with unicode errors... so I had to put in the decode/encode parts to fix it
|
||||
liMetadata = unicode(storya).replace('\n','').replace('\r','').replace('\t',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ')
|
||||
liMetadata = stripHTML(liMetadata.replace(r'<br/>','-:-').replace('<!-- <br /-->','-:-'))
|
||||
liMetadata = liMetadata.strip('-:-').strip('-:-').encode('utf-8')
|
||||
for i, value in enumerate(liMetadata.decode('utf-8').split('-:-')):
|
||||
if i == 0:
|
||||
# The value for the title has been manipulated, so may not be the same as gotten at the start.
|
||||
# I'm going to use the href from the storya retrieved from the author's page to determine if it is correct.
|
||||
if storya.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))['href'] != url:
|
||||
raise exceptions.StoryDoesNotExist('Did not find story in author story list: {0}'.format(author_Url))
|
||||
elif i == 1:
|
||||
##Get the description
|
||||
self.setDescription(url,stripHTML(value.strip()))
|
||||
else:
|
||||
# the rest of the values can be missing, so instead of hardcoding the numbers, we search for them.
|
||||
if 'Located :' in value:
|
||||
self.story.setMetadata('category',value.replace(r'>',r'>').replace(r'Located :',r'').strip())
|
||||
elif 'Category :' in value:
|
||||
# Get the Category
|
||||
self.story.setMetadata('category',value.replace(r'>',r'>').replace(r'Located :',r'').strip())
|
||||
elif 'Content Tags :' in value:
|
||||
# Get the Erotic Tags
|
||||
value = stripHTML(value.replace(r'Content Tags :',r'')).strip()
|
||||
for code in re.split(r'\s',value):
|
||||
self.story.addToList('eroticatags',code)
|
||||
elif 'Posted :' in value:
|
||||
# Get the Posted Date
|
||||
value = value.replace(r'Posted :',r'').strip()
|
||||
if value.startswith('008'):
|
||||
# It is unknown how the 200 became 008, but I'm going to change it back here
|
||||
value = value.replace('008','200')
|
||||
elif value.startswith('0000'):
|
||||
# Since the date is showing as 0000,
|
||||
# I'm going to put the memberdate here
|
||||
value = asoup.find('div',{'id':'contentdata'}).find('p').get_text(strip=True).replace('Member Since','').strip()
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
elif 'Edited :' in value:
|
||||
# Get the 'Updated' Edited date
|
||||
# AFF has the time for the Updated date, and we only want the date,
|
||||
# so we take the first 10 characters only
|
||||
value = value.replace(r'Edited :',r'').strip()[0:10]
|
||||
if value.startswith('008'):
|
||||
# It is unknown how the 200 became 008, but I'm going to change it back here
|
||||
value = value.replace('008','200')
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
elif value.startswith('0000') or '-00-' in value:
|
||||
# Since the date is showing as 0000,
|
||||
# or there is -00- in the date,
|
||||
# I'm going to put the Published date here
|
||||
self.story.setMetadata('dateUpdated', self.story.getMetadata('datPublished'))
|
||||
else:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
else:
|
||||
# This catches the blank elements, and the Review and Dragon Prints.
|
||||
# I am not interested in these, so do nothing
|
||||
zzzzzzz=0
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
|
@ -241,11 +373,10 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
chaptertag = soup.select_one('div.chapter-body')
|
||||
chaptertag = soup.find('ul',{'class':'pagination'}).parent.parent.parent.findNextSibling('li')
|
||||
if None == chaptertag:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: {0}! Missing required element!".format(url))
|
||||
## chapter text includes a copy of story title, author,
|
||||
## chapter title, & eroticatags specific to the chapter. Did
|
||||
## before, too.
|
||||
# Change td to a div.
|
||||
chaptertag.name='div'
|
||||
|
||||
return self.utf8FromSoup(url,chaptertag)
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2026 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import re
|
||||
|
||||
from .base_xenforo2forum_adapter import BaseXenForo2ForumAdapter
|
||||
|
||||
def getClass():
|
||||
return AltHistoryComAdapter
|
||||
|
||||
## NOTE: This is a different site than www.alternatehistory.com.
|
||||
|
||||
class AltHistoryComAdapter(BaseXenForo2ForumAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseXenForo2ForumAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ahc')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'althistory.com'
|
||||
|
||||
|
|
@ -49,21 +49,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseOTWAdapter):
|
|||
return ['archiveofourown.org',
|
||||
'archiveofourown.com',
|
||||
'archiveofourown.net',
|
||||
'archiveofourown.gay',
|
||||
'download.archiveofourown.org',
|
||||
'download.archiveofourown.com',
|
||||
'download.archiveofourown.net',
|
||||
'ao3.org',
|
||||
]
|
||||
|
||||
def mod_url_request(self, url):
|
||||
return url
|
||||
|
||||
def mod_url_request(self, url):
|
||||
## add / to *not* replace media.archiveofourown.org
|
||||
if self.getConfig("use_archive_transformativeworks_org",False):
|
||||
return url.replace("/archiveofourown.org","/archive.transformativeworks.org")
|
||||
elif self.getConfig("use_archiveofourown_gay",False):
|
||||
return url.replace("/archiveofourown.org","/archiveofourown.gay")
|
||||
else:
|
||||
return url
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title', title.string)
|
||||
|
||||
# Author
|
||||
author = soup1.find('div',{'class':'story-info'}).find_all('div',{'class':'story-info-bl'})[1].find('a')
|
||||
author = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl'})[1].find('a')
|
||||
authorurl = author['href']
|
||||
self.story.setMetadata('author', author.string)
|
||||
self.story.setMetadata('authorUrl', authorurl)
|
||||
|
|
@ -112,7 +112,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
### add it before the rest of the pages, if any
|
||||
self.add_chapter('1', self.url)
|
||||
|
||||
chapterTable = soup1.find('div',{'class':'pages'}).find_all('a')
|
||||
chapterTable = soup1.find('div',{'class':'pages'}).findAll('a')
|
||||
|
||||
if chapterTable is not None:
|
||||
# Multi-chapter story
|
||||
|
|
@ -124,7 +124,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
self.add_chapter(chapterTitle, chapterUrl)
|
||||
|
||||
|
||||
rated = soup1.find('div',{'class':'story-info'}).find_all('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
|
||||
rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
|
||||
self.story.setMetadata('rating',rated)
|
||||
|
||||
self.story.setMetadata('dateUpdated', makeDate('01/01/2001', '%m/%d/%Y'))
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','asph')
|
||||
|
|
@ -64,10 +64,10 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
|
|
@ -92,7 +92,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
params['intent'] = ''
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/user.php'
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
|
|
@ -130,20 +130,20 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
asoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
|
||||
|
||||
try:
|
||||
# in case link points somewhere other than the first chapter
|
||||
a = soup.find_all('option')[1]['value']
|
||||
a = soup.findAll('option')[1]['value']
|
||||
self.story.setMetadata('storyId',a.split('=',)[1])
|
||||
url = 'https://'+self.host+'/'+a
|
||||
url = 'http://'+self.host+'/'+a
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
except:
|
||||
pass
|
||||
|
||||
for info in asoup.find_all('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
|
||||
for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
|
||||
a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
if a != None:
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
|
@ -151,13 +151,13 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
if len(chapters) == 0:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
for chapter in chapters:
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href'])
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
|
|
@ -170,7 +170,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
except:
|
||||
return ""
|
||||
|
||||
cats = info.find_all('a',href=re.compile('categories.php'))
|
||||
cats = info.findAll('a',href=re.compile('categories.php'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
|
|
@ -188,7 +188,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
## <td><span class="sb"><b>Published:</b> 04/08/2007</td>
|
||||
|
||||
## one story had <b>Updated...</b> in the description. Restrict to sub-table
|
||||
labels = info.find('table').find_all('b')
|
||||
labels = info.find('table').findAll('b')
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
# Find authorid and URL from... author url.
|
||||
mainmeta = soup.find('footer', {'class': 'main-meta'})
|
||||
alist = mainmeta.find('span', string='Author(s)')
|
||||
alist = alist.parent.find_all('a', href=re.compile(r"/profile/u/[^/]+"))
|
||||
alist = alist.parent.findAll('a', href=re.compile(r"/profile/u/[^/]+"))
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('/')[-1])
|
||||
self.story.addToList('authorUrl','https://'+self.host+a['href'])
|
||||
|
|
@ -159,10 +159,10 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
chapters=soup.find('select',{'name':'chapter-nav'})
|
||||
hrefattr=None
|
||||
if chapters:
|
||||
chapters=chapters.find_all('option')
|
||||
chapters=chapters.findAll('option')
|
||||
hrefattr='value'
|
||||
else: # didn't find <select name='chapter-nav', look for alternative
|
||||
chapters=soup.find('div',{'class':'widget--chapters'}).find_all('a')
|
||||
chapters=soup.find('div',{'class':'widget--chapters'}).findAll('a')
|
||||
hrefattr='href'
|
||||
for index, chapter in enumerate(chapters):
|
||||
if chapter.text != 'Foreword' and 'Collapse chapters' not in chapter.text:
|
||||
|
|
@ -202,7 +202,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
# story tags
|
||||
a = mainmeta.find('span',string='Tags')
|
||||
if a:
|
||||
tags = a.parent.find_all('a')
|
||||
tags = a.parent.findAll('a')
|
||||
for tag in tags:
|
||||
self.story.addToList('tags', tag.text)
|
||||
|
||||
|
|
@ -230,7 +230,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
|
||||
# upvote, subs, and views
|
||||
a = soup.find('div',{'class':'title-meta'})
|
||||
spans = a.find_all('span', recursive=False)
|
||||
spans = a.findAll('span', recursive=False)
|
||||
self.story.setMetadata('upvotes', re.search(r'\(([^)]+)', spans[0].find('span').text).group(1))
|
||||
self.story.setMetadata('subscribers', re.search(r'\(([^)]+)', spans[1].find('span').text).group(1))
|
||||
if len(spans) > 2: # views can be private
|
||||
|
|
@ -252,39 +252,13 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
|
||||
data = self.get_request(url)
|
||||
soup = self.make_soup(data)
|
||||
# logger.debug(data)
|
||||
|
||||
ageform = soup.select_one('form[action="/account/toggle_age"]')
|
||||
# logger.debug(ageform)
|
||||
if ageform and (self.is_adult or self.getConfig("is_adult")):
|
||||
params = {}
|
||||
params['is_of_age']=ageform.select_one('input#is_of_age')['value']
|
||||
params['current_url']=ageform.select_one('input#current_url')['value']
|
||||
params['csrf_aff_token']=ageform.select_one('input[name="csrf_aff_token"]')['value']
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/account/mark_over_18'
|
||||
logger.info("Will now toggle age to URL (%s)" % (loginUrl))
|
||||
# logger.debug(params)
|
||||
data = self.post_request(loginUrl, params)
|
||||
soup = self.make_soup(data)
|
||||
# logger.debug(data)
|
||||
|
||||
content = soup.find('div', {'id': 'user-submitted-body'})
|
||||
|
||||
if self.getConfig('inject_chapter_image'):
|
||||
logger.debug("Injecting chapter image")
|
||||
imgdiv = soup.select_one('div#bodyText div.bot-spacer')
|
||||
if imgdiv:
|
||||
content.insert(0, "\n")
|
||||
content.insert(0, imgdiv)
|
||||
content.insert(0, "\n")
|
||||
|
||||
if self.getConfig('inject_chapter_title'):
|
||||
logger.debug("Injecting full-length chapter title")
|
||||
title = soup.find('h1', {'id' : 'chapter-title'}).text
|
||||
newTitle = soup.new_tag('h3')
|
||||
newTitle.string = title
|
||||
content.insert(0, "\n")
|
||||
content.insert(0, newTitle)
|
||||
content.insert(0, "\n")
|
||||
|
||||
return self.utf8FromSoup(url,content)
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Find the chapters:
|
||||
# The update date is with the chapter links... so we will update it here as well
|
||||
for chapter in soup.find_all('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+r"&chapterid=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+r"&chapterid=\d+$")):
|
||||
value = chapter.findNext('td').findNext('td').string.replace('(added on','').replace(')','').strip()
|
||||
self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
|
||||
self.add_chapter(chapter,'https://'+self.getSiteDomain()+chapter['href'])
|
||||
|
|
@ -134,11 +134,11 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Get the MetaData
|
||||
# Erotia Tags
|
||||
tags = soup.find_all('a',href=re.compile(r'/stories/search.php\?selectedcode'))
|
||||
tags = soup.findAll('a',href=re.compile(r'/stories/search.php\?selectedcode'))
|
||||
for tag in tags:
|
||||
self.story.addToList('eroticatags',tag.text)
|
||||
|
||||
for td in soup.find_all('td'):
|
||||
for td in soup.findAll('td'):
|
||||
if len(td.text)>0:
|
||||
if 'Added on:' in td.text and '<table' not in unicode(td):
|
||||
value = td.text.replace('Added on:','').strip()
|
||||
|
|
@ -169,20 +169,20 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload("Error downloading Chapter: {0}! Missing required element!".format(url))
|
||||
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in chaptertag.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in chaptertag.findAll(string=lambda text:isinstance(text, Comment))]
|
||||
|
||||
# BDSM Library basically wraps it's own html around the document,
|
||||
# so we will be removing the script, title and meta content from the
|
||||
# storyblock
|
||||
for tag in chaptertag.find_all('head') + chaptertag.find_all('style') + chaptertag.find_all('title') + chaptertag.find_all('meta') + chaptertag.find_all('o:p') + chaptertag.find_all('link'):
|
||||
for tag in chaptertag.findAll('head') + chaptertag.findAll('style') + chaptertag.findAll('title') + chaptertag.findAll('meta') + chaptertag.findAll('o:p') + chaptertag.findAll('link'):
|
||||
tag.extract()
|
||||
|
||||
for tag in chaptertag.find_all('o:smarttagtype'):
|
||||
for tag in chaptertag.findAll('o:smarttagtype'):
|
||||
tag.name = 'span'
|
||||
|
||||
## I'm going to take the attributes off all of the tags
|
||||
## because they usually refer to the style that we removed above.
|
||||
for tag in chaptertag.find_all(True):
|
||||
for tag in chaptertag.findAll(True):
|
||||
tag.attrs = None
|
||||
|
||||
return self.utf8FromSoup(url,chaptertag)
|
||||
|
|
|
|||
|
|
@ -157,6 +157,9 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
|
|||
|
||||
self.story.addToList('warnings', warning)
|
||||
|
||||
elif key == 'Chapters':
|
||||
self.story.setMetadata('numChapters', int(value))
|
||||
|
||||
elif key == 'Words':
|
||||
# Apparently only numChapters need to be an integer for
|
||||
# some strange reason. Remove possible ',' characters as to
|
||||
|
|
@ -171,7 +174,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
|
|||
# ugly %p(am/pm) hack moved into makeDate so other sites can use it.
|
||||
self.story.setMetadata('dateUpdated', date)
|
||||
|
||||
if self.story.getMetadataRaw('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
|
||||
if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
def getChapterText(self, url):
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
|
||||
value = labels[0].previousSibling
|
||||
svalue = ""
|
||||
|
|
@ -154,22 +154,22 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value.split(' -')[0])
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -194,7 +194,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ class ChireadsComSiteAdapter(BaseSiteAdapter):
|
|||
intro = stripHTML(info.select_one('.inform-inform-txt').span)
|
||||
self.setDescription(self.url, intro)
|
||||
|
||||
for content in soup.find_all('div', {'id': 'content'}):
|
||||
for a in content.find_all('a'):
|
||||
for content in soup.findAll('div', {'id': 'content'}):
|
||||
for a in content.findAll('a'):
|
||||
self.add_chapter(a.get_text(), a['href'])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
## Title
|
||||
## Some stories have a banner that has it's own a tag before the actual text title...
|
||||
## so I'm checking the pagetitle div for all a tags that match the criteria, then taking the last.
|
||||
a = soup.find('div',{'id':'pagetitle'}).find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))[-1]
|
||||
a = soup.find('div',{'id':'pagetitle'}).findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))[-1]
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
|
|
@ -110,7 +110,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
#self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
|
||||
self.add_chapter(chapter,'https://{0}/{1}{2}'.format(self.host, chapter['href'],addURL))
|
||||
|
|
@ -127,7 +127,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
val = labelspan.nextSibling
|
||||
value = unicode('')
|
||||
|
|
@ -149,27 +149,27 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', stripHTML(value))
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Pairing' in label:
|
||||
ships = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
for ship in ships:
|
||||
self.story.addToList('ships',ship.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -196,7 +196,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# this site has several links to each story.
|
||||
|
|
|
|||
|
|
@ -236,21 +236,13 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
|
|||
if commentsdiv:
|
||||
commentsdiv.parent.decompose()
|
||||
|
||||
# three different 'content' tags to look for.
|
||||
# This is the current in Oct 2024
|
||||
content = soup.select_one('[data-editor-viewer="1"]')
|
||||
|
||||
content = soup.select_one('[data-id=rich-content-viewer]')
|
||||
if content is None:
|
||||
# older story? I can't find any of this style in Oct2024
|
||||
content = soup.select_one('[data-id="rich-content-viewer"]')
|
||||
|
||||
if content is None:
|
||||
# olderer story, but used by some older (2018) posts
|
||||
# older story
|
||||
content = soup.select_one('.legacy-journal')
|
||||
|
||||
if content is None:
|
||||
raise exceptions.FailedToDownload(
|
||||
'Could not find story text. Please open a bug with the URL %s' % self.url
|
||||
if content is None:
|
||||
raise exceptions.FailedToDownload(
|
||||
'Could not find story text. Please open a bug with the URL %s' % self.url
|
||||
)
|
||||
|
||||
return self.utf8FromSoup(url, content)
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
params['Submit'] = 'Submit'
|
||||
|
||||
# copy all hidden input tags to pick up appropriate tokens.
|
||||
for tag in soup.find_all('input',{'type':'hidden'}):
|
||||
for tag in soup.findAll('input',{'type':'hidden'}):
|
||||
params[tag['name']] = tag['value']
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
|
||||
|
|
@ -153,7 +153,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
chapters = soup.find('select').find_all('option')
|
||||
chapters = soup.find('select').findAll('option')
|
||||
if len(chapters)==1:
|
||||
self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1')
|
||||
else:
|
||||
|
|
@ -168,7 +168,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
|
||||
|
||||
#grab the rest of the metadata from the author's page
|
||||
for div in asoup.find_all('div'):
|
||||
for div in asoup.findAll('div'):
|
||||
nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
|
||||
if nav != None:
|
||||
break
|
||||
|
|
@ -208,7 +208,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
|
||||
else:
|
||||
asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
|
||||
for div in asoup.find_all('div'):
|
||||
for div in asoup.findAll('div'):
|
||||
nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
|
||||
if nav != None:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -181,13 +181,13 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
|
||||
self.setDescription(url,content.find('blockquote'))
|
||||
|
||||
for genre in content.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
|
||||
for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
for warning in content.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
|
||||
for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
labels = content.find_all('b')
|
||||
labels = content.findAll('b')
|
||||
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
|
|
@ -208,22 +208,22 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
# no selector found, so it's a one-chapter story.
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
allOptions = select.find_all('option', {'value' : re.compile(r'viewstory')})
|
||||
allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
|
||||
for o in allOptions:
|
||||
url = u'https://%s/%s' % ( self.getSiteDomain(),
|
||||
o['value'])
|
||||
|
|
@ -170,14 +170,14 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
if authsoup != None:
|
||||
# last author link with offset should be the 'next' link.
|
||||
authurl = u'https://%s/%s' % ( self.getSiteDomain(),
|
||||
authsoup.find_all('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
|
||||
authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
|
||||
|
||||
# Need author page for most of the metadata.
|
||||
logger.debug("fetching author page: (%s)"%authurl)
|
||||
authsoup = self.make_soup(self.get_request(authurl))
|
||||
#print("authsoup:%s"%authsoup)
|
||||
|
||||
storyas = authsoup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
|
||||
storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
|
||||
for storya in storyas:
|
||||
#print("======storya:%s"%storya)
|
||||
storyblock = storya.findParent('div',{'class':'storybloc'})
|
||||
|
|
@ -194,7 +194,7 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
# Tipo di coppia: Het | Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio | Note: OOC | Avvertimenti: Tematiche delicate<br />
|
||||
# Categoria: <a href="categories.php?catid=1&parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&a=">3</a> recensioni</div>
|
||||
|
||||
cats = noteblock.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
|
|
@ -262,7 +262,7 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
|
||||
|
|
@ -288,11 +288,11 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
# remove any header and 'o:p' tags.
|
||||
for tag in div.find_all("head") + div.find_all("o:p"):
|
||||
for tag in div.findAll("head") + div.findAll("o:p"):
|
||||
tag.extract()
|
||||
|
||||
# change any html and body tags to div.
|
||||
for tag in div.find_all("html") + div.find_all("body"):
|
||||
for tag in div.findAll("html") + div.findAll("body"):
|
||||
tag.name='div'
|
||||
|
||||
# remove extra bogus doctype.
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -144,7 +144,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
|
||||
value = labels[0].previousSibling
|
||||
svalue = ""
|
||||
|
|
@ -164,22 +164,22 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value.split(' -')[0])
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -204,7 +204,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -53,9 +53,6 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
#Setting the 'Zone' for each "Site"
|
||||
self.zone = self.parsedUrl.netloc.replace('.fanficauthors.net','')
|
||||
|
||||
# site change .nsns to -nsns
|
||||
self.zone = self.zone.replace('.nsns','-nsns')
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://{0}.{1}/{2}/'.format(
|
||||
self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
|
||||
|
|
@ -82,10 +79,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
|
||||
# need both .nsns(old) and -nsns(new) because it's a domain
|
||||
# change, not just URL change.
|
||||
return ['aaran-st-vines.nsns.fanficauthors.net',
|
||||
'aaran-st-vines-nsns.fanficauthors.net',
|
||||
'abraxan.fanficauthors.net',
|
||||
'bobmin.fanficauthors.net',
|
||||
'canoncansodoff.fanficauthors.net',
|
||||
|
|
@ -101,12 +95,9 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
'jeconais.fanficauthors.net',
|
||||
'kinsfire.fanficauthors.net',
|
||||
'kokopelli.nsns.fanficauthors.net',
|
||||
'kokopelli-nsns.fanficauthors.net',
|
||||
'ladya.nsns.fanficauthors.net',
|
||||
'ladya-nsns.fanficauthors.net',
|
||||
'lorddwar.fanficauthors.net',
|
||||
'mrintel.nsns.fanficauthors.net',
|
||||
'mrintel-nsns.fanficauthors.net',
|
||||
'musings-of-apathy.fanficauthors.net',
|
||||
'ruskbyte.fanficauthors.net',
|
||||
'seelvor.fanficauthors.net',
|
||||
|
|
@ -117,7 +108,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
################################################################################################
|
||||
@classmethod
|
||||
def getSiteExampleURLs(self):
|
||||
return ("https://aaran-st-vines-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
return ("https://aaran-st-vines.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://abraxan.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://bobmin.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://canoncansodoff.fanficauthors.net/A_Story_Name/ "
|
||||
|
|
@ -132,10 +123,10 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
+ "https://jbern.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://jeconais.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://kinsfire.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://kokopelli-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://ladya-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://kokopelli.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://ladya.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://lorddwar.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://mrintel-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://mrintel.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://musings-of-apathy.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://ruskbyte.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://seelvor.fanficauthors.net/A_Story_Name/ "
|
||||
|
|
@ -145,16 +136,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
|
||||
################################################################################################
|
||||
def getSiteURLPattern(self):
|
||||
## .nsns kept here to match both . and -
|
||||
return r'https?://(aaran-st-vines.nsns|abraxan|bobmin|canoncansodoff|chemprof|copperbadge|crys|deluded-musings|draco664|fp|frenchsession|ishtar|jbern|jeconais|kinsfire|kokopelli.nsns|ladya.nsns|lorddwar|mrintel.nsns|musings-of-apathy|ruskbyte|seelvor|tenhawk|viridian|whydoyouneedtoknow)\.fanficauthors\.net/([a-zA-Z0-9_]+)/'
|
||||
|
||||
@classmethod
|
||||
def get_section_url(cls,url):
|
||||
## only changing .nsns to -nsns and only when part of the
|
||||
## domain.
|
||||
url = url.replace('.nsns.fanficauthors.net','-nsns.fanficauthors.net')
|
||||
return url
|
||||
|
||||
################################################################################################
|
||||
def doExtractChapterUrlsAndMetadata(self, get_cover=True):
|
||||
|
||||
|
|
@ -180,7 +163,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
# Find the chapters:
|
||||
# The published and update dates are with the chapter links...
|
||||
# so we have to get them from there.
|
||||
chapters = soup.find_all('a', href=re.compile('/'+self.story.getMetadata(
|
||||
chapters = soup.findAll('a', href=re.compile('/'+self.story.getMetadata(
|
||||
'storyId')+'/([a-zA-Z0-9_]+)/'))
|
||||
|
||||
# Here we are getting the published date. It is the date the first chapter was "updated"
|
||||
|
|
@ -219,7 +202,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
## Raising AdultCheckRequired after collecting chapters gives
|
||||
## a double chapter list. So does genre, but it de-dups
|
||||
## automatically.
|
||||
if( self.story.getMetadataRaw('rating') in ['Mature','Adult Only']
|
||||
if( self.story.getMetadata('rating') == 'Mature'
|
||||
and not (self.is_adult or self.getConfig("is_adult")) ):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
|
|
@ -243,7 +226,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
if( self.story.getMetadataRaw('rating') in ['Mature','Adult Only'] and
|
||||
if( self.story.getMetadata('rating') == 'Mature' and
|
||||
(self.is_adult or self.getConfig("is_adult")) ):
|
||||
addurl = "?bypass=1"
|
||||
else:
|
||||
|
|
@ -258,8 +241,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
"Error downloading Chapter: '{0}'! Missing required element!".format(url))
|
||||
|
||||
#Now, there are a lot of extranious tags within the story division.. so we will remove them.
|
||||
for tag in story.find_all('ul',{'class':'pager'}) + story.find_all(
|
||||
'div',{'class':'alert'}) + story.find_all('div', {'class':'btn-group'}):
|
||||
for tag in story.findAll('ul',{'class':'pager'}) + story.findAll(
|
||||
'div',{'class':'alert'}) + story.findAll('div', {'class':'btn-group'}):
|
||||
tag.extract()
|
||||
|
||||
return self.utf8FromSoup(url,story)
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating',stripHTML(get_meta_content(u'Рейтинг')))
|
||||
|
||||
## Need to login for any rating higher than General.
|
||||
if self.story.getMetadataRaw('rating') != 'General' and self.needToLoginCheck(data):
|
||||
if self.story.getMetadata('rating') != 'General' and self.needToLoginCheck(data):
|
||||
self.performLogin(url)
|
||||
# reload after login.
|
||||
data = self.get_request(url,usecache=False)
|
||||
|
|
|
|||
|
|
@ -44,8 +44,9 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self._setURL('https://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ahpfftc')
|
||||
|
|
@ -56,24 +57,24 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return [cls.getSiteDomain(),'archive.hpfanfictalk.com','fanfictalk.com']
|
||||
return [cls.getSiteDomain(),'archive.hpfanfictalk.com']
|
||||
|
||||
@classmethod
|
||||
def getConfigSections(cls):
|
||||
"Only needs to be overriden if has additional ini sections."
|
||||
return [cls.getConfigSection(),'archive.hpfanfictalk.com','fanfictalk.com']
|
||||
return [cls.getConfigSection(),'archive.hpfanfictalk.com']
|
||||
|
||||
@staticmethod # must be @stgetAcceptDomainsaticmethod, don't remove it.
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'archive.fanfictalk.com'
|
||||
return 'fanfictalk.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
return "https://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://("+r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()])+r")(/archive)?/viewstory\.php\?sid=\d+$"
|
||||
return r"https?://(archive\.hp)?"+re.escape(self.getSiteDomain())+r"(/archive)?/viewstory\.php\?sid=\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
|
@ -117,7 +118,7 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href'])
|
||||
self.add_chapter(chapter,'https://'+self.host+'/archive/'+chapter['href'])
|
||||
|
||||
# categories
|
||||
for a in soup.select("div#sort a"):
|
||||
|
|
@ -170,14 +171,14 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# Site allows stories to be in several series at once. FFF
|
||||
# isn't thrilled with that, we have series00, series01, etc.
|
||||
# Example:
|
||||
# https://archive.fanfictalk.com/viewstory.php?sid=483
|
||||
# https://fanfictalk.com/archive/viewstory.php?sid=483
|
||||
|
||||
if self.getConfig("collect_series"):
|
||||
seriesspan = soup.find('span',label='Series')
|
||||
for i, seriesa in enumerate(seriesspan.find_all('a', href=re.compile(r"viewseries\.php\?seriesid=\d+"))):
|
||||
# logger.debug(seriesa)
|
||||
series_name = stripHTML(seriesa)
|
||||
series_url = 'https://'+self.host+'/'+seriesa['href']
|
||||
series_url = 'https://'+self.host+'/archive/'+seriesa['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
|
|
@ -204,17 +205,9 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=3"
|
||||
else:
|
||||
addurl=""
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % (url+addurl))
|
||||
soup = self.make_soup(self.get_request(url+addurl))
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
|
|
|
|||
|
|
@ -93,14 +93,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
# logger.debug("post-url:%s"%url)
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def get_url_search(cls,url):
|
||||
regexp = super(getClass(), cls).get_url_search(url)
|
||||
regexp = re.sub(r"^(?P<keep>.*net/s/\d+/\d+/)(?P<urltitle>[^\$]*)?",
|
||||
r"\g<keep>(.*)",regexp)
|
||||
logger.debug(regexp)
|
||||
return regexp
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return self._get_site_url_pattern()
|
||||
|
||||
|
|
@ -110,31 +102,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
return re.sub(r"https?://(www|m)\.(?P<keep>fanfiction\.net/s/\d+/\d+/).*",
|
||||
r"https://www.\g<keep>",url)+self.urltitle
|
||||
|
||||
def get_request(self,url,usecache=True):
|
||||
## use super version if not set or isn't a chapter URL with a
|
||||
## title.
|
||||
if( not self.getConfig("try_shortened_title_urls") or
|
||||
not re.match(r"https?://www\.fanfiction\.net/s/\d+/\d+/(?P<title>[^/]+)$", url) ):
|
||||
return super(getClass(), self).get_request(url,usecache)
|
||||
|
||||
## kludgey way to attempt more than one URL variant by
|
||||
## removing title one letter at a time. Note that network and
|
||||
## open_pages_in_browser retries still happen first.
|
||||
titlelen = len(url.split('/')[-1])
|
||||
maxcut = min([4,titlelen])
|
||||
j = 0
|
||||
while j < maxcut: # should actually leave loop either by
|
||||
# return or exception raise.
|
||||
try:
|
||||
useurl = url
|
||||
if j: # j==0, full URL, then remove letters.
|
||||
useurl = url[:-j]
|
||||
return super(getClass(), self).get_request(useurl,usecache)
|
||||
except exceptions.HTTPErrorFFF as fffe:
|
||||
if j >= maxcut or 'Page not found or expired' not in unicode(fffe):
|
||||
raise
|
||||
j = j+1
|
||||
|
||||
def doExtractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
|
||||
# fetch the chapter. From that we will get almost all the
|
||||
|
|
@ -167,7 +134,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
## the first chapter. It generates another server request and
|
||||
## doesn't seem to be needed lately, so now default it to off.
|
||||
try:
|
||||
chapcount = len(soup.find('select', { 'name' : 'chapter' } ).find_all('option'))
|
||||
chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
|
||||
# get chapter part of url.
|
||||
except:
|
||||
chapcount = 1
|
||||
|
|
@ -212,7 +179,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
## For 1, use the second link.
|
||||
## For 2, fetch the crossover page and pull the two categories from there.
|
||||
pre_links = soup.find('div',{'id':'pre_story_links'})
|
||||
categories = pre_links.find_all('a',{'class':'xcontrast_txt'})
|
||||
categories = pre_links.findAll('a',{'class':'xcontrast_txt'})
|
||||
#print("xcontrast_txt a:%s"%categories)
|
||||
if len(categories) > 1:
|
||||
# Strangely, the ones with *two* links are the
|
||||
|
|
@ -251,7 +218,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
|
||||
# for b in grayspan.find_all('button'):
|
||||
# for b in grayspan.findAll('button'):
|
||||
# b.extract()
|
||||
metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
|
||||
#logger.debug("metatext:(%s)"%metatext)
|
||||
|
|
@ -290,7 +257,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
|
||||
# Published: <span data-xutime='1384358726'>8m ago</span>
|
||||
dates = soup.find_all('span',{'data-xutime':re.compile(r'^\d+$')})
|
||||
dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
|
||||
if len(dates) > 1 :
|
||||
# updated get set to the same as published upstream if not found.
|
||||
self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
|
||||
|
|
@ -395,7 +362,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
# no selector found, so it's a one-chapter story.
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
allOptions = select.find_all('option')
|
||||
allOptions = select.findAll('option')
|
||||
for o in allOptions:
|
||||
## title URL will be put back on chapter URL during
|
||||
## normalize_chapterurl() anyway, but also here for
|
||||
|
|
|
|||
|
|
@ -52,11 +52,11 @@ class FanfictionsFrSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('storyId', story_id)
|
||||
fandom_name = match.group('fandom')
|
||||
|
||||
self._setURL('https://%s/fanfictions/%s/%s/chapters.html' % (self.getSiteDomain(), fandom_name, story_id))
|
||||
self._setURL('https://www.%s/fanfictions/%s/%s/chapters.html' % (self.getSiteDomain(), fandom_name, story_id))
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'www.fanfictions.fr'
|
||||
return 'fanfictions.fr'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find('select').find_all('option'):
|
||||
for chapter in soup.find('select').findAll('option'):
|
||||
self.add_chapter(chapter,'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value'])
|
||||
|
||||
## title="Wörter" failed with max_zalgo:1
|
||||
|
|
@ -181,13 +181,13 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
|
||||
# #find metadata on the author's page
|
||||
# asoup = self.make_soup(self.get_request("https://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
|
||||
# tr=asoup.find_all('tr')
|
||||
# tr=asoup.findAll('tr')
|
||||
# for i in range(1,len(tr)):
|
||||
# a = tr[i].find('a')
|
||||
# if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
|
||||
# break
|
||||
|
||||
# td = tr[i].find_all('td')
|
||||
# td = tr[i].findAll('td')
|
||||
# self.story.addToList('category',stripHTML(td[2]))
|
||||
# self.story.setMetadata('rating', stripHTML(td[5]))
|
||||
# self.story.setMetadata('numWords', stripHTML(td[6]))
|
||||
|
|
@ -204,7 +204,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'storytext'})
|
||||
for a in div.find_all('script'):
|
||||
for a in div.findAll('script'):
|
||||
a.extract()
|
||||
|
||||
if None == div:
|
||||
|
|
|
|||
168
fanficfare/adapters/adapter_fastnovelsnet.py
Normal file
168
fanficfare/adapters/adapter_fastnovelsnet.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
####################################################################################################
|
||||
### Adapted by Rikkit on November 7. 2017
|
||||
###=================================================================================================
|
||||
### Tested with Calibre
|
||||
####################################################################################################
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
import re
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def getClass():
|
||||
''' Initializing the class '''
|
||||
return FastNovelNetAdapter
|
||||
|
||||
class FastNovelNetAdapter(BaseSiteAdapter):
|
||||
''' Adapter for FASTNOVEL.net '''
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.story.setMetadata('siteabbrev', 'fstnvl')
|
||||
|
||||
self.dateformat = '%d/%m/%Y'
|
||||
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
match = re.match(self.getSiteURLPattern(), url)
|
||||
if not match:
|
||||
raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
|
||||
|
||||
story_id = match.group('id')
|
||||
self.story.setMetadata('storyId', story_id)
|
||||
self._setURL('https://%s/%s/' % (self.getSiteDomain(), story_id))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'fastnovels.net'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return [cls.getSiteDomain(),'fastnovel.net']
|
||||
|
||||
@classmethod
|
||||
def getConfigSections(cls):
|
||||
"Only needs to be overriden if has additional ini sections."
|
||||
return [cls.getConfigSection(),'fastnovel.net']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://fastnovels.net/a-story-name-id"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
# https://fastnovels.net/ultimate-scheming-system-158/
|
||||
# also accept fastnovel.net
|
||||
return r"https?://fastnovels?\.net/(?P<id>[^/]+)"
|
||||
|
||||
## Normalized chapter URLs by changing old titlenum part to be
|
||||
## same as storyId.
|
||||
def normalize_chapterurl(self,url):
|
||||
# https://fastnovels.net/cultivation-chat-group8-29/chapter-25206.html
|
||||
return re.sub(r"\.net/.*(?P<keep>/chapter-\d+.html)",
|
||||
r".net/"+self.story.getMetadata('storyId')+r"\g<keep>",url)
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
logger.debug('URL: %s', self.url)
|
||||
|
||||
(data,rurl) = self.get_request_redirected(self.url)
|
||||
if rurl != self.url:
|
||||
match = re.match(self.getSiteURLPattern(), rurl)
|
||||
if not match:
|
||||
## shouldn't happen, but in case it does...
|
||||
raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
|
||||
|
||||
story_id = match.group('id')
|
||||
self.story.setMetadata('storyId', story_id)
|
||||
self._setURL('https://%s/%s/' % (self.getSiteDomain(), story_id))
|
||||
logger.debug("set to redirected url:%s"%self.url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
self.story.setMetadata('title', soup.find('h1').string)
|
||||
|
||||
for li in soup.select('.meta-data li'):
|
||||
label = li.select_one('label')
|
||||
if not label:
|
||||
continue
|
||||
|
||||
if label.string == "Author:":
|
||||
for a in li.select('a'):
|
||||
self.story.setMetadata('authorId', a["href"].split('/')[2])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+a["href"])
|
||||
self.story.setMetadata('author', a["title"])
|
||||
|
||||
if label.string == "Genre:":
|
||||
for a in li.select('a'):
|
||||
self.story.addToList('genre',a["title"])
|
||||
|
||||
if label.string == "Status:":
|
||||
if li.select_one('strong').string.strip() == "Completed":
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if label.string == "Last updated:":
|
||||
dateUpd = label.next_sibling.strip()
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(dateUpd), self.dateformat))
|
||||
|
||||
coverurl = soup.select_one('div.book-cover')["data-original"]
|
||||
if coverurl != "https://fastnovels.net/images/novel/default.jpg":
|
||||
self.setCoverImage(self.url, coverurl)
|
||||
|
||||
tags = soup.select_one('.tags')
|
||||
if tags:
|
||||
for a in tags.select("li.tag-item a"):
|
||||
self.story.addToList('tags', a["title"])
|
||||
# extract tags, because it inside description
|
||||
tags.extract()
|
||||
|
||||
self.setDescription(self.url, soup.select_one('div.content p'))
|
||||
|
||||
## number from end of storyId, taken this way in case it changes.
|
||||
# <input id="film_id" type="hidden" value="10667">
|
||||
film_id = soup.select_one('input#post_id')['value']
|
||||
ch_data = self.post_request('https://'+self.host+'/',
|
||||
parameters={'id': film_id,
|
||||
'list_postdata': '1'})
|
||||
# logger.debug(ch_data)
|
||||
ch_soup = self.make_soup(ch_data)
|
||||
# logger.debug(ch_soup)
|
||||
# for book in soup.select("#list-chapters .book"):
|
||||
# volume = book.select_one('.title a').string
|
||||
for a in ch_soup.select(".list-chapters a.chapter"):
|
||||
# title = volume + " " + stripHTML(a)
|
||||
title = stripHTML(a)
|
||||
self.add_chapter(title, 'https://' + self.host + a["href"])
|
||||
|
||||
def getChapterText(self, url):
|
||||
data = self.get_request(url)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
story = soup.select_one('#chapter-body')
|
||||
if not story:
|
||||
raise exceptions.FailedToDownload(
|
||||
"Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url, story)
|
||||
|
|
@ -16,15 +16,16 @@
|
|||
#
|
||||
|
||||
from __future__ import absolute_import,unicode_literals
|
||||
# import datetime
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
# from .. import translit
|
||||
from .. import translit
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions# as exceptions
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
|
|
@ -58,7 +59,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = u"%d %m %Y г., %H:%M"
|
||||
self.dateformat = "%d %m %Y %H:%M"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
|
|
@ -86,8 +87,9 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
if 'Войти используя аккаунт на сайте' in d:
|
||||
raise exceptions.FailedToLogin(url,params['login'])
|
||||
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
|
|
@ -107,7 +109,11 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
try:
|
||||
a = soup.find('section',{'class':'chapter-info'}).find('h1')
|
||||
except AttributeError:
|
||||
raise exceptions.FailedToDownload("Error collecting meta: %s! Missing required element!" % url)
|
||||
# Handle 404 in a nicer way when using nsapa proxy
|
||||
if re.search(r'404 — Страница не найдена', soup.find('title').text):
|
||||
raise exceptions.StoryDoesNotExist(url)
|
||||
else:
|
||||
raise exceptions.FailedToDownload("Error collecting meta: %s! Missing required element!" % url)
|
||||
# kill '+' marks if present.
|
||||
sup = a.find('sup')
|
||||
if sup:
|
||||
|
|
@ -139,8 +145,8 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
# Find the chapters:
|
||||
pubdate = None
|
||||
chapters = soup.find('ul', {'class' : 'list-of-fanfic-parts'})
|
||||
if chapters is not None:
|
||||
for chapdiv in chapters.find_all('li', {'class':'part'}):
|
||||
if chapters != None:
|
||||
for chapdiv in chapters.findAll('li', {'class':'part'}):
|
||||
chapter=chapdiv.find('a',href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+r"/\d+#part_content$"))
|
||||
churl='https://'+self.host+chapter['href']
|
||||
|
||||
|
|
@ -152,11 +158,12 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
self.add_chapter(chapter,churl,
|
||||
{'date':chapterdate.strftime(self.getConfig("datechapter_format",self.getConfig("datePublished_format",self.dateformat)))})
|
||||
|
||||
if pubdate is None and chapterdate:
|
||||
if pubdate == None and chapterdate:
|
||||
pubdate = chapterdate
|
||||
update = chapterdate
|
||||
else:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
self.story.setMetadata('numChapters',1)
|
||||
date_str = soup.find('div', {'class' : 'part-date'}).find('span', {'title': True})['title'].replace(u"\u202fг. в", "")
|
||||
for month_name, month_num in fullmon.items():
|
||||
date_str = date_str.replace(month_name, month_num)
|
||||
|
|
@ -168,18 +175,21 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('datePublished', pubdate)
|
||||
self.story.setMetadata('language','Russian')
|
||||
|
||||
dlinfo = soup.select_one('header.d-flex.flex-column.gap-12.word-break')
|
||||
## after site change, I don't see word count anywhere.
|
||||
# pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
|
||||
# pr='https://'+self.host+pr['href']
|
||||
# pr = self.make_soup(self.get_request(pr))
|
||||
# pr=pr.findAll('div', {'class' : 'part_text'})
|
||||
# i=0
|
||||
# for part in pr:
|
||||
# i=i+len(stripHTML(part).split(' '))
|
||||
# self.story.setMetadata('numWords', unicode(i))
|
||||
|
||||
series_label = dlinfo.select_one('div.description.word-break').find('strong', string='Серия:')
|
||||
logger.debug('Series: %s'%str(series_label))
|
||||
if series_label:
|
||||
series_div = series_label.find_next_sibling("div")
|
||||
# No accurate series number as for that, additional request needs to be made
|
||||
self.setSeries(stripHTML(series_div.a), 1)
|
||||
self.story.setMetadata('seriesUrl','https://' + self.getSiteDomain() + series_div.a.get('href'))
|
||||
|
||||
dlinfo = soup.find('div',{'class':'fanfic-main-info'})
|
||||
|
||||
i=0
|
||||
fandoms = dlinfo.select_one('div:not([class])').find_all('a', href=re.compile(r'/fanfiction/\w+'))
|
||||
fandoms = dlinfo.find('div', {'class' : 'mb-10'}).findAll('a', href=re.compile(r'/fanfiction/\w+'))
|
||||
for fandom in fandoms:
|
||||
self.story.addToList('category',fandom.string)
|
||||
i=i+1
|
||||
|
|
@ -188,16 +198,13 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
tags = soup.find('div',{'class':'tags'})
|
||||
if tags:
|
||||
for genre in tags.find_all('a',href=re.compile(r'/tags/')):
|
||||
for genre in tags.findAll('a',href=re.compile(r'/tags/')):
|
||||
self.story.addToList('genre',stripHTML(genre))
|
||||
|
||||
logger.debug("category: (%s)"%self.story.getMetadata('category'))
|
||||
logger.debug("genre: (%s)"%self.story.getMetadata('genre'))
|
||||
|
||||
ratingdt = dlinfo.find('div',{'class':re.compile(r'badge-rating-.*')})
|
||||
self.story.setMetadata('rating', stripHTML(ratingdt.find('span')))
|
||||
|
||||
# meta=table.find_all('a', href=re.compile(r'/ratings/'))
|
||||
# meta=table.findAll('a', href=re.compile(r'/ratings/'))
|
||||
# i=0
|
||||
# for m in meta:
|
||||
# if i == 0:
|
||||
|
|
@ -215,11 +222,6 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
try:
|
||||
self.story.setMetadata('universe', stripHTML(dlinfo.find('a', href=re.compile('/fandom_universe/'))))
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
paircharsdt = soup.find('strong',string='Пэйринг и персонажи:')
|
||||
# site keeps both ships and indiv chars in /pairings/ links.
|
||||
if paircharsdt:
|
||||
|
|
@ -233,16 +235,17 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
self.story.addToList('characters',stripHTML(paira))
|
||||
|
||||
summary=soup.find('div', itemprop='description')
|
||||
if summary:
|
||||
# To get rid of an empty div on the title page.
|
||||
if summary.get_text():
|
||||
# Fix for the text not displaying properly
|
||||
summary['class'].append('part_text')
|
||||
self.setDescription(url,summary)
|
||||
#self.story.setMetadata('description', summary.text)
|
||||
|
||||
stats = soup.find('div', {'class':'hat-actions-container'})
|
||||
stats = soup.find('div', {'class':'mb-15 text-center'})
|
||||
targetdata = stats.find_all('span', {'class' : 'main-info'})
|
||||
for data in targetdata:
|
||||
svg_class = data.find('svg')['class'][1] if data.find('svg') else None
|
||||
svg_class = data.find('svg')['class'][0] if data.find('svg') else None
|
||||
value = int(stripHTML(data)) if stripHTML(data).isdigit() else 0
|
||||
|
||||
if svg_class == 'ic_thumbs-up' and value > 0:
|
||||
|
|
@ -255,23 +258,15 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numCollections', value)
|
||||
logger.debug("numCollections: (%s)"%self.story.getMetadata('numCollections'))
|
||||
|
||||
# Grab the amount of pages and words
|
||||
# Grab the amount of pages
|
||||
targetpages = soup.find('strong',string='Размер:').find_next('div')
|
||||
if targetpages:
|
||||
targetpages_text = re.sub(r"(?<!\,)\s| ", "", targetpages.text, flags=re.UNICODE | re.MULTILINE)
|
||||
|
||||
pages_raw = re.search(r'(\d+)(?:страницы|страниц)', targetpages_text, re.UNICODE)
|
||||
pages = int(pages_raw.group(1))
|
||||
pages_raw = re.search(r'(.+)\s+(?:страницы|страниц)', targetpages.text, re.UNICODE)
|
||||
pages = int(re.sub(r'[^\d]', '', pages_raw.group(1)))
|
||||
if pages > 0:
|
||||
self.story.setMetadata('pages', pages)
|
||||
logger.debug("pages: (%s)"%self.story.getMetadata('pages'))
|
||||
|
||||
numWords_raw = re.search(r"(\d+)(?:слова|слов)", targetpages_text, re.UNICODE)
|
||||
numWords = int(numWords_raw.group(1))
|
||||
if numWords > 0:
|
||||
self.story.setMetadata('numWords', numWords)
|
||||
logger.debug("numWords: (%s)"%self.story.getMetadata('numWords'))
|
||||
|
||||
# Grab FBN Category
|
||||
class_tag = soup.select_one('div[class^="badge-with-icon direction"]').find('span', {'class' : 'badge-text'}).text
|
||||
if class_tag:
|
||||
|
|
@ -280,7 +275,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
# Find dedication.
|
||||
ded = soup.find('div', {'class' : 'js-public-beta-dedication'})
|
||||
if ded:
|
||||
if ded != None:
|
||||
ded['class'].append('part_text')
|
||||
self.story.setMetadata('dedication',ded)
|
||||
|
||||
|
|
@ -290,11 +285,49 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
comm['class'].append('part_text')
|
||||
self.story.setMetadata('authorcomment',comm)
|
||||
|
||||
follows = stats.find('fanfic-follow-button')[':follow-count']
|
||||
# When using nsapa proxy the required elements are not returned.
|
||||
try:
|
||||
follows = stats.find('fanfic-follow-button')[':follow-count']
|
||||
except TypeError:
|
||||
follows = stripHTML(stats.find('button', {'class': 'btn btn-with-description btn-primary jsVueComponent', 'type': 'button'}).span)
|
||||
if int(follows) > 0:
|
||||
self.story.setMetadata('follows', int(follows))
|
||||
logger.debug("follows: (%s)"%self.story.getMetadata('follows'))
|
||||
|
||||
if "collections" in self.getConfigList('extra_valid_entries') and self.story.getMetadata('numCollections') != '' and int(re.sub(',', '', self.story.getMetadata('numCollections'))) > 0:
|
||||
# Because the number of collections is >0, we are assuming that as there has to be an element that has a link to collections
|
||||
collection = soup.select_one("div[class='mb-15']")
|
||||
try:
|
||||
collection = collection.find('fanfic-collections-link')['url']
|
||||
except TypeError:
|
||||
collection = collection.find('a')['href']
|
||||
collUrl = 'https://' + self.getSiteDomain() + collection
|
||||
|
||||
# Collect the names of the collections
|
||||
soupColl = self.make_soup(self.get_request(collUrl))
|
||||
# Process the first page.
|
||||
targetcoll = soupColl.find_all('div', {'class' : 'collection-thumb-info'})
|
||||
for coll in targetcoll:
|
||||
# Have to include entire a tag, if multiple ones have the same name only one will be included.
|
||||
o = coll.find('a', href=re.compile(r'/collections/'))
|
||||
o['href'] = 'https://' + self.getSiteDomain()+o['href']
|
||||
self.story.addToList('collections', str(o))
|
||||
|
||||
# See if there are more pages and get the number
|
||||
if soupColl.find('div', {'class' : 'paging-description'}):
|
||||
collpg = soupColl.find('div', {'class' : 'paging-description'}).select_one('div.paging-description b:last-child').text
|
||||
# Start requesting the remaining pages, omitting the first one.
|
||||
for c in range(int(collpg), 1, -1):
|
||||
soupColl = self.make_soup(self.get_request(collUrl + '?p=' + str(c)))
|
||||
targetcoll = soupColl.find_all('div', {'class' : 'collection-thumb-info'})
|
||||
for coll in targetcoll:
|
||||
o = coll.find('a', href=re.compile(r'/collections/'))
|
||||
o['href'] = 'https://' + self.getSiteDomain() + o['href']
|
||||
self.story.addToList('collections', str(o))
|
||||
|
||||
#logger.debug("collections: (%s)"%self.story.getMetadata('collections'))
|
||||
logger.debug("Collections: (%s/%s)" % (len(self.story.getMetadata('collections').split('</a>, ')), self.story.getMetadata('numCollections')))
|
||||
|
||||
# Grab the amount of awards
|
||||
numAwards = 0
|
||||
try:
|
||||
|
|
@ -303,9 +336,14 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
numAwards = int(len(award_list))
|
||||
# Grab the awards, but if multiple awards have the same name, only one will be kept; only an issue with hundreds of them.
|
||||
self.story.extendList('awards', [str(award['user_text']) for award in award_list])
|
||||
#logger.debug("awards (%s)"%self.story.getMetadata('awards'))
|
||||
#logger.debug("awards (%s)"%self.story.getMetadata('awards'))
|
||||
except (TypeError, KeyError):
|
||||
logger.debug("Could not grab the awards")
|
||||
awards = soup.find('div', {'class':'fanfic-reward-container rounded-block'}).find('section', {'class':'mb-15 jsVueComponent'})
|
||||
if awards is not None:
|
||||
numAwards = int(len(awards.div.find_all('div', class_='fanfic_reward_list', recursive=False)))
|
||||
naward = awards.div.find('span', {'class':'js-span-link'})
|
||||
if naward is not None:
|
||||
numAwards = numAwards + int(re.sub(r'[^\d]', '', naward.text))
|
||||
|
||||
if numAwards > 0:
|
||||
self.story.setMetadata('numAwards', numAwards)
|
||||
|
|
@ -313,19 +351,16 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
if get_cover:
|
||||
cover = soup.find('fanfic-cover', {'class':"jsVueComponent"})
|
||||
if cover is not None:
|
||||
if cover is None:
|
||||
# When using nsapa proxy the element is replaced by different one.
|
||||
cover = soup.find('picture', {'class':"fanfic-hat-cover-picture"})
|
||||
if cover is not None:
|
||||
cover = re.sub('/fanfic-covers/(?:m_|d_)', '/fanfic-covers/', cover.img['src'])
|
||||
logger.debug("Cover url (%s)"%cover)
|
||||
self.setCoverImage(url,cover)
|
||||
else:
|
||||
self.setCoverImage(url,cover['src-original'])
|
||||
|
||||
def replace_formatting(self,tag):
|
||||
tname = tag.name
|
||||
## operating on plain text because BS4 is hard to work on
|
||||
## text with.
|
||||
## stripHTML() discards whitespace around other tags, like <i>
|
||||
txt = tag.get_text()
|
||||
txt = txt.replace("\n","<br/>")
|
||||
soup = self.make_soup("<"+tname+">"+txt+"</"+tname+">")
|
||||
return soup.find(tname)
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
|
|
@ -334,60 +369,48 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
chapter = soup.find('div', {'id' : 'content'})
|
||||
if chapter is None: ## still needed?
|
||||
if chapter == None: ## still needed?
|
||||
chapter = soup.find('div', {'class' : 'public_beta_disabled'})
|
||||
|
||||
if chapter is None:
|
||||
if None == chapter:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
## ficbook uses weird CSS white-space: pre-wrap; for
|
||||
## paragraphing. Doesn't work with txt output
|
||||
if 'part_text' in chapter['class'] and self.getConfig('replace_text_formatting'):
|
||||
## copy classes, except part_text
|
||||
divclasses = chapter['class']
|
||||
divclasses.remove('part_text')
|
||||
chapter = self.replace_formatting(chapter)
|
||||
chapter['class'] = divclasses
|
||||
# Remove ads that show up when using NSAPA proxy.
|
||||
if self.getConfig("use_nsapa_proxy",True):
|
||||
for ads in chapter.find_all('div', {'class' : 'ads-in-text'}):
|
||||
ads.extract()
|
||||
|
||||
exclude_notes=self.getConfigList('exclude_notes')
|
||||
if 'headnotes' not in exclude_notes:
|
||||
# Find the headnote
|
||||
head_note = soup.select_one("div.part-comment-top div.js-public-beta-comment-before")
|
||||
head_note = soup.find('div', {'class': 'part-comment-top'})
|
||||
if head_note:
|
||||
head_notes_content = head_note.find('div', {'class': 'js-public-beta-comment-before'}).get_text(strip=True)
|
||||
# Create the structure for the headnote
|
||||
head_notes_div_tag = soup.new_tag('div', attrs={'class': 'fff_chapter_notes fff_head_notes'})
|
||||
head_b_tag = soup.new_tag('b')
|
||||
head_b_tag.string = 'Примечания:'
|
||||
if 'text-preline' in head_note['class'] and self.getConfig('replace_text_formatting'):
|
||||
head_blockquote_tag = self.replace_formatting(head_note)
|
||||
head_blockquote_tag.name = 'blockquote'
|
||||
else:
|
||||
head_blockquote_tag = soup.new_tag('blockquote')
|
||||
head_blockquote_tag.string = stripHTML(head_note)
|
||||
head_blockquote_tag = soup.new_tag('blockquote')
|
||||
head_blockquote_tag.string = head_notes_content
|
||||
head_notes_div_tag.append(head_b_tag)
|
||||
head_notes_div_tag.append(head_blockquote_tag)
|
||||
# Prepend the headnotes to the chapter, <hr> to mimic the site
|
||||
# Prepend the headnotes to the chapter
|
||||
chapter.insert(0, head_notes_div_tag)
|
||||
chapter.insert(1, soup.new_tag('hr'))
|
||||
|
||||
if 'footnotes' not in exclude_notes:
|
||||
# Find the endnote
|
||||
end_note = soup.select_one("div.part-comment-bottom div.js-public-beta-comment-after")
|
||||
end_note = soup.find('div', {'class': 'part-comment-bottom'})
|
||||
if end_note:
|
||||
end_notes_content = end_note.find('div', {'class': 'js-public-beta-comment-after'}).get_text(strip=True)
|
||||
# Create the structure for the footnote
|
||||
end_notes_div_tag = soup.new_tag('div', attrs={'class': 'fff_chapter_notes fff_foot_notes'})
|
||||
end_b_tag = soup.new_tag('b')
|
||||
end_b_tag.string = 'Примечания:'
|
||||
if 'text-preline' in end_note['class'] and self.getConfig('replace_text_formatting'):
|
||||
end_blockquote_tag = self.replace_formatting(end_note)
|
||||
end_blockquote_tag.name = 'blockquote'
|
||||
else:
|
||||
end_blockquote_tag = soup.new_tag('blockquote')
|
||||
end_blockquote_tag.string = stripHTML(end_note)
|
||||
end_blockquote_tag = soup.new_tag('blockquote')
|
||||
end_blockquote_tag.string = end_notes_content
|
||||
end_notes_div_tag.append(end_b_tag)
|
||||
end_notes_div_tag.append(end_blockquote_tag)
|
||||
# Append the endnotes to the chapter, <hr> to mimic the site
|
||||
chapter.append(soup.new_tag('hr'))
|
||||
# Append the endnotes to the chapter
|
||||
chapter.append(end_notes_div_tag)
|
||||
|
||||
return self.utf8FromSoup(url,chapter)
|
||||
|
|
|
|||
|
|
@ -201,10 +201,10 @@ class FictionAlleyArchiveOrgSiteAdapter(BaseSiteAdapter):
|
|||
# epubutils.py
|
||||
# Yes, this still applies to fictionalley-archive.
|
||||
|
||||
for tag in chaptext.find_all('head') + chaptext.find_all('meta') + chaptext.find_all('script'):
|
||||
for tag in chaptext.findAll('head') + chaptext.findAll('meta') + chaptext.findAll('script'):
|
||||
tag.extract()
|
||||
|
||||
for tag in chaptext.find_all('body') + chaptext.find_all('html'):
|
||||
for tag in chaptext.findAll('body') + chaptext.findAll('html'):
|
||||
tag.name = 'div'
|
||||
|
||||
if self.getConfig('include_author_notes'):
|
||||
|
|
|
|||
|
|
@ -55,8 +55,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
self.story_id = self.parsedUrl.path.split('/')[3]
|
||||
self.story.setMetadata('storyId', self.story_id)
|
||||
|
||||
self.chapter_id_to_api = {}
|
||||
|
||||
# normalize URL. omits title in the url
|
||||
self._setURL("https://fiction.live/stories//{s_id}".format(s_id = self.story_id));
|
||||
|
||||
|
|
@ -173,7 +171,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
tags = data['ta'] if 'ta' in data else []
|
||||
|
||||
if (self.story.getMetadataRaw('rating') in {"nsfw", "adult"} or 'smut' in tags) and \
|
||||
if (self.story.getMetadata('rating') in {"nsfw", "adult"} or 'smut' in tags) and \
|
||||
not (self.is_adult or self.getConfig("is_adult")):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
|
|
@ -241,17 +239,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
a, b = itertools.tee(iterable, 2)
|
||||
next(b, None)
|
||||
return list(zip(a, b))
|
||||
|
||||
def map_chap_ids_to_api(chapter_ids, route_ids, times):
|
||||
for index, bounds in enumerate(times):
|
||||
start, end = bounds
|
||||
end -= 1
|
||||
chapter_url = chunkrange_url.format(s_id = data['_id'], start = start, end = end)
|
||||
self.chapter_id_to_api[chapter_ids[index]] = chapter_url
|
||||
|
||||
for route_id in route_ids:
|
||||
chapter_url = route_chunkrange_url.format(c_id = route_id)
|
||||
self.chapter_id_to_api[route_id] = chapter_url
|
||||
|
||||
## first thing to do is seperate out the appendices
|
||||
appendices, maintext, routes = [], [], []
|
||||
|
|
@ -273,25 +260,22 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
## main-text chapter extraction processing. *should* now handle all the edge cases.
|
||||
## relies on fanficfare ignoring empty chapters!
|
||||
|
||||
titles = ["Home"] + [c['title'] for c in maintext]
|
||||
chapter_ids = ['home'] + [c['id'] for c in maintext]
|
||||
times = [data['ct']] + [c['ct'] for c in maintext] + [self.most_recent_chunk + 2] # need to be 1 over, and add_url etc does -1
|
||||
times = pair(times)
|
||||
titles = [c['title'] for c in maintext]
|
||||
titles = ["Home"] + titles
|
||||
|
||||
if self.getConfig('include_appendices', True): # Add appendices after main text if desired
|
||||
titles = titles + ["Appendix: " + a['title'][9:] for a in appendices]
|
||||
chapter_ids = chapter_ids + [a['id'] for a in appendices]
|
||||
times = times + [(a['ct'], a['ct'] + 2) for a in appendices]
|
||||
|
||||
route_ids = [r['id'] for r in routes]
|
||||
|
||||
map_chap_ids_to_api(chapter_ids, route_ids, times) # Map chapter ids to API URLs for use when comparing the two
|
||||
times = [c['ct'] for c in maintext]
|
||||
times = [data['ct']] + times + [self.most_recent_chunk + 2] # need to be 1 over, and add_url etc does -1
|
||||
|
||||
# doesn't actually run without the call to list.
|
||||
list(map(add_chapter_url, titles, times))
|
||||
list(map(add_chapter_url, titles, pair(times)))
|
||||
|
||||
for a in appendices: # add appendices afterwards
|
||||
chapter_start = a['ct']
|
||||
chapter_title = "Appendix: " + a['title'][9:] # 'Appendix: ' rather than '#special' at beginning of name
|
||||
add_chapter_url(chapter_title, (chapter_start, chapter_start + 2)) # 1 msec range = this one chunk only
|
||||
|
||||
for r in routes: # add route at the end, after appendices
|
||||
route_id = r['id'] # to get route chapter content, the route id is needed, not the timestamp
|
||||
route_id = r['id'] # to get route chapter content, the route id is needed, not the timestamp
|
||||
chapter_title = "Route: " + r['title'] # 'Route: ' at beginning of name, since it's a multiroute chapter
|
||||
add_route_chapter_url(chapter_title, route_id)
|
||||
|
||||
|
|
@ -434,7 +418,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
# so let's just ignore non-int values here
|
||||
if not isinstance(v, int):
|
||||
continue
|
||||
if 0 <= v < len(choices):
|
||||
if 0 <= v <= len(choices):
|
||||
output[v] += 1
|
||||
return output
|
||||
|
||||
|
|
@ -518,10 +502,8 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
# now matches the site and does *not* include dicerolls as posts!
|
||||
num_votes = str(len(posts)) + " posts" if len(posts) != 0 else "be the first to post."
|
||||
|
||||
posts_title = chunk['b'] if 'b' in chunk else "Reader Posts"
|
||||
|
||||
output = ""
|
||||
output += u"<h4><span>" + posts_title + " — <small> Posting " + closed
|
||||
output += u"<h4><span>Reader Posts — <small> Posting " + closed
|
||||
output += u" — " + num_votes + "</small></span></h4>\n"
|
||||
|
||||
## so. a voter can roll with their post. these rolls are in a seperate dict, but have the **same uid**.
|
||||
|
|
@ -547,35 +529,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
return output
|
||||
|
||||
def normalize_chapterurl(self, url):
|
||||
if url.startswith(r'https://fiction.live/api/anonkun/chapters'):
|
||||
return url
|
||||
|
||||
pattern = None
|
||||
|
||||
if url.startswith(r'https://fiction.live/api/anonkun/route'):
|
||||
pattern = r"https?://(?:beta\.)?fiction\.live/[^/]*/[^/]*/[a-zA-Z0-9]+/routes/([a-zA-Z0-9]+)"
|
||||
elif url.startswith(r'https://fiction.live/'):
|
||||
pattern = r"https?://(?:beta\.)?fiction\.live/[^/]*/[^/]*/[a-zA-Z0-9]+/[^/]*(/[a-zA-Z0-9]+|home)"
|
||||
# regex101 rocks
|
||||
|
||||
if not pattern:
|
||||
return url
|
||||
|
||||
match = re.match(pattern, url)
|
||||
if not match:
|
||||
return url
|
||||
|
||||
chapter_id = match.group(1)
|
||||
|
||||
if chapter_id.startswith('/'):
|
||||
chapter_id = chapter_id[1:]
|
||||
|
||||
if chapter_id and chapter_id in self.chapter_id_to_api:
|
||||
return self.chapter_id_to_api[chapter_id]
|
||||
|
||||
return url
|
||||
|
||||
def format_unknown(self, chunk):
|
||||
raise NotImplementedError("Unknown chunk type ({}) in fiction.live story.".format(chunk))
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
SITE_ABBREVIATION = 'fmt'
|
||||
SITE_DOMAIN = 'fictionmania.tv'
|
||||
|
||||
BASE_URL = 'https://' + SITE_DOMAIN + '/stories/'
|
||||
BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
|
||||
READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
|
||||
DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
|
||||
|
||||
|
|
@ -40,6 +40,10 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
|
||||
self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
|
||||
|
||||
# Always single chapters, probably should use the Anthology feature to
|
||||
# merge chapters of a story
|
||||
self.story.setMetadata('numChapters', 1)
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return FictionManiaTVAdapter.SITE_DOMAIN
|
||||
|
|
@ -49,7 +53,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r'https?' + re.escape(self.BASE_URL[len('https'):]) + r'(readtextstory|readhtmlstory|readxstory|details)\.html\?storyID=\d+$'
|
||||
return r'https?' + re.escape(self.BASE_URL[len('http'):]) + r'(readtextstory|readhtmlstory|readxstory|details)\.html\?storyID=\d+$'
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
|
||||
|
|
@ -163,30 +167,14 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
# <div style="margin-left:10ex;margin-right:10ex">
|
||||
## fetching SWI version now instead of text.
|
||||
htmlurl = url.replace('readtextstory','readhtmlstory')
|
||||
## Used to find by style, but it's inconsistent now. we've seen:
|
||||
## margin-left:10ex;margin-right:10ex
|
||||
## margin-right: 5%; margin-left: 5%
|
||||
## margin-left:5%; margin-right:5%
|
||||
## margin-left:5%; margin-right:5%; background: white
|
||||
## And there's some without a <div> tag (or an unclosed div)
|
||||
## Only the comments appear to be consistent.
|
||||
beginmarker='<!--Read or display the file-->'
|
||||
endmarker='''<hr size=1 noshade>
|
||||
<!--review add read, top and bottom-->
|
||||
'''
|
||||
data = self.get_request(htmlurl)
|
||||
try:
|
||||
## if both markers are found, assume whatever is in between
|
||||
## is the chapter text.
|
||||
soup = self.make_soup(data[data.index(beginmarker):data.index(endmarker)])
|
||||
return self.utf8FromSoup(htmlurl,soup)
|
||||
except Exception as e:
|
||||
# logger.debug(e)
|
||||
# logger.debug(soup)
|
||||
soup = self.make_soup(self.get_request(htmlurl))
|
||||
div = soup.find('div',style="margin-left:10ex;margin-right:10ex")
|
||||
if div:
|
||||
return self.utf8FromSoup(htmlurl,div)
|
||||
else:
|
||||
logger.debug("Story With Images(SWI) not found, falling back to HTML.")
|
||||
|
||||
## fetching html version now instead of text.
|
||||
## Note that html and SWI pages are *not* formatted the same.
|
||||
soup = self.make_soup(self.get_request(url.replace('readtextstory','readxstory')))
|
||||
# logger.debug(soup)
|
||||
|
||||
|
|
|
|||
|
|
@ -66,8 +66,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
params['username']))
|
||||
d = self.post_request(loginUrl,params,usecache=False)
|
||||
|
||||
if "Login attempt failed..." in d or \
|
||||
'<div id="error">Please enter your username and password.</div>' in d:
|
||||
if "Login attempt failed..." in d:
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['username']))
|
||||
raise exceptions.FailedToLogin(url,params['username'])
|
||||
|
|
@ -115,7 +114,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
|
||||
self.story.setMetadata('title', stripHTML(titleh4.a))
|
||||
|
||||
if 'Deleted story' in self.story.getMetadataRaw('title'):
|
||||
if 'Deleted story' in self.story.getMetadata('title'):
|
||||
raise exceptions.StoryDoesNotExist("This story was deleted. %s"%self.url)
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
|
|
@ -130,14 +129,14 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
#self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
|
||||
|
||||
# most of the meta data is here:
|
||||
metap = storydiv.find("div",{"class":"meta"})
|
||||
metap = storydiv.find("p",{"class":"meta"})
|
||||
self.story.addToList('category',metap.find("a",href=re.compile(r"^/category/\d+")).string)
|
||||
|
||||
# warnings
|
||||
# <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
|
||||
spanreq = metap.find("span",{"class":"story-warnings"})
|
||||
if spanreq: # can be no warnings.
|
||||
for a in spanreq.find_all("a"):
|
||||
for a in spanreq.findAll("a"):
|
||||
self.story.addToList('warnings',a['title'])
|
||||
|
||||
## perhaps not the most efficient way to parse this, using
|
||||
|
|
@ -187,7 +186,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
# no list found, so it's a one-chapter story.
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
chapterlistlis = storylistul.find_all('li')
|
||||
chapterlistlis = storylistul.findAll('li')
|
||||
for chapterli in chapterlistlis:
|
||||
if "blocked" in chapterli['class']:
|
||||
# paranoia check. We should already be logged in by now.
|
||||
|
|
|
|||
|
|
@ -99,17 +99,6 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
params['username']))
|
||||
raise exceptions.FailedToLogin(url,params['username'])
|
||||
|
||||
def make_soup(self,data):
|
||||
soup = super(FimFictionNetSiteAdapter, self).make_soup(data)
|
||||
for img in soup.select('img.lazy-img, img.user_image'):
|
||||
## FimF has started a 'camo' mechanism for images that
|
||||
## gets block by CF. attr data-source is original source.
|
||||
if img.has_attr('data-source'):
|
||||
img['src'] = img['data-source']
|
||||
elif img.has_attr('data-src'):
|
||||
img['src'] = img['data-src']
|
||||
return soup
|
||||
|
||||
def doExtractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
|
|
@ -117,8 +106,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
## Only needed with password protected stories, which you have
|
||||
## to have logged into in the website using this account.
|
||||
if self.getConfig("always_login"):
|
||||
self.performLogin(self.url)
|
||||
self.performLogin(self.url)
|
||||
|
||||
##---------------------------------------------------------------------------------------------------
|
||||
## Get the story's title page. Check if it exists.
|
||||
|
|
@ -151,8 +139,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata("authorId", author['href'].split('/')[2])
|
||||
self.story.setMetadata("authorUrl", "https://%s/user/%s/%s" % (self.getSiteDomain(),
|
||||
self.story.getMetadata('authorId'),
|
||||
# meta entry author can be changed by the user.
|
||||
stripHTML(author)))
|
||||
self.story.getMetadata('author')))
|
||||
|
||||
#Rating text is replaced with full words for historical compatibility after the site changed
|
||||
#on 2014-10-27
|
||||
|
|
@ -180,13 +167,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Cover image
|
||||
if get_cover:
|
||||
storyImage = soup.select_one('div.story_container__story_image img')
|
||||
storyImage = storyContentBox.find('img', {'class':'lazy-img'})
|
||||
if storyImage:
|
||||
coverurl = storyImage['data-fullsize']
|
||||
# try setting from data-fullsize, if fails, try using data-src
|
||||
cover_set = self.setCoverImage(self.url,coverurl)[0]
|
||||
if not cover_set or cover_set.startswith("failedtoload"):
|
||||
coverurl = storyImage['src']
|
||||
if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
|
||||
coverurl = storyImage['data-src']
|
||||
self.setCoverImage(self.url,coverurl)
|
||||
|
||||
coverSource = storyImage.parent.find('a', {'class':'source'})
|
||||
|
|
@ -298,26 +284,16 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
descriptionMeta = soup.find('meta', {'property':'og:description'})
|
||||
self.story.setMetadata("short_description", stripHTML(descriptionMeta['content']))
|
||||
|
||||
# groups.
|
||||
# If there are more than X groups, there's a 'Show all' button
|
||||
# that calls for a JSON containing HTML with the full list.
|
||||
# But it doesn't work reliably with FlareSolverr.
|
||||
groupList = None
|
||||
#groups
|
||||
groupButton = soup.find('button', {'data-click':'showAll'})
|
||||
if groupButton != None and groupButton.find('i', {'class':'fa-search-plus'}):
|
||||
try:
|
||||
groupResponse = self.get_request("https://www.fimfiction.net/ajax/stories/%s/groups" % (self.story.getMetadata("storyId")))
|
||||
groupData = json.loads(groupResponse)
|
||||
groupList = self.make_soup(groupData["content"])
|
||||
except Exception as e:
|
||||
logger.warning("Collecting 'groups' (AKA 'Featured In') from JSON failed:%s"%e)
|
||||
logger.warning("Only 'groups' initially shown on the page will be collected.")
|
||||
logger.warning("This is a known issue with JSON and FlareSolverr. See #1122")
|
||||
|
||||
if not groupList:
|
||||
groupResponse = self.get_request("https://www.fimfiction.net/ajax/stories/%s/groups" % (self.story.getMetadata("storyId")))
|
||||
groupData = json.loads(groupResponse)
|
||||
groupList = self.make_soup(groupData["content"])
|
||||
else:
|
||||
groupList = soup.find('ul', {'id':'story-groups-list'})
|
||||
|
||||
if groupList:
|
||||
if not (groupList == None):
|
||||
for groupContent in groupList.find_all('a'):
|
||||
self.story.addToList("groupsUrl", 'https://'+self.host+groupContent["href"])
|
||||
groupName = groupContent.find('span', {"class":"group-name"})
|
||||
|
|
@ -408,33 +384,3 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
# data = self.get_request(url)
|
||||
if self.getConfig("is_adult"):
|
||||
self.set_adult_cookie()
|
||||
|
||||
def get_urls_from_page(self,url,normalize):
|
||||
iterate = self.getConfig('scrape_bookshelf', default=False)
|
||||
if not re.search(r'fimfiction\.net/bookshelf/(?P<listid>.+?)/',url) or iterate == 'legacy':
|
||||
return super().get_urls_from_page(url,normalize)
|
||||
|
||||
self.before_get_urls_from_page(url,normalize)
|
||||
|
||||
final_urls = list()
|
||||
while True:
|
||||
data = self.get_request(url,usecache=True)
|
||||
soup = self.make_soup(data)
|
||||
paginator = soup.select_one('div.paginator-container > div.page_list > ul').find_all('li')
|
||||
logger.debug("Paginator: " + str(len(paginator)))
|
||||
stories_container = soup.select_one('div.content > div.two-columns > div.left').find_all('article', recursive=False)
|
||||
x = 0
|
||||
logger.debug("Container "+str(len(stories_container)))
|
||||
for story_raw in stories_container:
|
||||
x += 1
|
||||
story_url = story_raw.select_one('div.story_content_box > header.title > div > a.story_name').get('href')
|
||||
url_story = ('https://' + self.getSiteDomain() + story_url)
|
||||
#logger.debug(url_story)
|
||||
final_urls.append(url_story)
|
||||
logger.debug("Discovered %s new stories."%str(x))
|
||||
|
||||
next_button = paginator[-1].select_one('a')
|
||||
logger.debug("Next button: " + next_button.get_text())
|
||||
if next_button.get_text() or not iterate:
|
||||
return {'urllist': final_urls}
|
||||
url = ('https://' + self.getSiteDomain() + next_button.get('href'))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2024 FanFicFare team
|
||||
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -19,20 +19,22 @@ from __future__ import absolute_import
|
|||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .base_otw_adapter import BaseOTWAdapter
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .adapter_storiesonlinenet import StoriesOnlineNetAdapter
|
||||
|
||||
def getClass():
|
||||
return SuperloveAdapter
|
||||
return FineStoriesComAdapter
|
||||
|
||||
class SuperloveAdapter(BaseOTWAdapter):
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class FineStoriesComAdapter(StoriesOnlineNetAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseOTWAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','sluv')
|
||||
@classmethod
|
||||
def getSiteAbbrev(cls):
|
||||
return 'fnst'
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'superlove.sayitditto.net'
|
||||
return 'finestories.com'
|
||||
|
|
@ -93,9 +93,6 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
a = soup.find('a', href=re.compile(r"profileshow.aspx\?u="))
|
||||
self.story.setMetadata('authorId', a['href'].split('=')[1])
|
||||
if not self.story.getMetadata('authorId'):
|
||||
logger.warning("Site authorUrl missing authorId, using SiteMissingAuthorId")
|
||||
self.story.setMetadata('authorId', 'SiteMissingAuthorId')
|
||||
self.story.setMetadata('authorUrl', 'http://' +
|
||||
self.host + '/' + a['href'])
|
||||
self.story.setMetadata('author', a.string)
|
||||
|
|
@ -105,6 +102,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
|
|||
# to download them one at a time yourself. I'm also setting the status to
|
||||
# complete
|
||||
self.add_chapter(self.story.getMetadata('title'), self.url)
|
||||
self.story.setMetadata('numChapters', 1)
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
|
||||
## some stories do not have a summary listed, so I'm setting it here.
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -178,7 +178,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -199,22 +199,22 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -238,7 +238,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url,usecache=False))
|
||||
|
||||
# removing all of the scripts
|
||||
for tag in soup.find_all('script'):
|
||||
for tag in soup.findAll('script'):
|
||||
tag.extract()
|
||||
|
||||
|
||||
|
|
@ -200,11 +200,3 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload("Error downloading Chapter: %s No text block found -- non-story URL?" % url)
|
||||
|
||||
return self.utf8FromSoup(url, story)
|
||||
|
||||
def before_get_urls_from_page(self,url,normalize):
|
||||
# To display the links to stories that are not available to guests.
|
||||
if self.getConfig("username") and self.getConfig("always_login"):
|
||||
# performLogin extracts token from the soup
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
self.performLogin(url, soup)
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ class KakuyomuJpAdapter(BaseSiteAdapter):
|
|||
titles = []
|
||||
nestingLevel = 0
|
||||
newSection = False
|
||||
for tocNodeRef in info[workKey]['tableOfContentsV2']:
|
||||
for tocNodeRef in info[workKey]['tableOfContents']:
|
||||
tocNode = info[tocNodeRef['__ref']]
|
||||
|
||||
if tocNode['chapter'] is not None:
|
||||
|
|
@ -197,6 +197,8 @@ class KakuyomuJpAdapter(BaseSiteAdapter):
|
|||
self.add_chapter(epTitle, epUrl)
|
||||
newSection = False
|
||||
|
||||
self.story.setMetadata('numChapters', numEpisodes)
|
||||
|
||||
logger.debug("Story: <%s>", self.story)
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -144,13 +144,13 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Find authorid and URL from... author urls.
|
||||
pagetitle = soup.find('div',id='pagetitle')
|
||||
for a in pagetitle.find_all('a', href=re.compile(r"viewuser.php\?uid=\d+")):
|
||||
for a in pagetitle.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")):
|
||||
self.story.addToList('authorId',a['href'].split('=')[1])
|
||||
self.story.addToList('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.addToList('author',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -166,7 +166,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
|
@ -193,7 +193,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [stripHTML(cat) for cat in cats]
|
||||
for cat in catstext:
|
||||
# ran across one story with an empty <a href="browse.php?type=categories&catid=1"></a>
|
||||
|
|
@ -204,7 +204,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
if 'Characters' in label:
|
||||
self.story.addToList('characters','Kirk')
|
||||
self.story.addToList('characters','Spock')
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [stripHTML(char) for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',stripHTML(char))
|
||||
|
|
@ -213,7 +213,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genrestext = [stripHTML(genre) for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -223,7 +223,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
## has 'Story Type', which is much more what most sites
|
||||
## call genre.
|
||||
if 'Story Type' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
|
||||
genrestext = [stripHTML(genre) for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -233,21 +233,21 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warningstext = [stripHTML(warning) for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
self.story.addToList('warnings',stripHTML(warning))
|
||||
|
||||
if 'Universe' in label:
|
||||
universes = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
|
||||
universes = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
|
||||
universestext = [stripHTML(universe) for universe in universes]
|
||||
self.universe = ', '.join(universestext)
|
||||
for universe in universestext:
|
||||
self.story.addToList('universe',stripHTML(universe))
|
||||
|
||||
if 'Crossover Fandom' in label:
|
||||
crossoverfandoms = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX
|
||||
crossoverfandoms = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX
|
||||
crossoverfandomstext = [stripHTML(crossoverfandom) for crossoverfandom in crossoverfandoms]
|
||||
self.crossoverfandom = ', '.join(crossoverfandomstext)
|
||||
for crossoverfandom in crossoverfandomstext:
|
||||
|
|
@ -274,7 +274,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ from __future__ import absolute_import
|
|||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import json
|
||||
|
||||
from bs4.element import Comment
|
||||
from ..htmlcleanup import stripHTML
|
||||
|
|
@ -38,7 +37,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
#logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
|
||||
logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','litero')
|
||||
|
|
@ -54,7 +53,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## have been keeping the language when 'normalizing' to first
|
||||
## chapter.
|
||||
url = re.sub(r"^(https?://)"+LANG_RE+r"(\.i)?",
|
||||
r"https://\2",
|
||||
r"\1\2",
|
||||
url)
|
||||
url = url.replace('/beta/','/') # to allow beta site URLs.
|
||||
|
||||
|
|
@ -78,7 +77,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://www.literotica.com/s/story-title https://www.literotica.com/series/se/9999999 https://www.literotica.com/s/story-title https://www.literotica.com/i/image-or-comic-title https://www.literotica.com/p/poem-title https://portuguese.literotica.com/s/story-title https://german.literotica.com/s/story-title"
|
||||
return "http://www.literotica.com/s/story-title https://www.literotica.com/series/se/9999999 https://www.literotica.com/s/story-title https://www.literotica.com/i/image-or-comic-title https://www.literotica.com/p/poem-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
# also https://www.literotica.com/series/se/80075773
|
||||
|
|
@ -96,49 +95,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[-1])
|
||||
# logger.debug("language:%s"%self.story.getMetadata('language'))
|
||||
|
||||
## apply clean_chapter_titles
|
||||
def add_chapter(self,chapter_title,url,othermeta={}):
|
||||
if self.getConfig("clean_chapter_titles"):
|
||||
storytitle = self.story.getMetadataRaw('title').lower()
|
||||
chapter_name_type = None
|
||||
# strip trailing ch or pt before doing the chapter clean.
|
||||
# doesn't remove from story title metadata
|
||||
storytitle = re.sub(r'^(.*?)( (ch|pt))?$',r'\1',storytitle)
|
||||
if chapter_title.lower().startswith(storytitle):
|
||||
chapter = chapter_title[len(storytitle):].strip()
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter == '':
|
||||
chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
|
||||
# Sometimes the first chapter does not have type of chapter
|
||||
if self.num_chapters() == 0:
|
||||
# logger.debug('\tChapter: first chapter without chapter type')
|
||||
chapter_name_type = None
|
||||
else:
|
||||
separater_char = chapter[0]
|
||||
# logger.debug('\tseparater_char: "%s"' % separater_char)
|
||||
chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter.lower().startswith('ch.'):
|
||||
chapter = chapter[len('ch.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Chapter %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Chapter %s' % chapter
|
||||
chapter_name_type = 'Chapter' if chapter_name_type is None else chapter_name_type
|
||||
# logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif chapter.lower().startswith('pt.'):
|
||||
chapter = chapter[len('pt.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Part %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Part %s' % chapter
|
||||
chapter_name_type = 'Part' if chapter_name_type is None else chapter_name_type
|
||||
# logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif separater_char in [":", "-"]:
|
||||
chapter_title = chapter
|
||||
# logger.debug('\tChapter: taking chapter text as whole')
|
||||
super(LiteroticaSiteAdapter, self).add_chapter(chapter_title,url,othermeta)
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
"""
|
||||
In April 2024, site introduced significant changes, including
|
||||
|
|
@ -166,22 +122,13 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
if "This submission is awaiting moderator's approval" in data:
|
||||
raise exceptions.StoryDoesNotExist("This submission is awaiting moderator's approval. %s"%self.url)
|
||||
|
||||
## 2025Feb - domains other than www now use different HTML.
|
||||
## Need to look for two different versions of basically
|
||||
## everything.
|
||||
|
||||
## not series URL, assumed to be a chapter. Look for Story
|
||||
## Info block of post-beta page. I don't think it should happen?
|
||||
if '/series/se' not in self.url:
|
||||
#logger.debug(data)
|
||||
## looking for /series/se URL to indicate this is a
|
||||
## chapter.
|
||||
if not soup.select_one('div.page__aside') and not soup.select_one('div.sidebar') and not soup.select_one('div[class^="_sidebar_"]'):
|
||||
if not soup.select_one('div.page__aside'):
|
||||
raise exceptions.FailedToDownload("Missing Story Info block, Beta turned off?")
|
||||
|
||||
storyseriestag = soup.select_one('a.bn_av')
|
||||
if not storyseriestag:
|
||||
storyseriestag = soup.select_one('a[class^="_files__link_"]')
|
||||
# logger.debug("Story Series Tag:%s"%storyseriestag)
|
||||
|
||||
if storyseriestag:
|
||||
|
|
@ -195,18 +142,8 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
isSingleStory = '/series/se' not in self.url
|
||||
|
||||
if not isSingleStory:
|
||||
# Normilize the url?
|
||||
state = re.findall(r"prefix\=\"/series/\",state='(.+?)'</script>", data)
|
||||
json_state = json.loads(state[0].replace("\\'","'").replace("\\\\","\\"))
|
||||
url_series_id = unicode(re.match(self.getSiteURLPattern(),self.url).group('storyseriesid'))
|
||||
json_series_id = unicode(json_state['series']['data']['id'])
|
||||
if json_series_id != url_series_id:
|
||||
res = re.sub(url_series_id, json_series_id, unicode(self.url))
|
||||
logger.debug("Normalized url: %s"%res)
|
||||
self._setURL(res)
|
||||
|
||||
## common between one-shots and multi-chapters
|
||||
|
||||
# title
|
||||
self.story.setMetadata('title', stripHTML(soup.select_one('h1')))
|
||||
# logger.debug(self.story.getMetadata('title'))
|
||||
|
|
@ -220,8 +157,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## Should change to /authors/ if/when it starts appearing.
|
||||
## Assuming it's in the same place.
|
||||
authora = soup.find("a", class_="y_eU")
|
||||
if not authora:
|
||||
authora = soup.select_one('a[class^="_author__title"]')
|
||||
authorurl = authora['href']
|
||||
if authorurl.startswith('//'):
|
||||
authorurl = self.parsedUrl.scheme+':'+authorurl
|
||||
|
|
@ -236,29 +171,19 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
else: # if all else fails
|
||||
self.story.setMetadata('authorId', stripHTML(authora))
|
||||
|
||||
if soup.select('div#tabpanel-tags'):
|
||||
# logger.debug("tags1")
|
||||
self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div#tabpanel-tags a.av_as') ])
|
||||
if soup.select('div[class^="_widget__tags_"]'):
|
||||
# logger.debug("tags2")
|
||||
self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div[class^="_widget__tags_"] a[class^="_tag_item_"]') ])
|
||||
# logger.debug(self.story.getList('eroticatags'))
|
||||
self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div#tabpanel-tags a.av_as') ])
|
||||
|
||||
## look first for 'Series Introduction', then Info panel short desc
|
||||
## series can have either, so put in common code.
|
||||
desc = []
|
||||
introtag = soup.select_one('div.bp_rh')
|
||||
descdiv = soup.select_one('div#tabpanel-info div.bn_B') or \
|
||||
soup.select_one('div[class^="_tab__pane_"] div[class^="_widget__info_"]')
|
||||
introtag = soup.select_one('div.bp_rh p')
|
||||
descdiv = soup.select_one('div#tabpanel-info div.bn_B')
|
||||
if introtag and stripHTML(introtag):
|
||||
# make sure there's something in the tag.
|
||||
# logger.debug("intro %s"%introtag)
|
||||
desc.append(unicode(introtag))
|
||||
self.setDescription(self.url,introtag)
|
||||
elif descdiv and stripHTML(descdiv):
|
||||
# make sure there's something in the tag.
|
||||
# logger.debug("desc %s"%descdiv)
|
||||
desc.append(unicode(descdiv))
|
||||
if not desc or self.getConfig("include_chapter_descriptions_in_summary"):
|
||||
self.setDescription(self.url,descdiv)
|
||||
else:
|
||||
## Only for backward compatibility with 'stories' that
|
||||
## don't have an intro or short desc.
|
||||
descriptions = []
|
||||
|
|
@ -268,9 +193,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
descriptions.append("%d. %s" % (i + 1, stripHTML(chapterdesctag)))
|
||||
# now put it back--it's used below
|
||||
chapterdesctag.append(a)
|
||||
desc.append(unicode("<p>"+"</p>\n<p>".join(descriptions)+"</p>"))
|
||||
|
||||
self.setDescription(self.url,u''.join(desc))
|
||||
self.setDescription(authorurl,"<p>"+"</p>\n<p>".join(descriptions)+"</p>")
|
||||
|
||||
if isSingleStory:
|
||||
## one-shots don't *display* date info, but they have it
|
||||
|
|
@ -280,30 +203,16 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## multichap also have "date_approve", but they have
|
||||
## several and they're more than just the story chapters.
|
||||
date = re.search(r'"date_approve":"(\d\d/\d\d/\d\d\d\d)"',data)
|
||||
if not date:
|
||||
date = re.search(r'date_approve:"(\d\d/\d\d/\d\d\d\d)"',data)
|
||||
if date:
|
||||
dateval = makeDate(date.group(1), self.dateformat)
|
||||
self.story.setMetadata('datePublished', dateval)
|
||||
self.story.setMetadata('dateUpdated', dateval)
|
||||
|
||||
## one-shots don't have same json data to get aver_rating
|
||||
## from below. This kludge matches the data_approve
|
||||
rateall = re.search(r'rate_all:([\d\.]+)',data)
|
||||
if rateall:
|
||||
self.story.setMetadata('averrating', '%4.2f' % float(rateall.group(1)))
|
||||
|
||||
## one-shots assumed completed.
|
||||
self.story.setMetadata('status','Completed')
|
||||
|
||||
# Add the category from the breadcumb.
|
||||
breadcrumbs = soup.find('div', id='BreadCrumbComponent')
|
||||
if not breadcrumbs:
|
||||
breadcrumbs = soup.select_one('ul[class^="_breadcrumbs_list_"]')
|
||||
if not breadcrumbs:
|
||||
# _breadcrumbs_18u7l_1
|
||||
breadcrumbs = soup.select_one('nav[class^="_breadcrumbs_"]')
|
||||
self.story.addToList('category', breadcrumbs.find_all('a')[1].string)
|
||||
self.story.addToList('category', soup.find('div', id='BreadCrumbComponent').findAll('a')[1].string)
|
||||
|
||||
## one-shot chapter
|
||||
self.add_chapter(self.story.getMetadata('title'), self.url)
|
||||
|
|
@ -312,8 +221,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## Multi-chapter stories. AKA multi-part 'Story Series'.
|
||||
bn_antags = soup.select('div#tabpanel-info p.bn_an')
|
||||
# logger.debug(bn_antags)
|
||||
if bn_antags and not self.getConfig("dates_from_chapters"):
|
||||
## Use dates from series metadata unless dates_from_chapters is enabled
|
||||
if bn_antags:
|
||||
dates = []
|
||||
for datetag in bn_antags[:2]:
|
||||
datetxt = stripHTML(datetag)
|
||||
|
|
@ -335,11 +243,52 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## category from chapter list
|
||||
self.story.extendList('category',[ stripHTML(t) for t in soup.select('a.br_rl') ])
|
||||
|
||||
storytitle = self.story.getMetadata('title').lower()
|
||||
chapter_name_type = None
|
||||
for chapteratag in soup.select('a.br_rj'):
|
||||
chapter_title = stripHTML(chapteratag)
|
||||
# logger.debug('\tChapter: "%s"' % chapteratag)
|
||||
if self.getConfig("clean_chapter_titles"):
|
||||
# strip trailing ch or pt before doing the chapter clean.
|
||||
# doesn't remove from story title metadata
|
||||
storytitle = re.sub(r'^(.*?)( (ch|pt))?$',r'\1',storytitle)
|
||||
if chapter_title.lower().startswith(storytitle):
|
||||
chapter = chapter_title[len(storytitle):].strip()
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter == '':
|
||||
chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
|
||||
# Sometimes the first chapter does not have type of chapter
|
||||
if self.num_chapters() == 0:
|
||||
# logger.debug('\tChapter: first chapter without chapter type')
|
||||
chapter_name_type = None
|
||||
else:
|
||||
separater_char = chapter[0]
|
||||
# logger.debug('\tseparater_char: "%s"' % separater_char)
|
||||
chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter.lower().startswith('ch.'):
|
||||
chapter = chapter[len('ch.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Chapter %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Chapter %s' % chapter
|
||||
chapter_name_type = 'Chapter' if chapter_name_type is None else chapter_name_type
|
||||
# logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif chapter.lower().startswith('pt.'):
|
||||
chapter = chapter[len('pt.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Part %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Part %s' % chapter
|
||||
chapter_name_type = 'Part' if chapter_name_type is None else chapter_name_type
|
||||
# logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif separater_char in [":", "-"]:
|
||||
chapter_title = chapter
|
||||
# logger.debug('\tChapter: taking chapter text as whole')
|
||||
|
||||
# /series/se does include full URLs current.
|
||||
chapurl = chapteratag['href']
|
||||
|
||||
# logger.debug("Chapter URL: " + chapurl)
|
||||
self.add_chapter(chapter_title, chapurl)
|
||||
|
||||
|
|
@ -349,7 +298,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
self.setCoverImage(self.url,coverimg['src'])
|
||||
|
||||
#### Attempting averrating from JS metadata.
|
||||
#### also alternate chapters from json
|
||||
try:
|
||||
state_start="state='"
|
||||
state_end="'</script>"
|
||||
|
|
@ -358,48 +306,20 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
state = data[i+len(state_start):data.index(state_end,i)].replace("\\'","'").replace("\\\\","\\")
|
||||
if state:
|
||||
# logger.debug(state)
|
||||
import json
|
||||
json_state = json.loads(state)
|
||||
# logger.debug(json.dumps(json_state, sort_keys=True,indent=2, separators=(',', ':')))
|
||||
all_rates = []
|
||||
if 'series' in json_state:
|
||||
## one-shot
|
||||
if 'story' in json_state:
|
||||
all_rates = [ float(json_state['story']['data']['rate_all']) ]
|
||||
## series
|
||||
elif 'series' in json_state:
|
||||
all_rates = [ float(x['rate_all']) for x in json_state['series']['works'] ]
|
||||
|
||||
## Extract dates from chapter approval dates if dates_from_chapters is enabled
|
||||
if self.getConfig("dates_from_chapters"):
|
||||
date_approvals = []
|
||||
for work in json_state['series']['works']:
|
||||
if 'date_approve' in work:
|
||||
try:
|
||||
date_approvals.append(makeDate(work['date_approve'], self.dateformat))
|
||||
except:
|
||||
pass
|
||||
if date_approvals:
|
||||
# Oldest date is published, newest is updated
|
||||
date_approvals.sort()
|
||||
self.story.setMetadata('datePublished', date_approvals[0])
|
||||
self.story.setMetadata('dateUpdated', date_approvals[-1])
|
||||
if all_rates:
|
||||
self.story.setMetadata('averrating', '%4.2f' % (sum(all_rates) / float(len(all_rates))))
|
||||
|
||||
## alternate chapters from JSON
|
||||
if self.num_chapters() < 1:
|
||||
logger.debug("Getting Chapters from series JSON")
|
||||
seriesid = json_state.get('series',{}).get('data',{}).get('id',None)
|
||||
if seriesid:
|
||||
logger.info("Fetching chapter data from JSON")
|
||||
logger.debug(seriesid)
|
||||
series_json = json.loads(self.get_request('https://literotica.com/api/3/series/%s/works'%seriesid))
|
||||
# logger.debug(json.dumps(series_json, sort_keys=True,indent=2, separators=(',', ':')))
|
||||
for chap in series_json:
|
||||
self.add_chapter(chap['title'], 'https://www.literotica.com/s/'+chap['url'])
|
||||
|
||||
## Collect tags from series/story page if tags_from_chapters is enabled
|
||||
if self.getConfig("tags_from_chapters"):
|
||||
self.story.extendList('eroticatags', [ unicode(t['tag']).title() for t in chap['tags'] ])
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Processing JSON failed. (%s)"%e)
|
||||
logger.debug("Processing JSON to find averrating failed. (%s)"%e)
|
||||
|
||||
## Features removed because not supportable by new site form:
|
||||
## averrating metadata entry
|
||||
|
|
@ -408,13 +328,14 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
return
|
||||
|
||||
def getPageText(self, raw_page, url):
|
||||
logger.debug('Getting page text')
|
||||
# logger.debug('Getting page text')
|
||||
# logger.debug(soup)
|
||||
raw_page = raw_page.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
|
||||
# logger.debug("\tChapter text: %s" % raw_page)
|
||||
# logger.debug("\tChapter text: %s" % raw_page)
|
||||
page_soup = self.make_soup(raw_page)
|
||||
[comment.extract() for comment in page_soup.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in page_soup.findAll(string=lambda text:isinstance(text, Comment))]
|
||||
fullhtml = ""
|
||||
for aa_ht_div in page_soup.find_all('div', 'aa_ht') + page_soup.select('div[class^="_article__content_"]'):
|
||||
for aa_ht_div in page_soup.find_all('div', 'aa_ht'):
|
||||
if aa_ht_div.div:
|
||||
html = unicode(aa_ht_div.div)
|
||||
# Strip some starting and ending tags,
|
||||
|
|
@ -432,13 +353,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
raw_page = self.get_request(url)
|
||||
page_soup = self.make_soup(raw_page)
|
||||
pages = page_soup.find('div',class_='l_bH')
|
||||
if not pages:
|
||||
pages = page_soup.select_one('div._pagination_h0sum_1')
|
||||
if not pages:
|
||||
pages = page_soup.select_one('div.clearfix.panel._pagination_1400x_1')
|
||||
if not pages:
|
||||
pages = page_soup.select_one('div[class^="panel clearfix _pagination_"]')
|
||||
# logger.debug(pages)
|
||||
|
||||
fullhtml = ""
|
||||
chapter_description = ''
|
||||
|
|
@ -451,10 +365,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## look for highest numbered page, they're not all listed
|
||||
## when there are many.
|
||||
|
||||
last_page_links = pages.find_all('a', class_='l_bJ')
|
||||
if not last_page_links:
|
||||
last_page_links = pages.select('a[class^="_pagination__item_"]')
|
||||
last_page_link = last_page_links[-1]
|
||||
last_page_link = pages.find_all('a', class_='l_bJ')[-1]
|
||||
last_page_no = int(urlparse.parse_qs(last_page_link['href'].split('?')[1])['page'][0])
|
||||
# logger.debug(last_page_no)
|
||||
for page_no in range(2, last_page_no+1):
|
||||
|
|
@ -463,7 +374,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
raw_page = self.get_request(page_url)
|
||||
fullhtml += self.getPageText(raw_page, url)
|
||||
|
||||
#logger.debug(fullhtml)
|
||||
# logger.debug(fullhtml)
|
||||
page_soup = self.make_soup(fullhtml)
|
||||
fullhtml = self.utf8FromSoup(url, self.make_soup(fullhtml))
|
||||
fullhtml = chapter_description + fullhtml
|
||||
|
|
@ -471,123 +382,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
return fullhtml
|
||||
|
||||
def get_urls_from_page(self,url,normalize):
|
||||
from ..geturls import get_urls_from_html
|
||||
|
||||
## hook for logins, etc.
|
||||
self.before_get_urls_from_page(url,normalize)
|
||||
|
||||
# this way it uses User-Agent or other special settings.
|
||||
data = self.get_request(url,usecache=False)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
page_urls = get_urls_from_html(soup, url, configuration=self.configuration, normalize=normalize)
|
||||
|
||||
if not self.getConfig("fetch_stories_from_api",True):
|
||||
logger.debug('fetch_stories_from_api Not enabled')
|
||||
return {'urllist': page_urls}
|
||||
|
||||
user_story_list = re.search(r'literotica\.com/authors/.+?/lists\?listid=(?P<list_id>\d+)', url)
|
||||
fav_authors = re.search(r'literotica\.com/authors/.+?/favorites', url)
|
||||
written = re.search(r'literotica.com/authors/.+?/works/', url)
|
||||
logger.debug((bool(user_story_list), bool(fav_authors), bool(written)))
|
||||
|
||||
# If the url is not supported
|
||||
if not user_story_list and not fav_authors and not written:
|
||||
logger.debug('No supported link. %s', url)
|
||||
return {'urllist':page_urls}
|
||||
|
||||
# Grabbing the main list where chapters are contained.
|
||||
if user_story_list:
|
||||
js_story_list = re.search(r';\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]\);\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]=\{success:!\d,current_page:(?P<current_page>\d+?),last_page:(?P<last_page>\d+?),total:\d+?,per_page:\d+,(has_series:!\d)?data:\$R\[\d+?\]=\[\$R\[\d+?\]=(?P<data>.+)\}\]\}\);', data) # }] } } }); \$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]\);\$R\[\d+?]\(\$R\[\d+?\],\$R\[\d+?\]=\{sliders:
|
||||
logger.debug('user_story_list ID [%s]'%user_story_list.group('list_id'))
|
||||
else:
|
||||
js_story_list = re.search(r'\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]={current_page:(?P<current_page>\d+?),last_page:(?P<last_page>\d+?),total:\d+?,per_page:\d+,(has_series:!\d,)?data:\$R\[\d+\]=\[\$R\[\d+\]=\{(?!aim)(?P<data>.+)\}\);_\$HY\.r\[', data)
|
||||
|
||||
# In case the regex becomes outdated
|
||||
if not js_story_list:
|
||||
logger.debug('Failed to grab data from the js.')
|
||||
return {'urllist':page_urls}
|
||||
|
||||
user = None
|
||||
script_tags = soup.find_all('script')
|
||||
for script in script_tags:
|
||||
if not script.string:
|
||||
continue
|
||||
# Getting author from the js.
|
||||
user = re.search(r'_\$HY\.r\[\"AuthorQuery\[\\\"(?P<author>.+?)\\\"\]\"\]', script.string)
|
||||
if user != None:
|
||||
logger.debug("User: [%s]"%user.group('author'))
|
||||
break
|
||||
else:
|
||||
logger.debug('Failed to get a username')
|
||||
return {'urllist': page_urls}
|
||||
|
||||
# Extract the current (should be 1) and last page numbers from the js.
|
||||
logger.debug("Pages %s/%s"%(js_story_list.group('current_page'), js_story_list.group('last_page')))
|
||||
|
||||
urls = []
|
||||
# Necessary to format a proper link as there were no visible data specifying what kind of link that should be.
|
||||
cat_to_link = {'adult-comics': 'i', 'erotic-art': 'i', 'illustrated-poetry': 'p', 'erotic-audio-poetry': 'p', 'erotic-poetry': 'p', 'non-erotic-poetry': 'p'}
|
||||
stories_found = re.findall(r"category_info:\$R\[.*?type:\".+?\",pageUrl:\"(.+?)\"}.+?,type:\"(.+?)\",url:\"(.+?)\",", js_story_list.group('data'))
|
||||
for story in stories_found:
|
||||
story_category, story_type, story_url = story
|
||||
urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(story_category, 's'), story_url))
|
||||
|
||||
# Removes the duplicates
|
||||
seen = set()
|
||||
urls = [x for x in (page_urls + urls) if not (x in seen or seen.add(x))]
|
||||
logger.debug("Found [%s] stories so far."%len(urls))
|
||||
|
||||
# Sometimes the rest of the stories are burried in the js so no fetching in necessery.
|
||||
if js_story_list.group('last_page') == js_story_list.group('current_page'):
|
||||
return {'urllist': urls}
|
||||
|
||||
user = urlparse.quote(user.group(1))
|
||||
logger.debug("Escaped user: [%s]"%user)
|
||||
|
||||
if written:
|
||||
category = re.search(r"_\$HY\.r\[\"AuthorSeriesAndWorksQuery\[\\\".+?\\\",\\\"\D+?\\\",\\\"(?P<type>\D+?)\\\"\]\"\]=\$R\[\d+?\]=\$R\[\d+?\]\(\$R\[\d+?\]=\{", data)
|
||||
elif fav_authors:
|
||||
category = re.search(r"_\$HY\.r\[\"AuthorFavoriteWorksQuery\[\\\".+?\\\",\\\"(?P<type>\D+?)\\\",\d\]\"\]=\$R\[\d+?\]=\$R\[\d+?\]\(\$R\[\d+?\]={", data)
|
||||
|
||||
if not user_story_list and not category:
|
||||
logger.debug("Type of works not found")
|
||||
return {'urllist': urls}
|
||||
|
||||
last_page = int(js_story_list.group('last_page'))
|
||||
current_page = int(js_story_list.group('current_page')) + 1
|
||||
# Fetching the remaining urls from api. Can't trust the number given about the pages left from a website. Sometimes even the api returns outdated number of pages.
|
||||
while current_page <= last_page:
|
||||
i = len(urls)
|
||||
logger.debug("Pages %s/%s"%(current_page, int(last_page)))
|
||||
if fav_authors:
|
||||
jsn = self.get_request('https://literotica.com/api/3/users/{}/favorite/works?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22type%22%3A%22{}%22%2C%22withSeriesDetails%22%3Atrue%7D'.format(user, current_page, category.group('type')))
|
||||
elif user_story_list:
|
||||
jsn = self.get_request('https://literotica.com/api/3/users/{}/list/{}?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22withSeriesDetails%22%3Atrue%7D'.format(user, user_story_list.group('list_id'), current_page))
|
||||
else:
|
||||
jsn = self.get_request('https://literotica.com/api/3/users/{}/series_and_works?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22sort%22%3A%22date%22%2C%22type%22%3A%22{}%22%2C%22listType%22%3A%22expanded%22%7D'.format(user, current_page, category.group('type')))
|
||||
|
||||
urls_data = json.loads(jsn)
|
||||
last_page = urls_data["last_page"]
|
||||
current_page = int(urls_data["current_page"]) + 1
|
||||
for story in urls_data['data']:
|
||||
#logger.debug('parts' in story)
|
||||
if story['url'] and story.get('work_count') == None:
|
||||
urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(story["category_info"]["pageUrl"], 's'), str(story['url'])))
|
||||
continue
|
||||
# Most of the time series has no url specified and contains all of the story links belonging to the series
|
||||
urls.append('https://www.literotica.com/series/se/%s'%str(story['id']))
|
||||
for series_story in story['parts']:
|
||||
urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(series_story["category_info"]["pageUrl"], 's'), str(series_story['url'])))
|
||||
logger.debug("Found [%s] stories."%(len(urls) - i))
|
||||
|
||||
# Again removing duplicates.
|
||||
seen = set()
|
||||
urls = [x for x in urls if not (x in seen or seen.add(x))]
|
||||
|
||||
logger.debug("Found total of [%s] stories"%len(urls))
|
||||
return {'urllist':urls}
|
||||
|
||||
def getClass():
|
||||
return LiteroticaSiteAdapter
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
|
||||
value = labels[0].previousSibling
|
||||
svalue = ""
|
||||
|
|
@ -154,22 +154,22 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value.split(' -')[0])
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -194,7 +194,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
|
|||
self.story.extendList('authorId', [authorId])
|
||||
self.story.extendList('authorUrl', [authorUrl])
|
||||
|
||||
if not self.story.getMetadataRaw('rating'):
|
||||
if not self.story.getMetadata('rating'):
|
||||
ratingTitle = chapter.getRatingTitle()
|
||||
if ratingTitle:
|
||||
self.story.setMetadata('rating', ratingTitle)
|
||||
|
|
@ -204,6 +204,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('datePublished', datePublished)
|
||||
self.story.setMetadata('dateUpdated', dateUpdated)
|
||||
self.story.setMetadata('numWords', unicode(wordCount))
|
||||
self.story.setMetadata('numChapters', len(chapters))
|
||||
|
||||
# Site-specific metadata.
|
||||
self.story.setMetadata('language', self.SITE_LANGUAGE)
|
||||
|
|
@ -677,7 +678,7 @@ class Chapter(object):
|
|||
|
||||
def _excludeEditorSignature(self, root):
|
||||
"""Exclude editor signature from within `root' element."""
|
||||
for stringNode in root.find_all(string=True):
|
||||
for stringNode in root.findAll(string=True):
|
||||
if re.match(self.SIGNED_PATTERN, textNode.string):
|
||||
editorLink = textNode.findNext('a')
|
||||
if editorLink:
|
||||
|
|
|
|||
|
|
@ -64,9 +64,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
|
|||
return "https://mcstories.com/StoryTitle/ https://mcstories.com/StoryTitle/index.html https://mcstories.com/StoryTitle/StoryTitle1.html"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
## Note that this uses a regular expression *negative*
|
||||
## lookahead--story URLs *can't* have /Titles/ /Authors/ etc.
|
||||
return r"https?://(www\.)?mcstories\.com(?!/(Titles|Authors|Tags|ReadersPicks)/)/[a-zA-Z0-9_-]+/"
|
||||
return r"https?://(www\.)?mcstories\.com/([a-zA-Z0-9_-]+)/"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -148,12 +148,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# category
|
||||
# <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
|
||||
for a in soup.find_all('a',href=re.compile(r"^/fanfic/a/")):
|
||||
for a in soup.findAll('a',href=re.compile(r"^/fanfic/a/")):
|
||||
self.story.addToList('category',a.string)
|
||||
|
||||
# genre
|
||||
# <a href="/fanfic/src.php/g/567">Ranma 1/2</a>
|
||||
for a in soup.find_all('a',href=re.compile(r"^/fanfic/src.php/g/")):
|
||||
for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")):
|
||||
self.story.addToList('genre',a.string)
|
||||
|
||||
metasoup = soup.find("div",{"class":"post-meta"})
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -170,7 +170,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -191,13 +191,13 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
|
@ -206,7 +206,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -216,7 +216,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warningstext = [warning.string for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
|
|
@ -243,7 +243,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter):
|
|||
[a.extract() for a in story.find_all('a')]
|
||||
|
||||
# Some tags have non-standard tag name.
|
||||
for tag in story.find_all(recursive=True):
|
||||
for tag in story.findAll(recursive=True):
|
||||
if tag.name not in HTML_TAGS:
|
||||
tag.name = 'span'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2024 FanFicFare team
|
||||
# Copyright 2020 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -15,24 +15,34 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .base_otw_adapter import BaseOTWAdapter
|
||||
from .adapter_wuxiaworldxyz import WuxiaWorldXyzSiteAdapter
|
||||
|
||||
def getClass():
|
||||
return CFAAAdapter
|
||||
return NovelUpdatesCcSiteAdapter
|
||||
|
||||
class CFAAAdapter(BaseOTWAdapter):
|
||||
class NovelUpdatesCcSiteAdapter(WuxiaWorldXyzSiteAdapter):
|
||||
DATE_FORMAT = '%Y-%m-%d %H:%M'
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseOTWAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','cfaa')
|
||||
WuxiaWorldXyzSiteAdapter.__init__(self, config, url)
|
||||
self.story.setMetadata('siteabbrev', 'nucc')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.cfaarchive.org'
|
||||
return 'www.novelupdates.cc'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.novelupdates.cc','m.novelupdates.cc']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return 'https://%s/story-name' % cls.getSiteDomain()
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r'https?://(www|m)\.novelupdates\.cc/(?P<id>[^/]+)(/)?'
|
||||
|
|
@ -137,14 +137,14 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
try:
|
||||
# in case link points somewhere other than the first chapter
|
||||
a = soup.find_all('option')[1]['value']
|
||||
a = soup.findAll('option')[1]['value']
|
||||
self.story.setMetadata('storyId',a.split('=',)[1])
|
||||
url = 'http://'+self.host+'/'+a
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
except:
|
||||
pass
|
||||
|
||||
for info in asoup.find_all('table', {'class' : 'border'}):
|
||||
for info in asoup.findAll('table', {'class' : 'border'}):
|
||||
a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
if a != None:
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
|
@ -152,7 +152,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
if len(chapters) == 0:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
|
|
@ -171,7 +171,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
except:
|
||||
return ""
|
||||
|
||||
cats = info.find_all('a',href=re.compile('categories.php'))
|
||||
cats = info.findAll('a',href=re.compile('categories.php'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
|
|
@ -188,7 +188,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.setDescription(url,svalue)
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = info.find_all('b')
|
||||
labels = info.findAll('b')
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
|
|
|||
|
|
@ -93,26 +93,26 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
|
|||
chapters = soup.find('select')
|
||||
if chapters == None:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
for b in soup.find_all('b'):
|
||||
for b in soup.findAll('b'):
|
||||
if b.text == "Updated":
|
||||
date = b.nextSibling.string.split(': ')[1].split(',')
|
||||
self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
|
||||
else:
|
||||
i = 0
|
||||
chapters = chapters.find_all('option')
|
||||
chapters = chapters.findAll('option')
|
||||
for chapter in chapters:
|
||||
self.add_chapter(chapter,'https://'+self.host+chapter['value'])
|
||||
if i == 0:
|
||||
self.story.setMetadata('storyId',chapter['value'].split('/')[3])
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).find_all('b')
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).findAll('b')
|
||||
for b in head:
|
||||
if b.text == "Updated":
|
||||
date = b.nextSibling.string.split(': ')[1].split(',')
|
||||
self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
|
||||
|
||||
if i == (len(chapters)-1):
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).find_all('b')
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).findAll('b')
|
||||
for b in head:
|
||||
if b.text == "Updated":
|
||||
date = b.nextSibling.string.split(': ')[1].split(',')
|
||||
|
|
@ -160,20 +160,20 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
chapter=self.make_soup('<div class="story"></div>')
|
||||
for p in soup.find_all(['p','blockquote']):
|
||||
for p in soup.findAll(['p','blockquote']):
|
||||
if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p):
|
||||
break
|
||||
chapter.append(p)
|
||||
|
||||
for a in chapter.find_all('div'):
|
||||
for a in chapter.findAll('div'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('table'):
|
||||
for a in chapter.findAll('table'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('script'):
|
||||
for a in chapter.findAll('script'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('form'):
|
||||
for a in chapter.findAll('form'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('textarea'):
|
||||
for a in chapter.findAll('textarea'):
|
||||
a.extract()
|
||||
|
||||
|
||||
|
|
|
|||
241
fanficfare/adapters/adapter_ponyfictionarchivenet.py
Normal file
241
fanficfare/adapters/adapter_ponyfictionarchivenet.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return PonyFictionArchiveNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
# normalized story URL.
|
||||
if "explicit" in self.parsedUrl.netloc:
|
||||
self._setURL('https://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self.dateformat = "%d/%b/%y"
|
||||
else:
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self.dateformat = "%d %b %Y"
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','pffa')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'ponyfictionarchive.net'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234 https://explicit."+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&warning=9"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
if status: # apparently this site can have stories with neither In-Progress or Complete.
|
||||
self.story.setMetadata('status',status.string)
|
||||
|
||||
try:
|
||||
# explicit.site and .site have some differences now...
|
||||
section = soup.findAll('span', {'class' : 'General'})[1]
|
||||
self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
|
||||
|
||||
value = section.nextSibling
|
||||
svalue = ""
|
||||
while 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
except:
|
||||
# find rating in data
|
||||
# <br /> • Mature • <br />
|
||||
lead = "<br /> • "
|
||||
trail = " • <br />"
|
||||
rating = data[data.index(lead)+len(lead):data.index(trail)]
|
||||
if len(rating)<20: # minor sanity check.
|
||||
self.story.setMetadata('rating',rating)
|
||||
descstr = data[data.index(trail)+len(trail):] # from desc on
|
||||
descstr = descstr[:descstr.index('<span class="label">')] # remove after desc.
|
||||
self.setDescription(url,descstr)
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -80,7 +80,7 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/fanfiction/'+chapter['href'])
|
||||
|
||||
|
|
@ -92,7 +92,7 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -116,13 +116,13 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('reads', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
if "Snape and Harry (required)" in char:
|
||||
|
|
@ -132,27 +132,27 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
|
|||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Warning' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',stripHTML(warning))
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',stripHTML(genre))
|
||||
|
||||
if 'Takes Place' in label:
|
||||
takesplaces = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class'))
|
||||
takesplaces = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
|
||||
for takesplace in takesplaces:
|
||||
self.story.addToList('takesplaces',stripHTML(takesplace))
|
||||
|
||||
if 'Snape flavour' in label:
|
||||
snapeflavours = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class'))
|
||||
snapeflavours = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
|
||||
for snapeflavour in snapeflavours:
|
||||
self.story.addToList('snapeflavours',stripHTML(snapeflavour))
|
||||
|
||||
if 'Tags' in label:
|
||||
sitetags = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class'))
|
||||
sitetags = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
|
||||
for sitetag in sitetags:
|
||||
self.story.addToList('sitetags',stripHTML(sitetag))
|
||||
|
||||
|
|
@ -176,7 +176,7 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/fanfiction/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/missingpieces/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -138,7 +138,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -159,22 +159,22 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -198,7 +198,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -126,7 +126,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
|
|||
except:
|
||||
return ""
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -147,22 +147,22 @@ class PsychFicComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -186,7 +186,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -104,42 +104,6 @@ class RoyalRoadAdapter(BaseSiteAdapter):
|
|||
def getSiteURLPattern(self):
|
||||
return "https?"+re.escape("://")+r"(www\.|)royalroadl?\.com/fiction/\d+(/.*)?$"
|
||||
|
||||
|
||||
# rr won't send you future updates if you aren't 'caught up'
|
||||
# on the story. Login isn't required but logging in will
|
||||
# mark stories you've downloaded as 'read' on rr.
|
||||
def performLogin(self):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['Email'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['Email'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
|
||||
if not params['password']:
|
||||
return
|
||||
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/account/login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['Email']))
|
||||
|
||||
## need to pull empty login page first to get request token
|
||||
soup = self.make_soup(self.get_request(loginUrl))
|
||||
## FYI, this will fail if cookiejar is shared, but
|
||||
## use_basic_cache is false.
|
||||
params['__RequestVerificationToken']=soup.find('input', {'name':'__RequestVerificationToken'})['value']
|
||||
|
||||
d = self.post_request(loginUrl, params)
|
||||
if "Sign in" in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s (requires Email not name)" % (loginUrl,
|
||||
params['Email']))
|
||||
raise exceptions.FailedToLogin(self.url,"Failed to login as %s (RoyalRoad requires Email not name)" % params['Email'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## RR chapter URL only requires the chapter ID number field to be correct, story ID and title values are ignored
|
||||
## URL format after the domain /fiction/ is long form, storyID/storyTitle/chapter/chapterID/chapterTitle
|
||||
## short form has /fiction/chapter/chapterID both forms have optional final /
|
||||
|
|
@ -196,9 +160,6 @@ class RoyalRoadAdapter(BaseSiteAdapter):
|
|||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
# Log in so site will mark the chapers as read
|
||||
self.performLogin()
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
|
@ -226,7 +187,7 @@ class RoyalRoadAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
chapters = soup.find('table',{'id':'chapters'}).find('tbody')
|
||||
tds = [tr.find_all('td') for tr in chapters.find_all('tr')]
|
||||
tds = [tr.findAll('td') for tr in chapters.findAll('tr')]
|
||||
|
||||
if not tds:
|
||||
raise exceptions.FailedToDownload(
|
||||
|
|
@ -266,8 +227,6 @@ class RoyalRoadAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('status', 'Stub')
|
||||
elif 'DROPPED' == label:
|
||||
self.story.setMetadata('status', 'Dropped')
|
||||
elif 'INACTIVE' == label:
|
||||
self.story.setMetadata('status', 'Inactive')
|
||||
elif 'Fan Fiction' == label:
|
||||
self.story.addToList('category', 'FanFiction')
|
||||
elif 'Original' == label:
|
||||
|
|
@ -289,8 +248,7 @@ class RoyalRoadAdapter(BaseSiteAdapter):
|
|||
if img:
|
||||
cover_url = img['src']
|
||||
# usually URL is for thumbnail. Try expected URL for larger image, if fails fall back to the original URL
|
||||
cover_set = self.setCoverImage(url,cover_url.replace('/covers-full/', '/covers-large/'))[0]
|
||||
if not cover_set or cover_set.startswith("failedtoload"):
|
||||
if self.setCoverImage(url,cover_url.replace('/covers-full/', '/covers-large/'))[0] == "failedtoload":
|
||||
self.setCoverImage(url,cover_url)
|
||||
# some content is show as tables, this will preserve them
|
||||
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Find authorid and URL from... author url.
|
||||
# (fetch multiple authors)
|
||||
alist = soup.find_all('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('=')[1])
|
||||
self.story.addToList('authorUrl','http://'+self.host+'/fanfics/'+a['href'])
|
||||
|
|
@ -201,11 +201,11 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Reviews
|
||||
reviewdata = soup.find('div', {'id' : 'sort'})
|
||||
a = reviewdata.find_all('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
|
||||
a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
|
||||
self.story.setMetadata('reviews',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/fanfics/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -222,7 +222,7 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -237,13 +237,13 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
|
@ -252,7 +252,7 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -262,7 +262,7 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warningstext = [warning.string for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
|
|
@ -291,7 +291,7 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'http://'+self.host+'/fanfics/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -57,9 +57,16 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
self.urltitle = "some-title"
|
||||
|
||||
self.set_story_idurl(url)
|
||||
m = re.match(self.getSiteURLPattern(),url)
|
||||
# logger.debug("id:%s"%m.group('id'))
|
||||
# logger.debug("title:%s"%m.group('title'))
|
||||
|
||||
# get storyId from url
|
||||
self.story.setMetadata('storyId', m.group('id'))
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/series/' + self.story.getMetadata('storyId') + '/' + m.group('title') + '/')
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','scrhub') # XXX
|
||||
|
|
@ -68,19 +75,6 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%b %d, %Y" # XXX
|
||||
|
||||
def set_story_idurl(self,url):
|
||||
m = re.match(self.getSiteURLPattern(),url)
|
||||
# logger.debug("id:%s"%m.group('id'))
|
||||
# logger.debug("urltitle:%s"%m.group('urltitle'))
|
||||
|
||||
# get storyId from url
|
||||
self.story.setMetadata('storyId', m.group('id'))
|
||||
if m.group('urltitle'):
|
||||
self.urltitle = m.group('urltitle')
|
||||
# logger.debug("urltitle:%s"%self.urltitle)
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/series/' + self.story.getMetadata('storyId') + '/' + self.urltitle + '/')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
|
|
@ -94,44 +88,8 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
return "https://"+cls.getSiteDomain()+"/series/1234/storyname/"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return self._get_site_url_pattern()
|
||||
|
||||
## here so getSiteURLPattern and get_section_url(class method) can
|
||||
## both use it. Note adapter_fictionpresscom has one too.
|
||||
@classmethod
|
||||
def _get_site_url_pattern(cls):
|
||||
return re.escape("https://"+cls.getSiteDomain())+r"/(series|read)/(?P<id>\d+)([/-](?P<urltitle>[^/]+))?"
|
||||
|
||||
@classmethod
|
||||
def get_section_url(cls,url):
|
||||
## minimal URL used for section names in INI and reject list
|
||||
## for comparison
|
||||
# logger.debug("pre section--url:%s"%url)
|
||||
m = re.match(cls._get_site_url_pattern(),url)
|
||||
if m:
|
||||
url = "https://"+cls.getSiteDomain()\
|
||||
+"/series/"+m.group('id')+"/a-title/"
|
||||
# logger.debug("post-section url:%s"%url)
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def get_url_search(cls,url):
|
||||
regexp = super(getClass(), cls).get_url_search(url)
|
||||
regexp = re.sub(r"^(?P<keep>.*com/series/\d+/)(?P<urltitle>[^$]*)?",
|
||||
r"\g<keep>(.*)",regexp)
|
||||
logger.debug(regexp)
|
||||
return regexp
|
||||
|
||||
## normalized chapter URLs DO contain the story title now, but
|
||||
## normalized to current urltitle in case of title changes.
|
||||
def normalize_chapterurl(self,url):
|
||||
# https://www.scribblehub.com/read/862913-hp-the-arcane-thief-litrpg/chapter/1175961/
|
||||
# logger.debug("pre normal chapter--url:%s"%url)
|
||||
url = re.sub(r"https?://(?P<keep>www\.scribblehub\.com/read/\d+-).*(?P<chapter>/chapter/\d+/)",
|
||||
(r"https://\g<keep>"+self.urltitle+r"\g<chapter>"),url)
|
||||
# logger.debug("post normal chapter-url:%s"%url)
|
||||
return url
|
||||
|
||||
return re.escape("https://"+self.getSiteDomain())+r"/(series|read)/(?P<id>\d+)[/-](?P<title>[^/]+)?"
|
||||
|
||||
def post_request(self, url,
|
||||
parameters=None,
|
||||
usecache=True):
|
||||
|
|
@ -139,8 +97,8 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
return super(getClass(), self).post_request(url, parameters, usecache)
|
||||
except exceptions.HTTPErrorFFF as e:
|
||||
## this is a fix for the scribblehub ajax request sometimes returning
|
||||
# a 400 but only with flaresolverr. Have not been able to reproduce
|
||||
# in curl/firefox. See: https://github.com/JimmXinu/FanFicFare/pull/900
|
||||
# a 400 but only with flaresolverr. Have not been able to reproduce
|
||||
# in curl/firefox. See: https://github.com/JimmXinu/FanFicFare/pull/900
|
||||
logger.debug("HTTPErrorFFF/Scribblehub: " + str(e.status_code))
|
||||
if e.status_code == 400 and self.getConfig('use_flaresolverr_proxy'):
|
||||
return self.decode_data(e.data)
|
||||
|
|
@ -178,15 +136,11 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
|
||||
## Title
|
||||
pagetitle = soup.find('div',{'class':'fic_title'})
|
||||
self.story.setMetadata('title',stripHTML(pagetitle))
|
||||
|
||||
## <link rel="canonical" href="https://www.scribblehub.com/series/862913/hp-the-arcane-thief-litrpg/" />
|
||||
canonicalurl = soup.select_one('link[rel=canonical]')['href']
|
||||
self.set_story_idurl(canonicalurl)
|
||||
url = canonicalurl
|
||||
|
||||
# Find authorid and URL from main story page
|
||||
self.story.setMetadata('authorId',stripHTML(soup.find('span',{'class':'auth_name_fic'})))
|
||||
self.story.setMetadata('authorUrl',soup.find('div',{'class':'author'}).find('div',{'property':'author'}).find('span',{'property':'name'}).find('a').get('href'))
|
||||
|
|
@ -197,20 +151,33 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Get the contents list from scribblehub, iterate through and add to chapters
|
||||
# Can be fairly certain this will not 404 - we know the story id is valid
|
||||
contents_payload = {"action": "wi_getreleases_pagination",
|
||||
"pagenum": -1,
|
||||
"mypostid": self.story.getMetadata('storyId')}
|
||||
contents_payload = {"action": "wi_gettocchp",
|
||||
"strSID": self.story.getMetadata('storyId'),
|
||||
"strmypostid": 0,
|
||||
"strFic": "yes"}
|
||||
|
||||
# 14/12/22 - Looks like it should follow this format now (below), but still returns a 400
|
||||
# but not a 403. tested in browser getting rid of all other cookies to try and get a 400 and nopes.
|
||||
|
||||
# contents_payload = {"action": "wi_getreleases_pagination",
|
||||
# "pagenum": 1,
|
||||
# "mypostid": 421879}
|
||||
# contents_payload = "action=wi_getreleases_pagination&pagenum=1&mypostid=421879"
|
||||
|
||||
contents_data = self.post_request("https://www.scribblehub.com/wp-admin/admin-ajax.php", contents_payload)
|
||||
# logger.debug(contents_data)
|
||||
|
||||
contents_soup = self.make_soup(contents_data)
|
||||
|
||||
for toca in contents_soup.select('a.toc_a'):
|
||||
chapter_url = toca['href']
|
||||
chapter_name = stripHTML(toca)
|
||||
# logger.debug("Found Chapter: " + chapter_name + ", url: " + chapter_url)
|
||||
for i in range(1, int(contents_soup.find('ol',{'id':'ol_toc'}).get('count')) + 1):
|
||||
chapter_url = contents_soup.find('li',{'cnt':str(i)}).find('a').get('href')
|
||||
chapter_name = contents_soup.find('li',{'cnt':str(i)}).find('a').get('title')
|
||||
# logger.debug("Found Chapter " + str(i) + ", name: " + chapter_name + ", url: " + chapter_url)
|
||||
self.add_chapter(chapter_name, chapter_url)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
|
|
@ -227,13 +194,13 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Categories
|
||||
if soup.find('span',{'class': 'wi_fic_showtags_inner'}):
|
||||
categories = soup.find('span',{'class': 'wi_fic_showtags_inner'}).find_all('a')
|
||||
categories = soup.find('span',{'class': 'wi_fic_showtags_inner'}).findAll('a')
|
||||
for category in categories:
|
||||
self.story.addToList('category', stripHTML(category))
|
||||
|
||||
# Genres
|
||||
if soup.find('a',{'class': 'fic_genre'}):
|
||||
genres = soup.find_all('a',{'class': 'fic_genre'})
|
||||
genres = soup.findAll('a',{'class': 'fic_genre'})
|
||||
for genre in genres:
|
||||
self.story.addToList('genre', stripHTML(genre))
|
||||
|
||||
|
|
@ -245,7 +212,7 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Content Warnings
|
||||
if soup.find('ul',{'class': 'ul_rate_expand'}):
|
||||
warnings = soup.find('ul',{'class': 'ul_rate_expand'}).find_all('a')
|
||||
warnings = soup.find('ul',{'class': 'ul_rate_expand'}).findAll('a')
|
||||
for warn in warnings:
|
||||
self.story.addToList('warnings', stripHTML(warn))
|
||||
|
||||
|
|
@ -299,7 +266,7 @@ class ScribbleHubComAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata(metadata, stripHTML(row.find('td')))
|
||||
|
||||
if soup.find('table',{'class': 'table_pro_overview'}):
|
||||
stats_table = soup.find('table',{'class': 'table_pro_overview'}).find_all('tr')
|
||||
stats_table = soup.find('table',{'class': 'table_pro_overview'}).findAll('tr')
|
||||
for row in stats_table:
|
||||
find_stats_data("Total Views (All)", row, "views")
|
||||
find_stats_data("Word Count", row, "numWords")
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Find authorid and URL from... author url.
|
||||
# (fetch multiple authors)
|
||||
alist = soup.find_all('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('=')[1])
|
||||
self.story.addToList('authorUrl','https://'+self.host+'/fanfics/'+a['href'])
|
||||
|
|
@ -180,12 +180,12 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Reviews
|
||||
reviewdata = soup.find('div', {'id' : 'sort'})
|
||||
a = reviewdata.find_all('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
|
||||
a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
|
||||
self.story.setMetadata('reviews',stripHTML(a))
|
||||
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/fanfics/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -208,7 +208,7 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
self.setDescription(url,self.make_soup(summarydata))
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -220,13 +220,13 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
|
@ -235,7 +235,7 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -245,7 +245,7 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warningstext = [warning.string for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
|
|
@ -273,7 +273,7 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'https://'+self.host+'/fanfics/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
47
fanficfare/adapters/adapter_sinfuldreamscomwhisperedmuse.py
Normal file
47
fanficfare/adapters/adapter_sinfuldreamscomwhisperedmuse.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
from .base_efiction_adapter import BaseEfictionAdapter
|
||||
|
||||
class SinfulDreamsComWhisperedMuse(BaseEfictionAdapter):
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'sinful-dreams.com'
|
||||
|
||||
@classmethod
|
||||
def getPathToArchive(self):
|
||||
return '/whispered/muse'
|
||||
|
||||
@classmethod
|
||||
def getConfigSection(cls):
|
||||
"Overriden because [domain/path] section for multiple-adapter domain."
|
||||
return cls.getSiteDomain()+cls.getPathToArchive()
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(self):
|
||||
return 'snfldrms-wm'
|
||||
|
||||
@classmethod
|
||||
def getDateFormat(self):
|
||||
return "%m/%d/%Y"
|
||||
|
||||
def getClass():
|
||||
return SinfulDreamsComWhisperedMuse
|
||||
|
||||
|
|
@ -109,7 +109,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('title',stripHTML(titlea))
|
||||
|
||||
# Find the chapters (from soup, not authsoup):
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/siye/'+chapter['href'])
|
||||
|
||||
|
|
@ -121,7 +121,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
|
|||
metatable = soup.find('table',{'width':'95%'})
|
||||
|
||||
# Categories
|
||||
cat_as = metatable.find_all('a', href=re.compile(r'categories.php'))
|
||||
cat_as = metatable.findAll('a', href=re.compile(r'categories.php'))
|
||||
for cat_a in cat_as:
|
||||
self.story.addToList('category',stripHTML(cat_a))
|
||||
|
||||
|
|
@ -209,7 +209,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -76,43 +76,6 @@ class SpiritFanfictionComAdapter(BaseSiteAdapter):
|
|||
return 'spirit'
|
||||
|
||||
|
||||
# Login
|
||||
def needToLoginCheck(self, data):
|
||||
if 'nao-logado' in data or 'Acessar sua Conta' in data:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def performLogin(self, url, data):
|
||||
|
||||
params = {}
|
||||
params['Usuario'] = self.getConfig("username")
|
||||
params['Senha'] = self.getConfig("password")
|
||||
params['Login'] = 'Fazer Login'
|
||||
|
||||
login_url = 'https://' + self.getSiteDomain() + '/login'
|
||||
logger.info("Will now login to URL (%s) as (%s)" % (login_url,
|
||||
params['Usuario']))
|
||||
|
||||
login_page_html = self.get_request(login_url, usecache=False)
|
||||
login_page_soup = self.make_soup(login_page_html)
|
||||
|
||||
session_input = login_page_soup.find('input', {'name': "SessionHash"})
|
||||
params['SessionHash'] = session_input['value'] if session_input else ""
|
||||
|
||||
return_url_input = login_page_soup.find('input', {'name': 'ReturnUrl'})
|
||||
params['ReturnUrl'] = return_url_input['value'] if return_url_input else ""
|
||||
|
||||
response_html = self.post_request(login_url, params)
|
||||
|
||||
if 'nao-logado' in response_html or "Acessar sua Conta" in response_html:
|
||||
logger.info("Failed to login to URL %s as %s" % (login_url,
|
||||
params['Usuario']))
|
||||
raise exceptions.FailedToLogin(login_url,params['Usuario'])
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def getStoryId(self, url):
|
||||
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
|
|
@ -126,11 +89,9 @@ class SpiritFanfictionComAdapter(BaseSiteAdapter):
|
|||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
data = self.get_request(self.url)
|
||||
if self.needToLoginCheck(data):
|
||||
self.performLogin(self.url, data)
|
||||
data = self.get_request(self.url,usecache=False)
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = self.make_soup(data)
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
# Title
|
||||
title = soup.find('h1', {'class':'tituloPrincipal'})
|
||||
|
|
@ -138,7 +99,7 @@ class SpiritFanfictionComAdapter(BaseSiteAdapter):
|
|||
|
||||
# Authors
|
||||
# Find authorid and URL
|
||||
authors = (title.find_next('div', {'class':'left'})).find_all('span', {'class':'usuario'})
|
||||
authors = soup.findAll('span', {'class':'usuario'})
|
||||
|
||||
for author in authors:
|
||||
self.story.addToList('authorId', author.find('a')['href'].split('/')[-1])
|
||||
|
|
@ -153,10 +114,10 @@ class SpiritFanfictionComAdapter(BaseSiteAdapter):
|
|||
newestChapter = None
|
||||
self.newestChapterNum = None # save for comparing during update.
|
||||
# Find the chapters:
|
||||
chapters = soup.find_all('table', {'class':'listagemCapitulos espacamentoTop'})
|
||||
chapters = soup.findAll('table', {'class':'listagemCapitulos espacamentoTop'})
|
||||
for chapter in chapters:
|
||||
|
||||
for row in chapter.find_all('tr', {'class': 'listagem-textoBg1'}): # Find each row with chapter info
|
||||
for row in chapter.findAll('tr', {'class': 'listagem-textoBg1'}): # Find each row with chapter info
|
||||
a = row.find('a') # Chapter link
|
||||
|
||||
# Datetime
|
||||
|
|
@ -383,11 +344,3 @@ class SpiritFanfictionComAdapter(BaseSiteAdapter):
|
|||
element.string = decoded_email
|
||||
return unicode(html_text)
|
||||
|
||||
|
||||
def before_get_urls_from_page(self,url,normalize):
|
||||
|
||||
if self.getConfig("username"):
|
||||
|
||||
data = self.get_request(url)
|
||||
if self.needToLoginCheck(data):
|
||||
self.performLogin(url, data)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2019 FanFicFare team
|
||||
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -16,20 +16,23 @@
|
|||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .adapter_test1 import TestSiteAdapter
|
||||
# Software: eFiction
|
||||
from .base_efiction_adapter import BaseEfictionAdapter
|
||||
|
||||
class Test4SiteAdapter(TestSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
TestSiteAdapter.__init__(self, config, url)
|
||||
class StarskyHutchArchiveNetSiteAdapter(BaseEfictionAdapter):
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'test4.com'
|
||||
return 'www.starskyhutcharchive.net'
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(self):
|
||||
return 'shan'
|
||||
|
||||
@classmethod
|
||||
def getDateFormat(self):
|
||||
return "%m/%d/%Y"
|
||||
|
||||
def getClass():
|
||||
return Test4SiteAdapter
|
||||
|
||||
return StarskyHutchArchiveNetSiteAdapter
|
||||
|
|
@ -93,7 +93,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find the chapters: chapterview.asp?sid=7000&cid=30919
|
||||
chapters=soup.find_all('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+r"&cid=\d+$"))
|
||||
chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+r"&cid=\d+$"))
|
||||
if len(chapters)==1:
|
||||
self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href'])
|
||||
else:
|
||||
|
|
@ -109,14 +109,14 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
|
|||
|
||||
# no convenient way to get word count
|
||||
|
||||
for td in asoup.find_all('td', {'colspan' : '3'}):
|
||||
for td in asoup.findAll('td', {'colspan' : '3'}):
|
||||
if td.find('a', href=re.compile(r'chapterlistview.asp\?SID='+self.story.getMetadata('storyId'))) != None:
|
||||
break
|
||||
td=td.nextSibling.nextSibling
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(td).split(': ')[1], self.dateformat))
|
||||
try:
|
||||
tr=td.parent.nextSibling.nextSibling.nextSibling.nextSibling
|
||||
td=tr.find_all('td')
|
||||
td=tr.findAll('td')
|
||||
self.story.setMetadata('rating', td[0].string.split(': ')[1])
|
||||
self.story.setMetadata('status', td[2].string.split(': ')[1])
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(td[4]).split(': ')[1], self.dateformat))
|
||||
|
|
|
|||
|
|
@ -147,21 +147,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
postAction,
|
||||
'','',''))
|
||||
data = self.post_request(postUrl,params,usecache=False)
|
||||
# logger.debug(data)
|
||||
while '<h2>Enter TOTP Code:</h2>' in data:
|
||||
if self.totp:
|
||||
logger.debug("Trying to TOTP with %s code."%self.totp)
|
||||
params = {}
|
||||
params['cmd'] = 'finishTotpVerification'
|
||||
# google auth app at least shows "123 123", but site expects
|
||||
# "123123". Remove space if user enters it.
|
||||
params['totp_code'] = self.totp.replace(' ','')
|
||||
params['action'] = "continue"
|
||||
data = self.post_request(postUrl,params,usecache=False)
|
||||
# logger.debug(data)
|
||||
self.totp = None
|
||||
else:
|
||||
raise exceptions.NeedTimedOneTimePassword(url)
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
|
|
@ -173,10 +158,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
## Some stories give 404 if not logged in now. See #1185
|
||||
if self.getConfig("always_login"):
|
||||
self.performLogin(self.url)
|
||||
|
||||
## Hit story URL to check for changed title part -- if the
|
||||
## title has changed or (more likely?) the ID number has
|
||||
## been reassigned to a different title, this will 404
|
||||
|
|
@ -188,7 +169,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
if e.status_code in (401, 403, 410):
|
||||
data = 'Log In' # to trip needToLoginCheck
|
||||
elif e.status_code == 404:
|
||||
raise exceptions.FailedToDownload("Page Not Found - always_login needed? (%s)" % url)
|
||||
raise exceptions.FailedToDownload("Page Not Found - Story ID Reused? (%s)" % url)
|
||||
else:
|
||||
raise e
|
||||
if self.needToLoginCheck(data):
|
||||
|
|
@ -196,24 +177,13 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
self.performLogin(url)
|
||||
data = self.get_request(url,usecache=False)
|
||||
|
||||
## SOL adds intermediate page to remind users to renew at 3-30 days before expiration - this breaks the soup 'a' search below
|
||||
if "Your premier membership is going to expire" in data:
|
||||
soup = self.make_soup(data)
|
||||
expire = soup.find(string=re.compile("Your premier membership is going to expire"))
|
||||
remindurl=(soup.find(href=re.compile("later.php"))).get('href')
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: "+expire+"\n"+"Renew or reduce expiration warning time in account setting\n"+remindurl)
|
||||
|
||||
## Premium account might redirect to a chapter, while regular
|
||||
## account doesn't redirect to the URL with embedded /story-title
|
||||
## So pull url from <a href="/s/000/story-title" rel="bookmark">
|
||||
## regardless.
|
||||
soup = self.make_soup(data)
|
||||
a = soup.find('a',rel="bookmark")
|
||||
if a:
|
||||
url = 'https://'+self.host+a['href']
|
||||
else:
|
||||
# Contest entries do not have bookmark HREF
|
||||
logger.info("No Bookmark HREF, using URL="+url)
|
||||
url = 'https://'+self.host+a['href']
|
||||
|
||||
## Premium has "?ind=1" to force index.
|
||||
## May not be needed w/o premium
|
||||
|
|
@ -232,12 +202,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
|
||||
elif "Error! Daily Limit Reached" in data or "Sorry! You have reached your daily limit of" in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! Daily Limit Reached")
|
||||
elif "by (Hidden)" in data:
|
||||
#Contest entries have author set to "(Hidden)" which breaks author lookups below
|
||||
logger.info("Contest entry, setting authorId=(Hidden)")
|
||||
self.story.addToList('authorId',"(Hidden)")
|
||||
logger.info("Contest entry, setting author=(Hidden)")
|
||||
self.story.addToList('author',"(Hidden)")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# logger.debug(data)
|
||||
|
|
@ -246,23 +210,26 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
a = soup.find('h1')
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
authfrom = soup.find('footer')
|
||||
alist = authfrom.find_all('a', {'rel' : 'author'})
|
||||
if alist:
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('/')[2])
|
||||
self.story.addToList('authorUrl','https://'+self.host+a['href'])
|
||||
## both 's Page and ’s Page
|
||||
self.story.addToList('author',re.sub(r".s Page$","",stripHTML(a)))
|
||||
else:
|
||||
logger.info("AuthorList empty. Contest entry?")
|
||||
# Find authorid and URL from... author url. Sometimes in top,
|
||||
# other times in footer.
|
||||
authfrom = soup.find('div', {'id':'top-header'})
|
||||
if authfrom is None or 'author' not in str(authfrom):
|
||||
authfrom = soup.find('footer')
|
||||
alist = authfrom.findAll('a', {'rel' : 'author'})
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('/')[2])
|
||||
self.story.addToList('authorUrl','https://'+self.host+a['href'])
|
||||
## both 's Page and ’s Page
|
||||
self.story.addToList('author',re.sub(r".s Page$","",stripHTML(a)))
|
||||
|
||||
# The rest of the metadata is within the article tag.
|
||||
soup = soup.find('article')
|
||||
|
||||
# Find the chapters:
|
||||
# If multiple chapters, they are in "index-list" div.
|
||||
# <a href="/s/00001/This-is-a-test/1">Chapter 1</a>
|
||||
# <a href="/n/00001/This-is-a-test/1">Chapter 1</a>
|
||||
chapters = soup.select('div#index-list a[href*="/s/"],div#index-list a[href*="/n/"]')
|
||||
# logger.debug(chapters)
|
||||
logger.debug(chapters)
|
||||
if len(chapters) != 0:
|
||||
logger.debug("Number of chapters: {0}".format(len(chapters)))
|
||||
for chapter in chapters:
|
||||
|
|
@ -271,15 +238,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
else:
|
||||
self.add_chapter(self.story.getMetadata('title'),self.story.getMetadata('storyUrl'))
|
||||
|
||||
# The rest of the metadata is within the article tag.
|
||||
soup = soup.find('article')
|
||||
|
||||
if self.story.getList('authorUrl'):
|
||||
self.getStoryMetadataFromAuthorPage()
|
||||
else:
|
||||
logger.info("No authorurl found, setting to homepage. Could be contest story...")
|
||||
self.story.setMetadata('authorUrl','https://' + self.getSiteDomain() + '/')
|
||||
|
||||
self.getStoryMetadataFromAuthorPage()
|
||||
|
||||
# Some books have a cover in the index page.
|
||||
# Samples are:
|
||||
|
|
@ -323,7 +283,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
self.has_universes = False
|
||||
|
||||
title_cell = story_row.find('td', {'class' : 'lc2'})
|
||||
for cat in title_cell.find_all('div', {'class' : 'typediv'}):
|
||||
for cat in title_cell.findAll('div', {'class' : 'typediv'}):
|
||||
self.story.addToList('genre',cat.text)
|
||||
|
||||
# in lieu of word count.
|
||||
|
|
@ -400,16 +360,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
series_name = stripHTML(series_soup.find('h1', {'id' : 'ptitle'}))
|
||||
series_name = re.sub(r' . a (series by|collection from).*$','',series_name)
|
||||
# logger.debug("Series name: '%s'" % series_name)
|
||||
if i == 0:
|
||||
# find number in series from series page--not
|
||||
# included in story page anymore.
|
||||
# ... <a id="t20130r"></a>2 ...
|
||||
seriesi = series_soup.select_one("a[id='t"+self.story.getMetadata('storyId')+"r']").parent
|
||||
# logger.debug(seriesi)
|
||||
try:
|
||||
i = int(stripHTML(seriesi))
|
||||
except:
|
||||
logger.debug("Failed to convert series number(%s)"%seriesi)
|
||||
self.setSeries(series_name, i)
|
||||
# Check if series is in a universe
|
||||
if self.has_universes:
|
||||
|
|
@ -417,7 +367,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
universes_soup = self.make_soup(self.get_request(universe_url) )
|
||||
# logger.debug("Universe url='{0}'".format(universe_url))
|
||||
if universes_soup:
|
||||
universes = universes_soup.find_all('div', {'class' : 'ser-box'})
|
||||
universes = universes_soup.findAll('div', {'class' : 'ser-box'})
|
||||
# logger.debug("Number of Universes: %d" % len(universes))
|
||||
for universe in universes:
|
||||
# logger.debug("universe.find('a')={0}".format(universe.find('a')))
|
||||
|
|
@ -512,7 +462,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
return value
|
||||
|
||||
def parseOtherAttributes(self, other_attribute_element):
|
||||
for b in other_attribute_element.find_all('b'):
|
||||
for b in other_attribute_element.findAll('b'):
|
||||
#logger.debug('Getting metadata: "%s"' % b)
|
||||
label = b.text
|
||||
if label in ['Posted:', 'Concluded:', 'Updated:']:
|
||||
|
|
@ -596,7 +546,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
srtag = soup.find('div', id='sr')
|
||||
|
||||
if srtag != None:
|
||||
# logger.debug('Getting more chapter text for: %s' % url)
|
||||
logger.debug('Getting more chapter text for: %s' % url)
|
||||
moretext = self.getMoreText(html)
|
||||
if moretext != None:
|
||||
moresoup = self.make_soup(moretext)
|
||||
|
|
@ -611,7 +561,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
|
||||
if pager != None:
|
||||
|
||||
urls=pager.find_all('a')
|
||||
urls=pager.findAll('a')
|
||||
urls=urls[:len(urls)-1]
|
||||
# logger.debug("pager urls:%s"%urls)
|
||||
pager.extract()
|
||||
|
|
@ -638,13 +588,11 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
|
||||
chapter_title = None
|
||||
if self.getConfig('inject_chapter_title'):
|
||||
if self.num_chapters() > 1:
|
||||
cttag = pagetag.find('h2')
|
||||
else:
|
||||
## single chapter stories formatted a little differently.
|
||||
cttag = pagetag.find('h1')
|
||||
if cttag:
|
||||
chapter_title = cttag.extract()
|
||||
h2tag = pagetag.find('h2')
|
||||
if h2tag:
|
||||
# I'm seeing an h1 now, but it's not logged in?
|
||||
# Something's broken...
|
||||
chapter_title = h2tag.extract()
|
||||
|
||||
# Strip te header section
|
||||
tag = pagetag.find('header')
|
||||
|
|
@ -667,7 +615,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
# putting a 'conTag' at the *top* now, too. So this
|
||||
# was nuking every page but the first and last. Now
|
||||
# only if 'Continues'
|
||||
for contag in pagetag.find_all('span', {'class' : 'conTag'}):
|
||||
for contag in pagetag.findAll('span', {'class' : 'conTag'}):
|
||||
# remove everything after continues...
|
||||
if 'Continuation' in contag.text:
|
||||
tag = contag
|
||||
|
|
@ -696,7 +644,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
|
||||
# If it is a chapter, there are dates at the start for when it was posted or modified. These plus
|
||||
# everything before them can be discarded.
|
||||
postedDates = pagetag.find_all('div', {'class' : 'date'})
|
||||
postedDates = pagetag.findAll('div', {'class' : 'date'})
|
||||
# logger.debug(postedDates)
|
||||
if postedDates:
|
||||
a = postedDates[0].previousSibling
|
||||
|
|
@ -705,7 +653,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
b = a.previousSibling
|
||||
a.extract()
|
||||
a = b
|
||||
for a in pagetag.find_all('div', {'class' : 'date'}):
|
||||
for a in pagetag.findAll('div', {'class' : 'date'}):
|
||||
a.extract()
|
||||
|
||||
# Kill the vote form and everything after it.
|
||||
|
|
@ -726,5 +674,4 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
|
|||
# inject_chapter_title
|
||||
if chapter_title:
|
||||
chapter_title.name='h3'
|
||||
chapter_title['class']='inject_chapter_title'
|
||||
pagetag.insert(0,chapter_title)
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .adapter_storiesonlinenet import StoriesOnlineNetAdapter
|
||||
|
||||
def getClass():
|
||||
return StoryRoomComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class StoryRoomComAdapter(StoriesOnlineNetAdapter):
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(cls):
|
||||
return 'stryrm'
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'storyroom.com'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['finestories.com',cls.getSiteDomain()]
|
||||
|
||||
@classmethod
|
||||
def getConfigSections(cls):
|
||||
"Only needs to be overriden if has additional ini sections."
|
||||
return ['finestories.com',cls.getSiteDomain()]
|
||||
|
||||
@classmethod
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://("+r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()])+r")/(?P<path>s|n|library)/(storyInfo.php\?id=)?(?P<id>\d+)(?P<chapter>:\d+)?(?P<title>/.+)?((;\d+)?$|(:i)?$)?"
|
||||
144
fanficfare/adapters/adapter_swiorgru.py
Normal file
144
fanficfare/adapters/adapter_swiorgru.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
|
||||
def getClass():
|
||||
return SwiOrgRuAdapter
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class SwiOrgRuAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
storyId = self.parsedUrl.path.split('/',)[3]
|
||||
self.story.setMetadata('storyId', storyId)
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/mlp-fim/story/'+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','swiorgru')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%Y.%m.%d"
|
||||
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
return 'www.swi.org.ru'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://" + cls.getSiteDomain() + "/mlp-fim/story/11341/ http://" + cls.getSiteDomain() + "/mlp-fim/story/11341/chapter1.html"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"http://" + re.escape(self.getSiteDomain() + "/mlp-fim/story/")+r"\d+"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
url=self.url
|
||||
logger.debug("URL: "+url)
|
||||
data = self.get_request(url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
title = soup.find('h1')
|
||||
for tag in title.findAll('sup'):
|
||||
tag.extract()
|
||||
|
||||
self.story.setMetadata('title', stripHTML(title.text))
|
||||
logger.debug("Title: (%s)"%self.story.getMetadata('title'))
|
||||
|
||||
author_title = soup.find('strong', string = re.compile(u"Автор: "))
|
||||
if author_title == None:
|
||||
raise exceptions.FailedToDownload("Error downloading page: %s! Missing required author_title element!" % url)
|
||||
|
||||
author = author_title.next_sibling
|
||||
|
||||
self.story.setMetadata('authorId', author.text) # Author's name is unique
|
||||
self.story.setMetadata('authorUrl','http://'+self.host + author['href'])
|
||||
self.story.setMetadata('author', author.text)
|
||||
logger.debug("Author: (%s)"%self.story.getMetadata('author'))
|
||||
|
||||
date_pub = soup.find('em', string = re.compile(r'\d{4}.\d{2}.\d{2}'))
|
||||
if not date_pub == None:
|
||||
self.story.setMetadata('datePublished', makeDate(date_pub.text, self.dateformat))
|
||||
|
||||
rating_label = soup.find('strong', string = re.compile(u"рейтинг:"))
|
||||
if not rating_label == None:
|
||||
rating = rating_label.next_sibling.next_sibling
|
||||
self.story.setMetadata('rating', stripHTML(rating))
|
||||
|
||||
if not self.is_adult or self.getConfig("is_adult"):
|
||||
if "NC-18" in rating:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
characters = soup.findAll('img', src=re.compile(r"/mlp-fim/img/chars/\d+.png"))
|
||||
logger.debug("numCharacters: (%s)"%str(len(characters)))
|
||||
|
||||
for x in range(0,len(characters)):
|
||||
character=characters[x]
|
||||
self.story.addToList('characters', character['title'])
|
||||
|
||||
if soup.find('font', color = r"green", string = u"завершен"):
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
categories_label = soup.find('strong', string = u"категории:")
|
||||
if not categories_label == None:
|
||||
categories_element = categories_label.next_sibling.next_sibling
|
||||
categories = re.findall(r'"(.+?)"', categories_element.text)
|
||||
for x in range(0, len(categories)):
|
||||
category=categories[x]
|
||||
self.story.addToList('category', category)
|
||||
|
||||
chapters_header = soup.find('h2', string = re.compile(u"Главы:"))
|
||||
if chapters_header==None:
|
||||
raise exceptions.FailedToDownload("Error downloading page: %s! Missing required chapters_header element!" % url)
|
||||
|
||||
chapters_table = chapters_header.next_sibling.next_sibling
|
||||
|
||||
self.story.setMetadata('language','Russian')
|
||||
|
||||
chapters=chapters_table.findAll('a', href=re.compile(r'/mlp-fim/story/'+self.story.getMetadata('storyId')+r"/chapter\d+"))
|
||||
self.story.setMetadata('numChapters', len(chapters))
|
||||
logger.debug("numChapters: (%s)"%str(self.story.getMetadata('numChapters')))
|
||||
|
||||
for x in range(0,len(chapters)):
|
||||
chapter=chapters[x]
|
||||
churl='http://'+self.host+chapter['href']
|
||||
self.add_chapter(chapter,churl)
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
chapter = soup.find('div', {'id' : 'content'})
|
||||
|
||||
chapter_header = chapter.find('h1', id = re.compile("chapter"))
|
||||
if not chapter_header == None:
|
||||
chapter_header.decompose()
|
||||
|
||||
if chapter == None:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,chapter)
|
||||
|
|
@ -37,9 +37,9 @@ def getClass():
|
|||
|
||||
def getEntry(soup, *args):
|
||||
for arg in args:
|
||||
target = soup.find('dt', string=arg)
|
||||
target = soup.find('th', string=arg)
|
||||
if target is not None:
|
||||
return target.findNext('dd')
|
||||
return target.findNext('td')
|
||||
return None
|
||||
|
||||
class SyosetuComAdapter(BaseSiteAdapter):
|
||||
|
|
@ -208,8 +208,7 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
updateElement = getEntry(infoSoup,
|
||||
'最終部分掲載日', # last part published (complete)
|
||||
'最新部分掲載日', # latest part published
|
||||
'最終更新日', # last update (complete)
|
||||
'最新掲載日' # last update
|
||||
'最終更新日' # last update
|
||||
)
|
||||
if updateElement is not None:
|
||||
updated = makeDate(updateElement.text.strip(),
|
||||
|
|
@ -226,10 +225,10 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
seriesUrl = series.find('a')['href']
|
||||
|
||||
seriesSoup = self.make_soup(self.get_request(seriesUrl))
|
||||
alist = seriesSoup.select('.p-series-novellist .p-series-novellist__title a')
|
||||
alist = seriesSoup.select('.serieslist .title a')
|
||||
i = 1
|
||||
for a in alist:
|
||||
if self.storyId in a['href']:
|
||||
if a['href'] == '/' + self.storyId + '/':
|
||||
self.setSeries(seriesName, i)
|
||||
self.story.setMetadata('seriesUrl', seriesUrl)
|
||||
break
|
||||
|
|
@ -245,16 +244,18 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
|
||||
# Status and Chapter count
|
||||
|
||||
noveltype = infoSoup.find('span', {'class':'p-infotop-type__type'})
|
||||
noveltype = (infoSoup.find(id='noveltype')
|
||||
or infoSoup.find(id='noveltype_notend'))
|
||||
if noveltype.text.strip() == '短編':
|
||||
numChapters = 1
|
||||
oneshot = True
|
||||
completed = True
|
||||
else:
|
||||
# '全1,292エピソード\n'
|
||||
numChapters = int(re.sub(r'[^\d]', '', infoSoup.find('span', {'class':'p-infotop-type__allep'}).text.strip()))
|
||||
numChapters = int(re.sub(r'[^\d]', '', noveltype.next_sibling.strip()))
|
||||
oneshot = False
|
||||
completed = True if noveltype == '完結済' else False
|
||||
self.story.setMetadata('numChapters', numChapters)
|
||||
self.story.setMetadata('status', 'Completed' if completed else 'In-Progress')
|
||||
|
||||
# Keywords
|
||||
|
|
@ -262,7 +263,12 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
flags = []
|
||||
# not sure what it looks like if a work has no tags
|
||||
tagsElement = getEntry(infoSoup, 'キーワード')
|
||||
for tag in tagsElement.text.split():
|
||||
if tagsElement.find('span'):
|
||||
# R15, ボーイズラブ, ガールズラブ, 残酷な描写あり, 異世界転生, 異世界転移
|
||||
flags = tagsElement.find('span').text.split()
|
||||
for flag in flags:
|
||||
self.story.addToList('warningtags', flag)
|
||||
for tag in tagsElement.contents[-1].split():
|
||||
self.story.addToList('freeformtags', tag)
|
||||
|
||||
# Rating, Genre, and Imprint
|
||||
|
|
@ -325,10 +331,9 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
if self.getConfig("always_login"):
|
||||
if infoSoup.find('div', {'data-remodal-id':'setting_bookmark'}) is None:
|
||||
self.story.setMetadata('bookmarked', False)
|
||||
self.story.setMetadata('subscribed', False)
|
||||
else:
|
||||
self.story.setMetadata('bookmarked', True)
|
||||
modal = infoSoup.find('div', {'data-remodal-id':'setting_bookmark'})
|
||||
modal = infoSoup.find('div', {'class':'favnovelmain_update'})
|
||||
|
||||
# bookmark category name
|
||||
bookmarkCategory = modal.find('option', {
|
||||
|
|
@ -363,22 +368,18 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
tocSoups = []
|
||||
for n in range(1, int(math.ceil(numChapters/100.0))+1):
|
||||
tocPage = self.make_soup(self.get_request(self.url + '?p=%s' % n))
|
||||
tocSoups.append(tocPage.find('div',{'class':'p-eplist'}))
|
||||
tocSoups.append(tocPage.find('div',{'class':'index_box'}))
|
||||
|
||||
sectionTitle = None
|
||||
newSection = False
|
||||
for tocSoup in tocSoups:
|
||||
for child in tocSoup.findChildren(recursive=False):
|
||||
if 'p-eplist__chapter-title' in child['class']:
|
||||
if 'chapter_title' in child['class']:
|
||||
sectionTitle = child.text.strip()
|
||||
newSection = True
|
||||
elif 'p-eplist__sublist' in child['class']:
|
||||
elif 'novel_sublist2' in child['class']:
|
||||
epTitle = child.find('a').text.strip()
|
||||
updateElement = child.find('div', {'class':'p-eplist__update'})
|
||||
if updateElement.find('span',{'class':'p-eplist__favep'}) is not None:
|
||||
# a bookmarked story has some extra text added
|
||||
updateElement.next_element.extract()
|
||||
updateElement.next_element.extract()
|
||||
updateElement = child.find('dt', {'class':'long_update'})
|
||||
epPublished = updateElement.next_element.strip()
|
||||
epUpdated = ''
|
||||
if updateElement.find('span') is not None:
|
||||
|
|
@ -405,20 +406,22 @@ class SyosetuComAdapter(BaseSiteAdapter):
|
|||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
divs = soup.find_all('div',{'class':'p-novel__text'})
|
||||
text_divs = []
|
||||
for div in divs:
|
||||
if 'p-novel__text--preface' in div['class']:
|
||||
div['class'] = 'novel_p'
|
||||
elif 'p-novel__text--afterword' in div['class']:
|
||||
div['class'] = 'novel_a'
|
||||
else:
|
||||
div['class'] = 'novel_honbun'
|
||||
if self.getConfig('include_author_notes', True) or div['class'] == 'novel_honbun':
|
||||
if self.getConfig('include_author_notes', True):
|
||||
divs = soup.find_all('div', id=re.compile(r'^novel_(p|honbun|a)$'))
|
||||
if divs is None:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
text_divs = []
|
||||
for div in divs:
|
||||
div['class'].append(div['id'])
|
||||
div.attrs.pop('id')
|
||||
text_divs.append(unicode(div))
|
||||
if not text_divs:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
soup = self.make_soup(' '.join(text_divs))
|
||||
soup = self.make_soup(' '.join(text_divs))
|
||||
else:
|
||||
soup = soup.find('div', id='novel_honbun')
|
||||
soup['class'].append(soup['id'])
|
||||
soup.attrs.pop('id')
|
||||
if soup is None:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url, soup)
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ class TenhawkPresentsSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -143,7 +143,7 @@ class TenhawkPresentsSiteAdapter(BaseSiteAdapter):
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -164,19 +164,19 @@ class TenhawkPresentsSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -203,7 +203,7 @@ class TenhawkPresentsSiteAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -29,13 +29,6 @@ from ..six import ensure_text
|
|||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
try: # just a way to switch between CLI and PI
|
||||
## webbrowser.open doesn't work on some linux flavors.
|
||||
## piggyback Calibre's version.
|
||||
from calibre.gui2 import safe_open_url as open_url
|
||||
except :
|
||||
from webbrowser import open as open_url
|
||||
|
||||
class TestSiteAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
|
|
@ -128,7 +121,7 @@ class TestSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',prefix+'Test Author aa')
|
||||
self.setDescription(self.url,u'<div>Description '+self.crazystring+u''' Done
|
||||
<p>
|
||||
Some more longer description. "I suck at summaries!" "Better than it sounds!" <span>A span!</span> "My first fic"
|
||||
Some more longer description. "I suck at summaries!" "Better than it sounds!" "My first fic"
|
||||
</div>''')
|
||||
self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
|
||||
if idstr == '669':
|
||||
|
|
@ -136,9 +129,6 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
else:
|
||||
self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
|
||||
|
||||
if idstr == '675' and self.totp != "123321" :
|
||||
raise exceptions.NeedTimedOneTimePassword(self.url)
|
||||
|
||||
if idstr != '674':
|
||||
self.story.setMetadata('numWords','123456')
|
||||
|
||||
|
|
@ -149,20 +139,20 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
|
||||
# greater than 10, no language or series.
|
||||
if idnum < 10:
|
||||
## non-English was changing series sort order which
|
||||
## confuses me more often than I test other langs.
|
||||
# langs = {
|
||||
# 0:"English",
|
||||
# 1:"Russian",
|
||||
# 2:"French",
|
||||
# 3:"German",
|
||||
# }
|
||||
# self.story.setMetadata('language',langs[idnum%len(langs)])
|
||||
self.setSeries('The Great Test',idnum)
|
||||
self.story.setMetadata('seriesUrl','http://'+self.getSiteDomain()+'/seriesid=1')
|
||||
elif idnum < 20:
|
||||
self.setSeries('魔法少女まどか★マギカ',idnum)
|
||||
self.story.setMetadata('seriesUrl','http://'+self.getSiteDomain()+'/seriesid=1')
|
||||
elif idnum < 30:
|
||||
langs = {
|
||||
0:"English",
|
||||
1:"Russian",
|
||||
2:"French",
|
||||
3:"German",
|
||||
}
|
||||
self.story.setMetadata('language',langs[idnum%len(langs)])
|
||||
|
||||
if idnum == 0:
|
||||
self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
|
||||
self.story.setMetadata('seriesUrl','http://'+self.getSiteDomain()+'/seriesid=0')
|
||||
|
|
@ -328,18 +318,13 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
rt = random.uniform(t*0.5, t*1.5)
|
||||
logger.debug("random sleep(%0.2f-%0.2f):%0.2f"%(t*0.5, t*1.5,rt))
|
||||
time.sleep(rt)
|
||||
# open_url("https://echo.free.beeceptor.com/%s.%s"%(self.story.getMetadata('siteabbrev'),
|
||||
# self.story.getMetadata('storyId')))
|
||||
|
||||
if "chapter=1" in url :
|
||||
text=u'''
|
||||
<div>
|
||||
<h3>Prologue</h3>
|
||||
<div class='leadpara'>
|
||||
<p>This is a fake adapter for testing purposes. Different sid's will give different errors:</p>
|
||||
<p>sid>=1000 will use custom test story data from your configuration(personal.ini)</p>
|
||||
</div>
|
||||
<div class='failids'>
|
||||
<p>Hard coded ids:</p>
|
||||
<p>http://test1.com?sid=664 - Crazy string title</p>
|
||||
<p>http://test1.com?sid=665, 711-720 - raises AdultCheckRequired</p>
|
||||
|
|
@ -356,7 +341,6 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
<p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p>
|
||||
<p>Odd sid's will be In-Progress, evens complete. sid<10 will be assigned one of four languages and included in a series.</p>
|
||||
</div>
|
||||
</div>
|
||||
'''
|
||||
elif self.story.getMetadata('storyId') == '0':
|
||||
text=u'''<div>
|
||||
|
|
@ -370,7 +354,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
<br />
|
||||
</div>
|
||||
'''
|
||||
elif self.story.getMetadata('storyId') == '667' and ("chapter=2" in url or "chapter=3" in url or "chapter=4" in url):
|
||||
elif self.story.getMetadata('storyId') == '667' and "chapter=2" in url:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
|
||||
elif self.getSiteDomain() not in url:
|
||||
## for chapter_urls setting.
|
||||
|
|
@ -415,13 +399,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
|
|||
|
||||
else:
|
||||
if self.story.getMetadata('storyId') == '92':
|
||||
imgtext='''
|
||||
<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl"><img src="http://i.imgur.com/bo8eD.png"></a>
|
||||
<style>
|
||||
.loremipsum { background-image: url("https://picsum.photos/2000/1500") }
|
||||
</style>
|
||||
<p style="background-image: url('https://picsum.photos/20/10')">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
|
||||
'''
|
||||
imgtext='<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim & Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a>'
|
||||
else:
|
||||
imgtext='img goes here when sid=92'
|
||||
text=u'''
|
||||
|
|
@ -442,9 +420,7 @@ Don't—e;ver—d;o—that—a;gain, 法 é
|
|||
<hr>
|
||||
horizontal rules
|
||||
<hr size=1 noshade>
|
||||
<div class="loremipsum">
|
||||
<p>"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
|
||||
</div>
|
||||
<br>
|
||||
<br>
|
||||
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br/>
|
||||
|
|
@ -456,6 +432,7 @@ Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
|
|||
<br/> <br/>
|
||||
<br/>
|
||||
"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.<br>
|
||||
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
|
||||
</div>
|
||||
'''%imgtext
|
||||
soup = self.make_soup(text)
|
||||
|
|
@ -491,7 +468,6 @@ Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
|
|||
desc = '<div><p>The Great Test Series of '+self.getSiteDomain()+'!</p><p>Now with two lines!</p></div>'
|
||||
return {'name':'The Great Test',
|
||||
'desc':desc,
|
||||
'status':'AStatus',
|
||||
'urllist':['http://'+self.getSiteDomain()+'?sid=1',
|
||||
'http://'+self.getSiteDomain()+'?sid=2',
|
||||
'http://'+self.getSiteDomain()+'?sid=3',
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2019 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .adapter_test1 import TestSiteAdapter
|
||||
|
||||
class Test2SiteAdapter(TestSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
TestSiteAdapter.__init__(self, config, url)
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'test2.com'
|
||||
|
||||
def getClass():
|
||||
return Test2SiteAdapter
|
||||
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2019 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .adapter_test1 import TestSiteAdapter
|
||||
|
||||
class Test3SiteAdapter(TestSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
TestSiteAdapter.__init__(self, config, url)
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'test3.com'
|
||||
|
||||
def getClass():
|
||||
return Test3SiteAdapter
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host + self.section + chapter['href']+addurl)
|
||||
|
||||
|
|
@ -186,7 +186,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
|
|||
# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.text
|
||||
|
|
@ -207,22 +207,22 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
|
|||
|
|
@ -45,9 +45,6 @@ class TheSietchComAdapter(BaseXenForo2ForumAdapter):
|
|||
# in case it needs more than just site/
|
||||
return '/index.php?'
|
||||
|
||||
def loginFormMarker(self):
|
||||
return 'href="/index.php?login/"'
|
||||
|
||||
def make_reader_url(self,tmcat_num,reader_page_num):
|
||||
# https://www.the-sietch.com/index.php?threads/shattered-sphere-the-arcadian-free-march.3243/reader/page-2
|
||||
# discard tmcat_num -- the-sietch.com doesn't have multiple
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ class TouchFluffyTailAdapter(BaseSiteAdapter):
|
|||
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
self.story.setMetadata('numChapters',1)
|
||||
|
||||
avrrate = body.find_all('footer', class_='entry-meta')[1].find('em').span.find_all('strong')
|
||||
averrating = avrrate[1].text
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue