mirror of
https://github.com/JimmXinu/FanFicFare.git
synced 2026-05-09 05:21:13 +02:00
Compare commits
No commits in common. "main" and "v4.14.3" have entirely different histories.
244 changed files with 36523 additions and 37506 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -20,9 +20,6 @@
|
|||
# pycharm project specific settings files
|
||||
.idea
|
||||
|
||||
# vscode project specific settings file
|
||||
.vscode
|
||||
|
||||
cleanup.sh
|
||||
FanFictionDownLoader.zip
|
||||
*.epub
|
||||
|
|
|
|||
15
README.md
15
README.md
|
|
@ -52,15 +52,22 @@ Test versions are available at:
|
|||
|
||||
- The [test plugin] is posted at MobileRead.
|
||||
- The test version of CLI for pip install is uploaded to the testpypi repository and can be installed with:
|
||||
```
|
||||
pip install --extra-index-url https://test.pypi.org/simple/ --upgrade FanFicFare
|
||||
```
|
||||
|
||||
> `pip install --extra-index-url https://testpypi.python.org/pypi --upgrade FanFicFare`
|
||||
|
||||
|
||||
### Other Releases
|
||||
|
||||
Other versions may be available depending on your OS. I(JimmXinu) don't directly support these:
|
||||
|
||||
- **Arch Linux**: The latest CLI release can be obtained from the [fanficfare](https://aur.archlinux.org/packages/fanficfare) AUR package. It will install the calibre plugin, if calibre is installed.
|
||||
- **Arch Linux**: The CLI can also be obtained on Arch Linux from the OS repositories:
|
||||
|
||||
```
|
||||
pacman -S fanficfare
|
||||
```
|
||||
|
||||
...or from git via the [AUR package](https://aur.archlinux.org/packages/fanficfare-git)
|
||||
(which will also update the calibre plugin, if calibre is installed).
|
||||
|
||||
|
||||
[this post in the old FFDL thread]: https://www.mobileread.com/forums/showthread.php?p=1982785#post1982785
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
[main]
|
||||
host = https://www.transifex.com
|
||||
|
||||
[o:calibre:p:calibre-plugins:r:fanfictiondownloader]
|
||||
[calibre-plugins.fanfictiondownloader]
|
||||
file_filter = translations/<lang>.po
|
||||
source_file = translations/en.po
|
||||
source_lang = en
|
||||
type = PO
|
||||
type = PO
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ except NameError:
|
|||
from calibre.customize import InterfaceActionBase
|
||||
|
||||
# pulled out from FanFicFareBase for saving in prefs.py
|
||||
__version__ = (4, 57, 7)
|
||||
__version__ = (4, 14, 3)
|
||||
|
||||
## Apparently the name for this class doesn't matter--it was still
|
||||
## 'demo' for the first few versions.
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2024, Jim Miller'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
## References:
|
||||
## https://www.mobileread.com/forums/showthread.php?p=4435205&postcount=65
|
||||
## https://www.mobileread.com/forums/showthread.php?p=4102834&postcount=389
|
||||
|
||||
from calibre_plugins.action_chains.events import ChainEvent
|
||||
|
||||
class FanFicFareDownloadFinished(ChainEvent):
|
||||
|
||||
# replace with the name of your event
|
||||
name = 'FanFicFare Download Finished'
|
||||
|
||||
def get_event_signal(self):
|
||||
return self.gui.iactions['FanFicFare'].download_finished_signal
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>, 2018, Jim Miller'
|
||||
|
|
@ -14,16 +15,13 @@ from PyQt5.Qt import (QApplication, Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLa
|
|||
QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
|
||||
QTextEdit, QListWidget, QAbstractItemView, QCursor)
|
||||
|
||||
from calibre.constants import numeric_version as calibre_version
|
||||
from calibre.constants import iswindows, DEBUG
|
||||
from calibre.gui2 import UNDEFINED_QDATETIME, gprefs, info_dialog
|
||||
from calibre.gui2.actions import menu_action_unique_name
|
||||
from calibre.gui2.keyboard import ShortcutConfig
|
||||
from calibre.utils.config import config_dir
|
||||
from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
|
||||
|
||||
import fanficfare.six as six
|
||||
from six import text_type as unicode
|
||||
from fanficfare.six import text_type as unicode
|
||||
|
||||
# Global definition of our plugin name. Used for common functions that require this.
|
||||
plugin_name = None
|
||||
|
|
@ -42,41 +40,8 @@ def set_plugin_icon_resources(name, resources):
|
|||
plugin_name = name
|
||||
plugin_icon_resources = resources
|
||||
|
||||
# print_tracebacks_for_missing_resources first appears in cal 6.2.0
|
||||
if calibre_version >= (6,2,0):
|
||||
def get_icons_nolog(icon_name,plugin_name):
|
||||
return get_icons(icon_name,
|
||||
plugin_name,
|
||||
print_tracebacks_for_missing_resources=False)
|
||||
else:
|
||||
get_icons_nolog = get_icons
|
||||
|
||||
def get_icon_6plus(icon_name):
|
||||
'''
|
||||
Retrieve a QIcon for the named image from
|
||||
1. Calibre's image cache
|
||||
2. resources/images
|
||||
3. the icon theme
|
||||
4. the plugin zip
|
||||
Only plugin zip has images/ in the image name for backward
|
||||
compatibility.
|
||||
'''
|
||||
icon = None
|
||||
if icon_name:
|
||||
icon = QIcon.ic(icon_name)
|
||||
## both .ic and get_icons return an empty QIcon if not found.
|
||||
if not icon or icon.isNull():
|
||||
# don't need a tracestack from get_icons just because
|
||||
# there's no icon in the theme
|
||||
icon = get_icons_nolog(icon_name.replace('images/',''),
|
||||
plugin_name)
|
||||
if not icon or icon.isNull():
|
||||
icon = get_icons(icon_name,plugin_name)
|
||||
if not icon:
|
||||
icon = QIcon()
|
||||
return icon
|
||||
|
||||
def get_icon_old(icon_name):
|
||||
def get_icon(icon_name):
|
||||
'''
|
||||
Retrieve a QIcon for the named image from the zip file if it exists,
|
||||
or if not then from Calibre's image cache.
|
||||
|
|
@ -90,11 +55,6 @@ def get_icon_old(icon_name):
|
|||
return QIcon(pixmap)
|
||||
return QIcon()
|
||||
|
||||
# get_icons changed in Cal6.
|
||||
if calibre_version >= (6,0,0):
|
||||
get_icon = get_icon_6plus
|
||||
else:
|
||||
get_icon = get_icon_old
|
||||
|
||||
def get_pixmap(icon_name):
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2021, Jim Miller'
|
||||
|
|
@ -22,9 +23,7 @@ from PyQt5.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QLabel,
|
|||
|
||||
from calibre.gui2 import dynamic, info_dialog
|
||||
from calibre.gui2.complete2 import EditWithComplete
|
||||
from calibre.gui2.dialogs.confirm_delete import confirm
|
||||
import fanficfare.six as six
|
||||
from six import text_type as unicode
|
||||
from fanficfare.six import text_type as unicode
|
||||
|
||||
try:
|
||||
from calibre.ebooks.covers import generate_cover as cal_generate_cover
|
||||
|
|
@ -279,6 +278,7 @@ class ConfigWidget(QWidget):
|
|||
prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
|
||||
prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
|
||||
prefs['bgmeta'] = self.basic_tab.bgmeta.isChecked()
|
||||
prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
|
||||
prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
|
||||
prefs['mark'] = self.basic_tab.mark.isChecked()
|
||||
prefs['mark_success'] = self.basic_tab.mark_success.isChecked()
|
||||
|
|
@ -333,7 +333,6 @@ class ConfigWidget(QWidget):
|
|||
prefs['calibre_gen_cover'] = self.calibrecover_tab.calibre_gen_cover.isChecked()
|
||||
prefs['plugin_gen_cover'] = self.calibrecover_tab.plugin_gen_cover.isChecked()
|
||||
prefs['gcnewonly'] = self.calibrecover_tab.gcnewonly.isChecked()
|
||||
prefs['covernewonly'] = self.calibrecover_tab.covernewonly.isChecked()
|
||||
gc_site_settings = {}
|
||||
for (site,combo) in six.iteritems(self.calibrecover_tab.gc_dropdowns):
|
||||
val = unicode(combo.itemData(combo.currentIndex()))
|
||||
|
|
@ -371,11 +370,9 @@ class ConfigWidget(QWidget):
|
|||
prefs['suppresstitlesort'] = self.std_columns_tab.suppresstitlesort.isChecked()
|
||||
prefs['authorcase'] = self.std_columns_tab.authorcase.isChecked()
|
||||
prefs['titlecase'] = self.std_columns_tab.titlecase.isChecked()
|
||||
prefs['seriescase'] = self.std_columns_tab.seriescase.isChecked()
|
||||
prefs['setanthologyseries'] = self.std_columns_tab.setanthologyseries.isChecked()
|
||||
|
||||
prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
|
||||
prefs['set_series_url'] =self.std_columns_tab.set_series_url.isChecked()
|
||||
prefs['includecomments'] =self.std_columns_tab.includecomments.isChecked()
|
||||
prefs['anth_comments_newonly'] =self.std_columns_tab.anth_comments_newonly.isChecked()
|
||||
|
||||
|
|
@ -417,10 +414,6 @@ class ConfigWidget(QWidget):
|
|||
prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
|
||||
prefs['update_existing_only_from_email'] = self.imap_tab.update_existing_only_from_email.isChecked()
|
||||
prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
|
||||
|
||||
prefs['site_split_jobs'] = self.other_tab.site_split_jobs.isChecked()
|
||||
prefs['reconsolidate_jobs'] = self.other_tab.reconsolidate_jobs.isChecked()
|
||||
|
||||
prefs.save_to_db()
|
||||
self.plugin_action.set_popup_mode()
|
||||
|
||||
|
|
@ -489,6 +482,11 @@ class BasicTab(QWidget):
|
|||
self.updatemeta.setChecked(prefs['updatemeta'])
|
||||
horz.addWidget(self.updatemeta)
|
||||
|
||||
self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
|
||||
self.updateepubcover.setToolTip(_("On each download, FanFicFare offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
|
||||
self.updateepubcover.setChecked(prefs['updateepubcover'])
|
||||
horz.addWidget(self.updateepubcover)
|
||||
|
||||
self.bgmeta = QCheckBox(_('Default Background Metadata?'),self)
|
||||
self.bgmeta.setToolTip(_("On each download, FanFicFare offers an option to Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail.<br />Only available for Update/Overwrite of existing books in case URL given isn't canonical or matches to existing book by Title/Author."))
|
||||
self.bgmeta.setChecked(prefs['bgmeta'])
|
||||
|
|
@ -761,7 +759,6 @@ class BasicTab(QWidget):
|
|||
tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
|
||||
rejectreasons=rejecturllist.get_reject_reasons(),
|
||||
reasonslabel=_('Add this reason to all URLs added:'),
|
||||
accept_storyurls=True,
|
||||
save_size_name='fff:Add Reject List')
|
||||
d.exec_()
|
||||
if d.result() == d.Accepted:
|
||||
|
|
@ -1000,7 +997,7 @@ class CalibreCoverTab(QWidget):
|
|||
|
||||
self.gencov_elements=[] ## used to disable/enable when gen
|
||||
## cover is off/on. This is more
|
||||
## about being a visual cue than real
|
||||
## about being a visual que than real
|
||||
## necessary function.
|
||||
|
||||
topl = self.l = QVBoxLayout()
|
||||
|
|
@ -1044,17 +1041,9 @@ class CalibreCoverTab(QWidget):
|
|||
horz.addWidget(self.updatecalcover)
|
||||
self.l.addLayout(horz)
|
||||
|
||||
self.covernewonly = QCheckBox(_("Set Calibre Cover Only for New Books"),self)
|
||||
self.covernewonly.setToolTip(_("Set the Calibre cover from EPUB only for new\nbooks, not updates to existing books."))
|
||||
self.covernewonly.setChecked(prefs['covernewonly'])
|
||||
horz = QHBoxLayout()
|
||||
horz.addItem(QtGui.QSpacerItem(20, 1))
|
||||
horz.addWidget(self.covernewonly)
|
||||
self.l.addLayout(horz)
|
||||
self.l.addSpacing(5)
|
||||
|
||||
tooltip = _("Generate a Calibre book cover image when Calibre metadata is updated.<br />"
|
||||
"Note that %(gc)s(Plugin) will only run if there is a %(gc)s setting configured below for Default or the appropriate site.")%no_trans
|
||||
"Defaults to 'Yes, Always' for backward compatibility and because %(gc)s(Plugin)"
|
||||
" will only run if configured for Default or site.")%no_trans
|
||||
horz = QHBoxLayout()
|
||||
label = QLabel(_('Generate Calibre Cover:'))
|
||||
label.setToolTip(tooltip)
|
||||
|
|
@ -1062,7 +1051,13 @@ class CalibreCoverTab(QWidget):
|
|||
self.gencalcover = QComboBox(self)
|
||||
for i in gencalcover_order:
|
||||
self.gencalcover.addItem(i)
|
||||
# back compat. If has own value, use.
|
||||
# if prefs['gencalcover']:
|
||||
self.gencalcover.setCurrentIndex(self.gencalcover.findText(prefs_save_options[prefs['gencalcover']]))
|
||||
# elif prefs['gencover']: # doesn't have own val, set YES if old value set.
|
||||
# self.gencalcover.setCurrentIndex(self.gencalcover.findText(prefs_save_options[SAVE_YES]))
|
||||
# else: # doesn't have own value, old value not set, NO.
|
||||
# self.gencalcover.setCurrentIndex(self.gencalcover.findText(prefs_save_options[SAVE_NO]))
|
||||
|
||||
self.gencalcover.setToolTip(tooltip)
|
||||
label.setBuddy(self.gencalcover)
|
||||
|
|
@ -1070,26 +1065,6 @@ class CalibreCoverTab(QWidget):
|
|||
self.l.addLayout(horz)
|
||||
self.gencalcover.currentIndexChanged.connect(self.endisable_elements)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
horz.addItem(QtGui.QSpacerItem(20, 1))
|
||||
vert = QVBoxLayout()
|
||||
horz.addLayout(vert)
|
||||
self.l.addLayout(horz)
|
||||
|
||||
self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
|
||||
self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
|
||||
" updated.<br />Used for both Calibre and Plugin generated covers."))
|
||||
self.gcnewonly.setChecked(prefs['gcnewonly'])
|
||||
vert.addWidget(self.gcnewonly)
|
||||
self.gencov_elements.append(self.gcnewonly)
|
||||
|
||||
self.gc_polish_cover = QCheckBox(_("Inject/update the generated cover inside EPUB"),self)
|
||||
self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
|
||||
" cover into the EPUB ebook file.<br />Used for both Calibre and Plugin generated covers."))
|
||||
self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
|
||||
vert.addWidget(self.gc_polish_cover)
|
||||
self.gencov_elements.append(self.gc_polish_cover)
|
||||
|
||||
# can't be local or it's destroyed when __init__ is done and
|
||||
# connected things don't fire.
|
||||
self.gencov_rdgrp = QButtonGroup()
|
||||
|
|
@ -1098,9 +1073,7 @@ class CalibreCoverTab(QWidget):
|
|||
self.gencov_gb.setLayout(horz)
|
||||
|
||||
self.plugin_gen_cover = QRadioButton(_('Plugin %(gc)s')%no_trans,self)
|
||||
self.plugin_gen_cover.setToolTip(_("Use the %(gc)s plugin to create covers.<br>"
|
||||
"Requires that you have the the %(gc)s plugin installed.<br>"
|
||||
"Additional settings are below.")%no_trans)
|
||||
self.plugin_gen_cover.setToolTip(_("Use plugin to create covers. Additional settings are below."))
|
||||
self.gencov_rdgrp.addButton(self.plugin_gen_cover)
|
||||
# always, new only, when no cover from site, inject yes/no...
|
||||
self.plugin_gen_cover.setChecked(prefs['plugin_gen_cover'])
|
||||
|
|
@ -1122,6 +1095,20 @@ class CalibreCoverTab(QWidget):
|
|||
#self.l.addLayout(horz)
|
||||
self.l.addWidget(self.gencov_gb)
|
||||
|
||||
self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
|
||||
self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
|
||||
" updated.<br />Used for both Calibre and Plugin generated covers."))
|
||||
self.gcnewonly.setChecked(prefs['gcnewonly'])
|
||||
self.l.addWidget(self.gcnewonly)
|
||||
self.gencov_elements.append(self.gcnewonly)
|
||||
|
||||
self.gc_polish_cover = QCheckBox(_("Inject/update the cover inside EPUB"),self)
|
||||
self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
|
||||
" cover into the EPUB ebook file.<br />Used for both Calibre and Plugin generated covers."))
|
||||
self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
|
||||
self.l.addWidget(self.gc_polish_cover)
|
||||
self.gencov_elements.append(self.gc_polish_cover)
|
||||
|
||||
self.gcp_gb = QGroupBox(_("%(gc)s(Plugin) Settings")%no_trans)
|
||||
topl.addWidget(self.gcp_gb)
|
||||
self.l = QVBoxLayout()
|
||||
|
|
@ -1280,31 +1267,6 @@ class OtherTab(QWidget):
|
|||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
||||
groupbox = QGroupBox()
|
||||
self.l.addWidget(groupbox)
|
||||
|
||||
groupl = QVBoxLayout()
|
||||
groupbox.setLayout(groupl)
|
||||
|
||||
label = QLabel("<h3>"+
|
||||
_("Background Job Settings")+
|
||||
"</h3>"
|
||||
)
|
||||
label.setWordWrap(True)
|
||||
groupl.addWidget(label)
|
||||
|
||||
self.site_split_jobs = QCheckBox(_('Split downloads into separate background jobs by site'),self)
|
||||
self.site_split_jobs.setToolTip(_("Launches a separate background Job for each site in the list of stories to download/update. Otherwise, there will be only one background job."))
|
||||
self.site_split_jobs.setChecked(prefs['site_split_jobs'])
|
||||
groupl.addWidget(self.site_split_jobs)
|
||||
|
||||
self.reconsolidate_jobs = QCheckBox(_('Reconsolidate split downloads before updating library'),self)
|
||||
self.reconsolidate_jobs.setToolTip(_("Hold all downloads/updates launched together until they all finish. Otherwise, there will be a 'Proceed to update' dialog for each site."))
|
||||
self.reconsolidate_jobs.setChecked(prefs['reconsolidate_jobs'])
|
||||
groupl.addWidget(self.reconsolidate_jobs)
|
||||
|
||||
self.l.addSpacing(5)
|
||||
|
||||
label = QLabel(_("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFicFare confirmation dialogs back again."))
|
||||
label.setWordWrap(True)
|
||||
self.l.addWidget(label)
|
||||
|
|
@ -1610,39 +1572,22 @@ class StandardColumnsTab(QWidget):
|
|||
self.titlecase.setChecked(prefs['titlecase'])
|
||||
row.append(self.titlecase)
|
||||
elif key == 'authors':
|
||||
self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
|
||||
self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
|
||||
self.set_author_url.setChecked(prefs['set_author_url'])
|
||||
row.append(self.set_author_url)
|
||||
|
||||
self.suppressauthorsort = QCheckBox(_('Force Author into Author Sort?'),self)
|
||||
self.suppressauthorsort.setToolTip(_("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc."))
|
||||
self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
|
||||
row.append(self.suppressauthorsort)
|
||||
|
||||
self.authorcase = QCheckBox(_('Fix Author Case?'),self)
|
||||
self.authorcase.setToolTip(_("If checked, Calibre's routine for correcting the capitalization of author names will be applied.")
|
||||
+"\n"+_("Calibre remembers all authors in the library; changing the author case on one book will effect all books by that author.")
|
||||
+"\n"+_("This effects Calibre metadata only, not FanFicFare metadata in title page."))
|
||||
self.authorcase.setChecked(prefs['authorcase'])
|
||||
row.append(self.authorcase)
|
||||
|
||||
elif key == 'series':
|
||||
self.set_series_url = QCheckBox(_('Set Calibre Series URL'),self)
|
||||
self.set_series_url.setToolTip(_("Set Calibre Series URL to Series's URL on story site."))
|
||||
self.set_series_url.setChecked(prefs['set_series_url'])
|
||||
row.append(self.set_series_url)
|
||||
|
||||
self.setanthologyseries = QCheckBox(_("Set 'Series [0]' for New Anthologies?"),self)
|
||||
self.setanthologyseries.setToolTip(_("If checked, the Series column will be set to 'Series Name [0]' when an Anthology for a series is first created."))
|
||||
self.setanthologyseries.setChecked(prefs['setanthologyseries'])
|
||||
row.append(self.setanthologyseries)
|
||||
|
||||
self.seriescase = QCheckBox(_('Fix Series Case?'),self)
|
||||
self.seriescase.setToolTip(_("If checked, Calibre's routine for correcting the capitalization of title will be applied.")
|
||||
+"\n"+_("This effects Calibre metadata only, not FanFicFare metadata in title page."))
|
||||
self.seriescase.setChecked(prefs['seriescase'])
|
||||
row.append(self.seriescase)
|
||||
grid = QGridLayout()
|
||||
for rownum, row in enumerate(rows):
|
||||
for colnum, col in enumerate(row):
|
||||
|
|
@ -1655,6 +1600,11 @@ class StandardColumnsTab(QWidget):
|
|||
self.l.addWidget(label)
|
||||
self.l.addSpacing(5)
|
||||
|
||||
self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
|
||||
self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
|
||||
self.set_author_url.setChecked(prefs['set_author_url'])
|
||||
self.l.addWidget(self.set_author_url)
|
||||
|
||||
self.includecomments = QCheckBox(_("Include Books' Comments in Anthology Comments?"),self)
|
||||
self.includecomments.setToolTip(_('''Include all the merged books' comments in the new book's comments.
|
||||
Default is a list of included titles only.'''))
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ from datetime import datetime
|
|||
|
||||
from PyQt5 import QtWidgets as QtGui
|
||||
from PyQt5 import QtCore
|
||||
from PyQt5.Qt import (QApplication, QDialog, QWidget, QTableWidget, QTableWidgetItem, QVBoxLayout,
|
||||
QHBoxLayout, QGridLayout, QPushButton, QFont, QLabel, QCheckBox, QIcon,
|
||||
from PyQt5.Qt import (QApplication, QDialog, QWidget, QTableWidget, QVBoxLayout, QHBoxLayout,
|
||||
QGridLayout, QPushButton, QFont, QLabel, QCheckBox, QIcon,
|
||||
QLineEdit, QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
|
||||
QScrollArea, QPixmap, Qt, QAbstractItemView, QTextEdit,
|
||||
pyqtSignal, QGroupBox, QFrame, QTextCursor)
|
||||
|
|
@ -38,7 +38,6 @@ from calibre.gui2 import gprefs
|
|||
show_download_options = 'fff:add new/update dialogs:show_download_options'
|
||||
from calibre.gui2.dialogs.confirm_delete import confirm
|
||||
from calibre.gui2.complete2 import EditWithComplete
|
||||
from fanficfare.exceptions import NotGoingToDownload
|
||||
from fanficfare.six import text_type as unicode, ensure_text
|
||||
|
||||
# pulls in translation files for _() strings
|
||||
|
|
@ -156,6 +155,15 @@ class RejectUrlEntry:
|
|||
|
||||
return retval
|
||||
|
||||
class NotGoingToDownload(Exception):
|
||||
def __init__(self,error,icon='dialog_error.png',showerror=True):
|
||||
self.error=error
|
||||
self.icon=icon
|
||||
self.showerror=showerror
|
||||
|
||||
def __str__(self):
|
||||
return self.error
|
||||
|
||||
class DroppableQTextEdit(QTextEdit):
|
||||
def __init__(self,parent):
|
||||
QTextEdit.__init__(self,parent)
|
||||
|
|
@ -181,32 +189,12 @@ class DroppableQTextEdit(QTextEdit):
|
|||
else:
|
||||
return QTextEdit.insertFromMimeData(self, mime_data)
|
||||
|
||||
class HotKeyedSizePersistedDialog(SizePersistedDialog):
|
||||
|
||||
def __init__(self, gui, save_size_name):
|
||||
super(HotKeyedSizePersistedDialog,self).__init__(gui, save_size_name)
|
||||
self.keys=dict()
|
||||
|
||||
def addCtrlKeyPress(self,key,func):
|
||||
# print("addKeyPress: key(0x%x)"%key)
|
||||
# print("control: 0x%x"%QtCore.Qt.ControlModifier)
|
||||
self.keys[key]=func
|
||||
|
||||
def keyPressEvent(self, event):
|
||||
# print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
|
||||
if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
|
||||
func = self.keys[event.key()]
|
||||
return func()
|
||||
else:
|
||||
return super(HotKeyedSizePersistedDialog,self).keyPressEvent(event)
|
||||
|
||||
class AddNewDialog(HotKeyedSizePersistedDialog):
|
||||
class AddNewDialog(SizePersistedDialog):
|
||||
|
||||
go_signal = pyqtSignal(object, object, object, object)
|
||||
|
||||
def __init__(self, gui, prefs, icon):
|
||||
super(AddNewDialog,self).__init__(gui, 'fff:add new dialog')
|
||||
|
||||
SizePersistedDialog.__init__(self, gui, 'fff:add new dialog')
|
||||
self.prefs = prefs
|
||||
|
||||
self.setMinimumWidth(300)
|
||||
|
|
@ -228,25 +216,19 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.toplabel=QLabel("Toplabel")
|
||||
self.l.addWidget(self.toplabel)
|
||||
|
||||
## scrollable area for lengthy series comments.
|
||||
scrollable = QScrollArea()
|
||||
scrollcontent = QWidget()
|
||||
scrollable.setWidget(scrollcontent)
|
||||
scrollable.setWidgetResizable(True)
|
||||
self.l.addWidget(scrollable)
|
||||
|
||||
grid = QGridLayout()
|
||||
scrollcontent.setLayout(grid)
|
||||
self.mergeshow.append(scrollable)
|
||||
|
||||
## XXX add labels for series name and desc? Desc in tooltip?
|
||||
row = 0
|
||||
grid = QGridLayout()
|
||||
label = QLabel('<b>'+_('Series')+':</b>')
|
||||
grid.addWidget(label,row,0)
|
||||
self.mergedname=QLabel("mergedname")
|
||||
tt = _('This name will be used with the %s setting to set the title of the new book.')%'<i>anthology_title_pattern</i>'
|
||||
label.setToolTip(tt)
|
||||
self.mergeshow.append(label)
|
||||
self.mergedname.setToolTip(tt)
|
||||
grid.addWidget(self.mergedname,row,1,1,-1)
|
||||
self.l.addLayout(grid)
|
||||
self.mergeshow.append(self.mergedname)
|
||||
|
||||
row+=1
|
||||
label = QLabel('<b>'+_('Comments')+':</b>')
|
||||
|
|
@ -254,15 +236,18 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.mergeddesc=QLabel("mergeddesc")
|
||||
tt = _('These comments about the series will be included in the Comments of the new book.')+'<i></i>' # for html for auto-wrap
|
||||
label.setToolTip(tt)
|
||||
self.mergeshow.append(label)
|
||||
self.mergeddesc.setToolTip(tt)
|
||||
self.mergeddesc.setWordWrap(True)
|
||||
grid.addWidget(self.mergeddesc,row,1,1,-1)
|
||||
self.l.addLayout(grid)
|
||||
self.mergeshow.append(self.mergeddesc)
|
||||
grid.setColumnStretch(1,1)
|
||||
|
||||
self.url = DroppableQTextEdit(self)
|
||||
self.url.setToolTip("UrlTooltip")
|
||||
self.url.setLineWrapMode(QTextEditNoWrap)
|
||||
self.l.addWidget(self.url,1) # 1 higher 'stretch'==higher priority
|
||||
self.l.addWidget(self.url)
|
||||
|
||||
self.groupbox = QGroupBox(_("Show Download Options"))
|
||||
self.groupbox.setCheckable(True)
|
||||
|
|
@ -326,6 +311,12 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.mergehide.append(self.updatemeta)
|
||||
self.mergeupdateshow.append(self.updatemeta)
|
||||
|
||||
self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
|
||||
self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
|
||||
self.updateepubcover.setChecked(self.prefs['updateepubcover'])
|
||||
horz.addWidget(self.updateepubcover)
|
||||
self.mergehide.append(self.updateepubcover)
|
||||
|
||||
self.gbl.addLayout(horz)
|
||||
|
||||
## bgmeta not used with Add New because of stories that change
|
||||
|
|
@ -345,9 +336,6 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.button_box.rejected.connect(self.reject)
|
||||
self.l.addWidget(self.button_box)
|
||||
|
||||
self.addCtrlKeyPress(QtCore.Qt.Key_Return,self.ok_clicked)
|
||||
self.addCtrlKeyPress(QtCore.Qt.Key_Enter,self.ok_clicked) # num pad
|
||||
|
||||
def click_show_download_options(self,x):
|
||||
self.gbf.setVisible(x)
|
||||
gprefs[show_download_options] = x
|
||||
|
|
@ -458,6 +446,9 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.updatemeta.setChecked(self.prefs['updatemeta'])
|
||||
# self.bgmeta.setChecked(self.prefs['bgmeta'])
|
||||
|
||||
if not self.merge:
|
||||
self.updateepubcover.setChecked(self.prefs['updateepubcover'])
|
||||
|
||||
self.url.setText(url_list_text)
|
||||
if url_list_text:
|
||||
self.button_box.button(QDialogButtonBox.Ok).setFocus()
|
||||
|
|
@ -490,18 +481,19 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
self.collision.setCurrentIndex(i)
|
||||
|
||||
def get_fff_options(self):
|
||||
retval = dict(self.extraoptions)
|
||||
retval.update( {
|
||||
'fileform': unicode(self.fileform.currentText()),
|
||||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': False, # self.bgmeta.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation'],
|
||||
'do_wordcount':self.prefs['do_wordcount'],
|
||||
} )
|
||||
retval = {
|
||||
'fileform': unicode(self.fileform.currentText()),
|
||||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': False, # self.bgmeta.isChecked(),
|
||||
'updateepubcover': self.updateepubcover.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation'],
|
||||
'do_wordcount':self.prefs['do_wordcount'],
|
||||
}
|
||||
|
||||
if self.merge:
|
||||
retval['fileform']=='epub'
|
||||
retval['updateepubcover']=True
|
||||
if self.newmerge:
|
||||
retval['updatemeta']=True
|
||||
retval['collision']=ADDNEW
|
||||
|
|
@ -513,6 +505,7 @@ class AddNewDialog(HotKeyedSizePersistedDialog):
|
|||
def get_urlstext(self):
|
||||
return unicode(self.url.toPlainText())
|
||||
|
||||
|
||||
class FakeLineEdit():
|
||||
def __init__(self):
|
||||
pass
|
||||
|
|
@ -588,83 +581,35 @@ class UserPassDialog(QDialog):
|
|||
QDialog.__init__(self, gui)
|
||||
self.status=False
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.l = QGridLayout()
|
||||
self.setLayout(self.l)
|
||||
|
||||
grid = QGridLayout()
|
||||
self.l.addLayout(grid)
|
||||
|
||||
if exception and exception.passwdonly:
|
||||
self.setWindowTitle(_('Password'))
|
||||
grid.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
|
||||
self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
|
||||
# user isn't used, but it's easier to still have it for
|
||||
# post processing.
|
||||
self.user = FakeLineEdit()
|
||||
else:
|
||||
self.setWindowTitle(_('User/Password'))
|
||||
grid.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
|
||||
self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
|
||||
|
||||
grid.addWidget(QLabel(_("User:")),1,0)
|
||||
self.l.addWidget(QLabel(_("User:")),1,0)
|
||||
self.user = QLineEdit(self)
|
||||
grid.addWidget(self.user,1,1)
|
||||
self.l.addWidget(self.user,1,1)
|
||||
|
||||
grid.addWidget(QLabel(_("Password:")),2,0)
|
||||
self.l.addWidget(QLabel(_("Password:")),2,0)
|
||||
self.passwd = QLineEdit(self)
|
||||
self.passwd.setEchoMode(QLineEdit.Password)
|
||||
grid.addWidget(self.passwd,2,1)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
self.l.addLayout(horz)
|
||||
self.l.addWidget(self.passwd,2,1)
|
||||
|
||||
self.ok_button = QPushButton(_('OK'), self)
|
||||
self.ok_button.clicked.connect(self.ok)
|
||||
horz.addWidget(self.ok_button)
|
||||
self.l.addWidget(self.ok_button,3,0)
|
||||
|
||||
self.cancel_button = QPushButton(_('Cancel'), self)
|
||||
self.cancel_button.clicked.connect(self.cancel)
|
||||
horz.addWidget(self.cancel_button)
|
||||
|
||||
self.resize(self.sizeHint())
|
||||
|
||||
def ok(self):
|
||||
self.status=True
|
||||
self.hide()
|
||||
|
||||
def cancel(self):
|
||||
self.status=False
|
||||
self.hide()
|
||||
|
||||
class TOTPDialog(QDialog):
|
||||
'''
|
||||
Need to collect Timebased One Time Password(TOTP) for some sites.
|
||||
'''
|
||||
def __init__(self, gui, site, exception=None):
|
||||
QDialog.__init__(self, gui)
|
||||
self.status=False
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
||||
grid = QGridLayout()
|
||||
self.l.addLayout(grid)
|
||||
|
||||
self.setWindowTitle(_('Time-based One Time Password(TOTP)'))
|
||||
grid.addWidget(QLabel(_("Site requires a Time-based One Time Password(TOTP) for this url:\n%s")%exception.url),0,0,1,2)
|
||||
|
||||
grid.addWidget(QLabel(_("TOTP:")),2,0)
|
||||
self.totp = QLineEdit(self)
|
||||
grid.addWidget(self.totp,2,1)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
self.l.addLayout(horz)
|
||||
|
||||
self.ok_button = QPushButton(_('OK'), self)
|
||||
self.ok_button.clicked.connect(self.ok)
|
||||
horz.addWidget(self.ok_button)
|
||||
|
||||
self.cancel_button = QPushButton(_('Cancel'), self)
|
||||
self.cancel_button.clicked.connect(self.cancel)
|
||||
horz.addWidget(self.cancel_button)
|
||||
self.l.addWidget(self.cancel_button,3,1)
|
||||
|
||||
self.resize(self.sizeHint())
|
||||
|
||||
|
|
@ -682,15 +627,13 @@ def LoopProgressDialog(gui,
|
|||
finish_function,
|
||||
init_label=_("Fetching metadata for stories..."),
|
||||
win_title=_("Downloading metadata for stories"),
|
||||
status_prefix=_("Fetched metadata for"),
|
||||
disable_cancel=False):
|
||||
status_prefix=_("Fetched metadata for")):
|
||||
ld = _LoopProgressDialog(gui,
|
||||
book_list,
|
||||
foreach_function,
|
||||
init_label,
|
||||
win_title,
|
||||
status_prefix,
|
||||
disable_cancel)
|
||||
status_prefix)
|
||||
|
||||
# Mac OS X gets upset if the finish_function is called from inside
|
||||
# the real _LoopProgressDialog class.
|
||||
|
|
@ -708,12 +651,10 @@ class _LoopProgressDialog(QProgressDialog):
|
|||
foreach_function,
|
||||
init_label=_("Fetching metadata for stories..."),
|
||||
win_title=_("Downloading metadata for stories"),
|
||||
status_prefix=_("Fetched metadata for"),
|
||||
disable_cancel=False):
|
||||
status_prefix=_("Fetched metadata for")):
|
||||
QProgressDialog.__init__(self,
|
||||
init_label,
|
||||
_('Cancel'), 0, len(book_list), gui)
|
||||
self.gui = gui
|
||||
self.setWindowTitle(win_title)
|
||||
self.setMinimumWidth(500)
|
||||
self.book_list = book_list
|
||||
|
|
@ -721,6 +662,7 @@ class _LoopProgressDialog(QProgressDialog):
|
|||
self.status_prefix = status_prefix
|
||||
self.i = 0
|
||||
self.start_time = datetime.now()
|
||||
self.first = True
|
||||
|
||||
# can't import at file load.
|
||||
from calibre_plugins.fanficfare_plugin.prefs import prefs
|
||||
|
|
@ -729,27 +671,11 @@ class _LoopProgressDialog(QProgressDialog):
|
|||
self.setLabelText('%s %d / %d' % (self.status_prefix, self.i, len(self.book_list)))
|
||||
self.setValue(self.i)
|
||||
|
||||
if disable_cancel:
|
||||
self.setCancelButton(None)
|
||||
self.reject = self.disabled_reject
|
||||
self.closeEvent = self.disabled_closeEvent
|
||||
|
||||
## self.do_loop does QTimer.singleShot on self.do_loop also.
|
||||
## A weird way to do a loop, but that was the example I had.
|
||||
## 100 instead of 0 on the first go due to Win10(and later
|
||||
## qt6) not displaying dialog properly.
|
||||
QTimer.singleShot(100, self.do_loop)
|
||||
QTimer.singleShot(0, self.do_loop)
|
||||
self.exec_()
|
||||
|
||||
# used when disable_cancel = True
|
||||
def disabled_reject(self):
|
||||
pass
|
||||
|
||||
# used when disable_cancel = True
|
||||
def disabled_closeEvent(self, event):
|
||||
if event.spontaneous():
|
||||
event.ignore()
|
||||
|
||||
def updateStatus(self):
|
||||
remaining_time_string = ''
|
||||
if self.show_est_time and self.i > -1:
|
||||
|
|
@ -763,6 +689,15 @@ class _LoopProgressDialog(QProgressDialog):
|
|||
|
||||
def do_loop(self):
|
||||
|
||||
if self.first:
|
||||
## Windows 10 doesn't want to show the prog dialog content
|
||||
## until after the timer's been called again. Something to
|
||||
## do with cooperative multi threading maybe?
|
||||
## So this just trips the timer loop an extra time at the start.
|
||||
self.first = False
|
||||
QTimer.singleShot(0, self.do_loop)
|
||||
return
|
||||
|
||||
book = self.book_list[self.i]
|
||||
try:
|
||||
## collision spec passed into getadapter by partial from fff_plugin
|
||||
|
|
@ -961,6 +896,11 @@ class UpdateExistingDialog(SizePersistedDialog):
|
|||
self.updatemeta.setChecked(self.prefs['updatemeta'])
|
||||
horz.addWidget(self.updatemeta)
|
||||
|
||||
self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
|
||||
self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
|
||||
self.updateepubcover.setChecked(self.prefs['updateepubcover'])
|
||||
horz.addWidget(self.updateepubcover)
|
||||
|
||||
self.bgmeta = QCheckBox(_('Background Metadata?'),self)
|
||||
self.bgmeta.setToolTip(_("Collect Metadata from sites in a Background process.<br />This returns control to you quicker while updating, but you won't be asked for username/passwords or if you are an adult--stories that need those will just fail."))
|
||||
self.bgmeta.setChecked(self.prefs['bgmeta'])
|
||||
|
|
@ -1012,6 +952,7 @@ class UpdateExistingDialog(SizePersistedDialog):
|
|||
'collision': unicode(self.collision.currentText()),
|
||||
'updatemeta': self.updatemeta.isChecked(),
|
||||
'bgmeta': self.bgmeta.isChecked(),
|
||||
'updateepubcover': self.updateepubcover.isChecked(),
|
||||
'smarten_punctuation':self.prefs['smarten_punctuation'],
|
||||
'do_wordcount':self.prefs['do_wordcount'],
|
||||
}
|
||||
|
|
@ -1092,7 +1033,6 @@ class StoryListTableWidget(QTableWidget):
|
|||
def remove_selected_rows(self):
|
||||
self.setFocus()
|
||||
rows = self.selectionModel().selectedRows()
|
||||
rows = sorted(rows, key=lambda x: x.row(), reverse=True)
|
||||
if len(rows) == 0:
|
||||
return
|
||||
message = '<p>'+_('Are you sure you want to remove this book from the list?')
|
||||
|
|
@ -1101,7 +1041,7 @@ class StoryListTableWidget(QTableWidget):
|
|||
if not confirm(message,'fff_delete_item', self):
|
||||
return
|
||||
first_sel_row = self.currentRow()
|
||||
for selrow in rows:
|
||||
for selrow in reversed(rows):
|
||||
self.removeRow(selrow.row())
|
||||
if first_sel_row < self.rowCount():
|
||||
self.select_and_scroll_to_row(first_sel_row)
|
||||
|
|
@ -1112,19 +1052,6 @@ class StoryListTableWidget(QTableWidget):
|
|||
self.selectRow(row)
|
||||
self.scrollToItem(self.currentItem())
|
||||
|
||||
## Added to allow sorting by Notes column
|
||||
class NotesWidgetItem(QTableWidgetItem):
|
||||
def __init__(self,content):
|
||||
QTableWidgetItem.__init__(self)
|
||||
self.content=content
|
||||
|
||||
def currentText(self):
|
||||
return self.content.currentText()
|
||||
|
||||
def __lt__(self, other):
|
||||
return (unicode(self.currentText()).lower().strip() <
|
||||
unicode(other.currentText()).lower().strip())
|
||||
|
||||
class RejectListTableWidget(QTableWidget):
|
||||
|
||||
def __init__(self, parent,rejectreasons=[]):
|
||||
|
|
@ -1169,7 +1096,6 @@ class RejectListTableWidget(QTableWidget):
|
|||
self.setItem(row, 1, EditableTableWidgetItem(rej.title))
|
||||
self.setItem(row, 2, EditableTableWidgetItem(rej.auth))
|
||||
|
||||
# sort_func orders dropdown-constant to preserve user order.
|
||||
note_cell = EditWithComplete(self,sort_func=lambda x:1)
|
||||
|
||||
items = [rej.note]+self.rejectreasons
|
||||
|
|
@ -1177,14 +1103,12 @@ class RejectListTableWidget(QTableWidget):
|
|||
note_cell.show_initial_value(rej.note)
|
||||
note_cell.set_separator(None)
|
||||
note_cell.setToolTip(_('Select or Edit Reject Note.'))
|
||||
self.setItem(row, 3, NotesWidgetItem(note_cell))
|
||||
self.setCellWidget(row, 3, note_cell)
|
||||
note_cell.setCursorPosition(0)
|
||||
|
||||
def remove_selected_rows(self):
|
||||
self.setFocus()
|
||||
rows = self.selectionModel().selectedRows()
|
||||
rows = sorted(rows, key=lambda x: x.row(), reverse=True)
|
||||
if len(rows) == 0:
|
||||
return
|
||||
message = '<p>'+_('Are you sure you want to remove this URL from the list?')
|
||||
|
|
@ -1193,7 +1117,7 @@ class RejectListTableWidget(QTableWidget):
|
|||
if not confirm(message,'fff_rejectlist_delete_item_again', self):
|
||||
return
|
||||
first_sel_row = self.currentRow()
|
||||
for selrow in rows:
|
||||
for selrow in reversed(rows):
|
||||
self.removeRow(selrow.row())
|
||||
if first_sel_row < self.rowCount():
|
||||
self.select_and_scroll_to_row(first_sel_row)
|
||||
|
|
@ -1320,7 +1244,6 @@ class EditTextDialog(SizePersistedDialog):
|
|||
icon=None, title=None, label=None, tooltip=None,
|
||||
read_only=False,
|
||||
rejectreasons=[],reasonslabel=None,
|
||||
accept_storyurls=False,
|
||||
save_size_name='fff:edit text dialog',
|
||||
):
|
||||
SizePersistedDialog.__init__(self, parent, save_size_name)
|
||||
|
|
@ -1334,10 +1257,7 @@ class EditTextDialog(SizePersistedDialog):
|
|||
self.setWindowIcon(icon)
|
||||
self.l.addWidget(self.label)
|
||||
|
||||
if accept_storyurls:
|
||||
self.textedit = DroppableQTextEdit(self)
|
||||
else:
|
||||
self.textedit = QTextEdit(self)
|
||||
self.textedit = QTextEdit(self)
|
||||
self.textedit.setLineWrapMode(QTextEditNoWrap)
|
||||
self.textedit.setReadOnly(read_only)
|
||||
self.textedit.setText(text)
|
||||
|
|
@ -1381,18 +1301,7 @@ class EditTextDialog(SizePersistedDialog):
|
|||
def get_reason_text(self):
|
||||
return unicode(self.reason_edit.currentText()).strip()
|
||||
|
||||
class QTextEditPlainPaste(QTextEdit):
|
||||
def insertFromMimeData(self, mimeData):
|
||||
# logger.debug("insertFromMimeData called")
|
||||
#Ensure it is text.
|
||||
if (mimeData.hasText()):
|
||||
text = mimeData.text()
|
||||
self.insertPlainText(text)
|
||||
#In case not text.
|
||||
else:
|
||||
QTextEdit.insertFromMimeData(self, mimeData)
|
||||
|
||||
class IniTextDialog(HotKeyedSizePersistedDialog):
|
||||
class IniTextDialog(SizePersistedDialog):
|
||||
|
||||
def __init__(self, parent, text,
|
||||
icon=None, title=None, label=None,
|
||||
|
|
@ -1400,7 +1309,9 @@ class IniTextDialog(HotKeyedSizePersistedDialog):
|
|||
read_only=False,
|
||||
save_size_name='fff:ini text dialog',
|
||||
):
|
||||
super(IniTextDialog,self).__init__(parent, save_size_name)
|
||||
SizePersistedDialog.__init__(self, parent, save_size_name)
|
||||
|
||||
self.keys=dict()
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
|
|
@ -1411,7 +1322,7 @@ class IniTextDialog(HotKeyedSizePersistedDialog):
|
|||
self.setWindowIcon(icon)
|
||||
self.l.addWidget(self.label)
|
||||
|
||||
self.textedit = QTextEditPlainPaste(self)
|
||||
self.textedit = QTextEdit(self)
|
||||
|
||||
highlighter = IniHighlighter(self.textedit,
|
||||
sections=get_valid_sections(),
|
||||
|
|
@ -1501,6 +1412,19 @@ class IniTextDialog(HotKeyedSizePersistedDialog):
|
|||
# print("call parent accept")
|
||||
return SizePersistedDialog.accept(self)
|
||||
|
||||
def addCtrlKeyPress(self,key,func):
|
||||
# print("addKeyPress: key(0x%x)"%key)
|
||||
# print("control: 0x%x"%QtCore.Qt.ControlModifier)
|
||||
self.keys[key]=func
|
||||
|
||||
def keyPressEvent(self, event):
|
||||
# print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
|
||||
if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
|
||||
func = self.keys[event.key()]
|
||||
return func()
|
||||
else:
|
||||
return SizePersistedDialog.keyPressEvent(self, event)
|
||||
|
||||
def get_plain_text(self):
|
||||
return unicode(self.textedit.toPlainText())
|
||||
|
||||
|
|
@ -1569,6 +1493,7 @@ class IniTextDialog(HotKeyedSizePersistedDialog):
|
|||
# And finally we set this new cursor as the parent's
|
||||
self.textedit.setTextCursor(cursor)
|
||||
|
||||
|
||||
class ViewLog(SizePersistedDialog):
|
||||
|
||||
def label_clicked(self, event, lineno=None):
|
||||
|
|
@ -1657,30 +1582,28 @@ class EmailPassDialog(QDialog):
|
|||
QDialog.__init__(self, gui)
|
||||
self.status=False
|
||||
|
||||
self.l = QVBoxLayout()
|
||||
self.l = QGridLayout()
|
||||
self.setLayout(self.l)
|
||||
|
||||
grid = QGridLayout()
|
||||
self.l.addLayout(grid)
|
||||
|
||||
self.setWindowTitle(_('Password'))
|
||||
grid.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
|
||||
self.l.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
|
||||
|
||||
# grid.addWidget(QLabel(_("Password:")),1,0)
|
||||
# self.l.addWidget(QLabel(_("Password:")),1,0)
|
||||
self.passwd = QLineEdit(self)
|
||||
self.passwd.setEchoMode(QLineEdit.Password)
|
||||
grid.addWidget(self.passwd,1,0,1,2)
|
||||
|
||||
horz = QHBoxLayout()
|
||||
self.l.addLayout(horz)
|
||||
self.l.addWidget(self.passwd,1,0,1,2)
|
||||
|
||||
self.ok_button = QPushButton(_('OK'), self)
|
||||
self.ok_button.clicked.connect(self.ok)
|
||||
horz.addWidget(self.ok_button)
|
||||
self.l.addWidget(self.ok_button,2,0)
|
||||
|
||||
self.cancel_button = QPushButton(_('Cancel'), self)
|
||||
self.cancel_button.clicked.connect(self.cancel)
|
||||
horz.addWidget(self.cancel_button)
|
||||
self.l.addWidget(self.cancel_button,2,1)
|
||||
|
||||
# set stretch factors the same.
|
||||
self.l.setColumnStretch(0,1)
|
||||
self.l.setColumnStretch(1,1)
|
||||
|
||||
self.resize(self.sizeHint())
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -33,8 +33,8 @@ def get_fff_config(url,fileform="epub",personalini=None):
|
|||
except Exception as e:
|
||||
logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
|
||||
configuration = Configuration(sections,fileform)
|
||||
configuration.read_file(StringIO(ensure_text(get_resources("plugin-defaults.ini"))))
|
||||
configuration.read_file(StringIO(ensure_text(personalini)))
|
||||
configuration.readfp(StringIO(ensure_text(get_resources("plugin-defaults.ini"))))
|
||||
configuration.readfp(StringIO(ensure_text(personalini)))
|
||||
|
||||
return configuration
|
||||
|
||||
|
|
|
|||
BIN
calibre-plugin/images/old icon.png
Normal file
BIN
calibre-plugin/images/old icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
|
|
@ -95,7 +95,7 @@ class IniHighlighter(QSyntaxHighlighter):
|
|||
if sections:
|
||||
# *known* sections
|
||||
resections = r'('+(r'|'.join(sections))+r')'
|
||||
resections = resections.replace('.',r'\.') #escape dots.
|
||||
resections = resections.replace('.','\.') #escape dots.
|
||||
self.highlightingRules.append( HighlightingRule( r"^\["+resections+r"\]\s*$", colors['knownsections'], QFontBold, blocknum=2 ) )
|
||||
|
||||
# test story sections
|
||||
|
|
@ -103,8 +103,7 @@ class IniHighlighter(QSyntaxHighlighter):
|
|||
self.highlightingRules.append( self.teststoryRule )
|
||||
|
||||
# storyUrl sections
|
||||
# StoryUrls are *not* checked beyond looking for https?://
|
||||
self.storyUrlRule = HighlightingRule( r"^\[https?://.*\]", colors['storyUrls'], QFontBold, blocknum=2 )
|
||||
self.storyUrlRule = HighlightingRule( r"^\[https?://.*\]", colors['storyUrls'], blocknum=4 )
|
||||
self.highlightingRules.append( self.storyUrlRule )
|
||||
|
||||
# NOT comments -- but can be custom columns, so don't flag.
|
||||
|
|
@ -135,10 +134,9 @@ class IniHighlighter(QSyntaxHighlighter):
|
|||
if blocknum == 3:
|
||||
self.setFormat( 0, len(text), self.teststoryRule.highlight )
|
||||
|
||||
## changed storyUrl section to also be blocknum=1 April 2023
|
||||
## storyUrl section rules:
|
||||
# if blocknum == 4:
|
||||
# self.setFormat( 0, len(text), self.storyUrlRule.highlight )
|
||||
# storyUrl section rules:
|
||||
if blocknum == 4:
|
||||
self.setFormat( 0, len(text), self.storyUrlRule.highlight )
|
||||
|
||||
self.setCurrentBlockState( blocknum )
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
import six
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2020, Jim Miller, 2011, Grant Drake <grant.drake@gmail.com>'
|
||||
|
|
@ -14,8 +15,10 @@ from time import sleep
|
|||
from datetime import time
|
||||
from io import StringIO
|
||||
from collections import defaultdict
|
||||
import sys
|
||||
|
||||
from calibre.utils.ipc.server import Empty, Server
|
||||
from calibre.utils.ipc.job import ParallelJob
|
||||
from calibre.constants import numeric_version as calibre_version
|
||||
from calibre.utils.date import local_tz
|
||||
|
||||
# pulls in translation files for _() strings
|
||||
|
|
@ -30,100 +33,160 @@ except NameError:
|
|||
#
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
def do_download_worker_single(site,
|
||||
book_list,
|
||||
options,
|
||||
merge,
|
||||
notification=lambda x,y:x):
|
||||
def do_download_worker(book_list,
|
||||
options,
|
||||
cpus,
|
||||
merge=False,
|
||||
notification=lambda x,y:x):
|
||||
'''
|
||||
Coordinator job, to launch child jobs to do downloads.
|
||||
This is run as a worker job in the background to keep the UI more
|
||||
responsive and get around any memory leak issues as it will launch
|
||||
a child job for each book as a worker process
|
||||
'''
|
||||
## Now running one BG proc per site, which downloads for the same
|
||||
## site in serial.
|
||||
logger.info("CPUs:%s"%cpus)
|
||||
server = Server(pool_size=cpus)
|
||||
|
||||
logger.info(options['version'])
|
||||
|
||||
## same info debug calibre prints out at startup. For when users
|
||||
## give me job output instead of debug log.
|
||||
from calibre.debug import print_basic_debug_info
|
||||
print_basic_debug_info(sys.stderr)
|
||||
sites_lists = defaultdict(list)
|
||||
[ sites_lists[x['site']].append(x) for x in book_list if x['good'] ]
|
||||
|
||||
totals = {}
|
||||
# can't do direct assignment in list comprehension? I'm sure it
|
||||
# makes sense to some pythonista.
|
||||
# [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
|
||||
[ totals.update({x['url']:0.0}) for x in book_list if x['good'] ]
|
||||
# logger.debug(sites_lists.keys())
|
||||
|
||||
# Queue all the jobs
|
||||
jobs_running = 0
|
||||
for site in sites_lists.keys():
|
||||
site_list = sites_lists[site]
|
||||
logger.info(_("Launch background process for site %s:")%site + "\n" +
|
||||
"\n".join([ x['url'] for x in site_list ]))
|
||||
# logger.debug([ x['url'] for x in site_list])
|
||||
args = ['calibre_plugins.fanficfare_plugin.jobs',
|
||||
'do_download_site',
|
||||
(site,site_list,options,merge)]
|
||||
job = ParallelJob('arbitrary_n',
|
||||
"site:(%s)"%site,
|
||||
done=None,
|
||||
args=args)
|
||||
job._site_list = site_list
|
||||
job._processed = False
|
||||
server.add_job(job)
|
||||
jobs_running += 1
|
||||
|
||||
# This server is an arbitrary_n job, so there is a notifier available.
|
||||
# Set the % complete to a small number to avoid the 'unavailable' indicator
|
||||
notification(0.01, _('Downloading FanFiction Stories'))
|
||||
from calibre_plugins.fanficfare_plugin import FanFicFareBase
|
||||
fffbase = FanFicFareBase(options['plugin_path'])
|
||||
with fffbase: # so the sys.path was modified while loading the
|
||||
# plug impl.
|
||||
from fanficfare.fff_profile import do_cprofile
|
||||
|
||||
## extra function just so I can easily use the same
|
||||
## @do_cprofile decorator
|
||||
@do_cprofile
|
||||
def profiled_func():
|
||||
count = 0
|
||||
totals = {}
|
||||
# can't do direct assignment in list comprehension? I'm sure it
|
||||
# makes sense to some pythonista.
|
||||
# [ totals[x['url']]=0.0 for x in book_list if x['good'] ]
|
||||
[ totals.update({x['url']:0.0}) for x in book_list if x['good'] ]
|
||||
# logger.debug(sites_lists.keys())
|
||||
|
||||
def do_indiv_notif(percent,msg):
|
||||
# dequeue the job results as they arrive, saving the results
|
||||
count = 0
|
||||
while True:
|
||||
job = server.changed_jobs_queue.get()
|
||||
# logger.debug("job get job._processed:%s"%job._processed)
|
||||
# A job can 'change' when it is not finished, for example if it
|
||||
# produces a notification.
|
||||
msg = None
|
||||
try:
|
||||
## msg = book['url']
|
||||
(percent,msg) = job.notifications.get_nowait()
|
||||
# logger.debug("%s<-%s"%(percent,msg))
|
||||
if percent == 10.0: # Only when signaling d/l done.
|
||||
count += 1
|
||||
totals[msg] = 1.0/len(totals)
|
||||
# logger.info("Finished: %s"%msg)
|
||||
else:
|
||||
totals[msg] = percent/len(totals)
|
||||
notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
|
||||
notification(max(0.01,sum(totals.values())), _('%(count)d of %(total)d stories finished downloading')%{'count':count,'total':len(totals)})
|
||||
except Empty:
|
||||
pass
|
||||
# without update, is_finished will never be set. however, we
|
||||
# do want to get all the notifications for status so we don't
|
||||
# miss the 'done' ones.
|
||||
job.update(consume_notifications=False)
|
||||
|
||||
do_list = []
|
||||
done_list = []
|
||||
logger.info("\n\n"+_("Downloading FanFiction Stories")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
|
||||
## pass failures from metadata through bg job so all results are
|
||||
## together.
|
||||
# if not job._processed:
|
||||
# sleep(0.5)
|
||||
## Can have a race condition where job.is_finished before
|
||||
## notifications for all downloads have been processed.
|
||||
## Or even after the job has been finished.
|
||||
# logger.debug("job.is_finished(%s) or job._processed(%s)"%(job.is_finished, job._processed))
|
||||
if not job.is_finished:
|
||||
continue
|
||||
|
||||
## only process each job once. We can get more than one loop
|
||||
## after job.is_finished.
|
||||
if not job._processed:
|
||||
# sleep(1)
|
||||
# A job really finished. Get the information.
|
||||
|
||||
## This is where bg proc details end up in GUI log.
|
||||
## job.details is the whole debug log for each proc.
|
||||
logger.info("\n\n" + ("="*80) + " " + job.details.replace('\r',''))
|
||||
# logger.debug("Finished background process for site %s:\n%s"%(job._site_list[0]['site'],"\n".join([ x['url'] for x in job._site_list ])))
|
||||
for b in job._site_list:
|
||||
book_list.remove(b)
|
||||
book_list.extend(job.result)
|
||||
job._processed = True
|
||||
jobs_running -= 1
|
||||
|
||||
## Can't use individual count--I've seen stories all reported
|
||||
## finished before results of all jobs processed.
|
||||
if jobs_running == 0:
|
||||
book_list = sorted(book_list,key=lambda x : x['listorder'])
|
||||
logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
|
||||
|
||||
good_lists = defaultdict(list)
|
||||
bad_lists = defaultdict(list)
|
||||
for book in book_list:
|
||||
if book['good']:
|
||||
do_list.append(book)
|
||||
good_lists[book['status']].append(book)
|
||||
else:
|
||||
done_list.append(book)
|
||||
for book in do_list:
|
||||
# logger.info("%s"%book['url'])
|
||||
done_list.append(do_download_for_worker(book,options,merge,do_indiv_notif))
|
||||
count += 1
|
||||
return finish_download(done_list)
|
||||
return profiled_func()
|
||||
bad_lists[book['status']].append(book)
|
||||
|
||||
def finish_download(donelist):
|
||||
book_list = sorted(donelist,key=lambda x : x['listorder'])
|
||||
logger.info("\n"+_("Download Results:")+"\n%s\n"%("\n".join([ "%(status)s %(url)s %(comment)s" % book for book in book_list])))
|
||||
order = [_('Add'),
|
||||
_('Update'),
|
||||
_('Meta'),
|
||||
_('Different URL'),
|
||||
_('Rejected'),
|
||||
_('Skipped'),
|
||||
_('Bad'),
|
||||
_('Error'),
|
||||
]
|
||||
j = 0
|
||||
for d in [ good_lists, bad_lists ]:
|
||||
for status in order:
|
||||
if d[status]:
|
||||
l = d[status]
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in l])))
|
||||
for book in l:
|
||||
book['reportorder'] = j
|
||||
j += 1
|
||||
del d[status]
|
||||
# just in case a status is added but doesn't appear in order.
|
||||
for status in d.keys():
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in d[status]])))
|
||||
break
|
||||
|
||||
good_lists = defaultdict(list)
|
||||
bad_lists = defaultdict(list)
|
||||
for book in book_list:
|
||||
if book['good']:
|
||||
good_lists[book['status']].append(book)
|
||||
else:
|
||||
bad_lists[book['status']].append(book)
|
||||
|
||||
order = [_('Add'),
|
||||
_('Update'),
|
||||
_('Meta'),
|
||||
_('Different URL'),
|
||||
_('Rejected'),
|
||||
_('Skipped'),
|
||||
_('Bad'),
|
||||
_('Error'),
|
||||
]
|
||||
stnum = 0
|
||||
for d in [ good_lists, bad_lists ]:
|
||||
for status in order:
|
||||
stnum += 1
|
||||
if d[status]:
|
||||
l = d[status]
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in l])))
|
||||
for book in l:
|
||||
# Add prior listorder to 10000 * status num for
|
||||
# ordering of accumulated results with multiple bg
|
||||
# jobs
|
||||
book['reportorder'] = stnum*10000 + book['listorder']
|
||||
del d[status]
|
||||
# just in case a status is added but doesn't appear in order.
|
||||
for status in d.keys():
|
||||
logger.info("\n"+status+"\n%s\n"%("\n".join([book['url'] for book in d[status]])))
|
||||
server.close()
|
||||
|
||||
# return the book list as the job result
|
||||
return book_list
|
||||
|
||||
def do_download_site(site,book_list,options,merge,notification=lambda x,y:x):
|
||||
# logger.info(_("Started job for %s")%site)
|
||||
retval = []
|
||||
for book in book_list:
|
||||
# logger.info("%s"%book['url'])
|
||||
retval.append(do_download_for_worker(book,options,merge,notification))
|
||||
notification(10.0,book['url'])
|
||||
return retval
|
||||
|
||||
def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
||||
'''
|
||||
Child job, to download story when run as a worker job
|
||||
|
|
@ -133,13 +196,13 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
fffbase = FanFicFareBase(options['plugin_path'])
|
||||
with fffbase: # so the sys.path was modified while loading the
|
||||
# plug impl.
|
||||
from calibre_plugins.fanficfare_plugin.dialogs import NotGoingToDownload
|
||||
from calibre_plugins.fanficfare_plugin.prefs import (
|
||||
SAVE_YES, SAVE_YES_UNLESS_SITE, OVERWRITE, OVERWRITEALWAYS, UPDATE,
|
||||
UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
|
||||
from calibre_plugins.fanficfare_plugin.wordcount import get_word_count
|
||||
from fanficfare import adapters, writers
|
||||
from fanficfare.epubutils import get_update_data
|
||||
from fanficfare.exceptions import NotGoingToDownload
|
||||
from fanficfare.six import text_type as unicode
|
||||
|
||||
from calibre_plugins.fanficfare_plugin.fff_util import get_fff_config
|
||||
|
|
@ -159,6 +222,9 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
options['fileform'],
|
||||
options['personal.ini'])
|
||||
|
||||
if not options['updateepubcover'] and 'epub_for_update' in book and book['collision'] in (UPDATE, UPDATEALWAYS):
|
||||
configuration.set("overrides","never_make_cover","true")
|
||||
|
||||
# images only for epub, html, even if the user mistakenly
|
||||
# turned it on else where.
|
||||
if options['fileform'] not in ("epub","html"):
|
||||
|
|
@ -168,12 +234,18 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
adapter.is_adult = book['is_adult']
|
||||
adapter.username = book['username']
|
||||
adapter.password = book['password']
|
||||
adapter.totp = book['totp']
|
||||
adapter.setChaptersRange(book['begin'],book['end'])
|
||||
|
||||
## each site download job starts with a new copy of the
|
||||
## cookiejar and basic_cache from the FG process. They
|
||||
## are not shared between different sites' BG downloads
|
||||
if configuration.getConfig('use_browser_cache'):
|
||||
if 'browser_cache' in options:
|
||||
configuration.set_browser_cache(options['browser_cache'])
|
||||
else:
|
||||
options['browser_cache'] = configuration.get_browser_cache()
|
||||
if 'browser_cachefile' in options:
|
||||
options['browser_cache'].load_cache(options['browser_cachefile'])
|
||||
if 'basic_cache' in options:
|
||||
configuration.set_basic_cache(options['basic_cache'])
|
||||
else:
|
||||
|
|
@ -189,17 +261,6 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
if not story.getMetadata("series") and 'calibre_series' in book:
|
||||
adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
|
||||
|
||||
# logger.debug(merge)
|
||||
# logger.debug(book.get('epub_for_update','(NONE)'))
|
||||
# logger.debug(options.get('mergebook','(NOMERGEBOOK)'))
|
||||
|
||||
# is a merge, is a pre-existing anthology, and is not a pre-existing book in anthology.
|
||||
if merge and 'mergebook' in options and 'epub_for_update' not in book:
|
||||
# internal for plugin anthologies to mark chapters
|
||||
# (new) in new stories
|
||||
story.setMetadata("newforanthology","true")
|
||||
logger.debug("metadata newforanthology:%s"%story.getMetadata("newforanthology"))
|
||||
|
||||
# set PI version instead of default.
|
||||
if 'version' in options:
|
||||
story.setMetadata('version',options['version'])
|
||||
|
|
@ -208,6 +269,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
|
||||
book['publisher'] = story.getMetadata("publisher")
|
||||
book['url'] = story.getMetadata("storyUrl", removeallentities=True)
|
||||
book['tags'] = story.getSubjectTags(removeallentities=True)
|
||||
book['comments'] = story.get_sanitized_description()
|
||||
book['series'] = story.getMetadata("series", removeallentities=True)
|
||||
|
||||
|
|
@ -284,21 +346,20 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
adapter.oldchaptersmap,
|
||||
adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
|
||||
|
||||
# dup handling from fff_plugin needed for anthology updates & BG metadata.
|
||||
if book['collision'] in (UPDATE,UPDATEALWAYS):
|
||||
if chaptercount == urlchaptercount and book['collision'] == UPDATE:
|
||||
# dup handling from fff_plugin needed for anthology updates.
|
||||
if book['collision'] == UPDATE:
|
||||
if chaptercount == urlchaptercount:
|
||||
if merge:
|
||||
## Deliberately pass for UPDATEALWAYS merge.
|
||||
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
|
||||
return book
|
||||
else:
|
||||
else: # not merge,
|
||||
raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
|
||||
elif chaptercount > urlchaptercount and not (book['collision'] == UPDATEALWAYS and adapter.getConfig('force_update_epub_always')):
|
||||
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite or force_update_epub_always to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
|
||||
elif chaptercount > urlchaptercount:
|
||||
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
|
||||
elif chaptercount == 0:
|
||||
raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
|
||||
|
||||
|
|
@ -336,11 +397,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
options['do_wordcount'] == SAVE_YES_UNLESS_SITE and not story.getMetadataRaw('numWords') ):
|
||||
try:
|
||||
wordcount = get_word_count(outfile)
|
||||
# logger.info("get_word_count:%s"%wordcount)
|
||||
# clear cache for the rather unusual case of
|
||||
# numWords affecting other previously cached
|
||||
# entries.
|
||||
story.clear_processed_metadata_cache()
|
||||
# logger.info("get_word_count:%s"%wordcount)
|
||||
story.setMetadata('numWords',wordcount)
|
||||
writer.writeStory(outfilename=outfile, forceOverwrite=True)
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
|
|
@ -349,7 +406,8 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
except:
|
||||
logger.error("WordCount failed")
|
||||
|
||||
if options['smarten_punctuation'] and options['fileform'] == "epub":
|
||||
if options['smarten_punctuation'] and options['fileform'] == "epub" \
|
||||
and calibre_version >= (0, 9, 39):
|
||||
# for smarten punc
|
||||
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
|
||||
from calibre.utils.logging import Log
|
||||
|
|
@ -359,14 +417,12 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
|
|||
data = {'smarten_punctuation':True}
|
||||
opts = ALL_OPTS.copy()
|
||||
opts.update(data)
|
||||
O = namedtuple('Options', ' '.join(ALL_OPTS.keys()))
|
||||
O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
|
||||
opts = O(**opts)
|
||||
|
||||
log = Log(level=Log.DEBUG)
|
||||
polish({outfile:outfile}, opts, log, logger.info)
|
||||
## here to catch tags set in chapters in literotica for
|
||||
## both overwrites and updates.
|
||||
book['tags'] = story.getSubjectTags(removeallentities=True)
|
||||
|
||||
except NotGoingToDownload as d:
|
||||
book['good']=False
|
||||
book['status']=_('Bad')
|
||||
|
|
@ -392,12 +448,11 @@ def inject_cal_cols(book,story,configuration):
|
|||
if 'calibre_columns' in book:
|
||||
injectini = ['[injected]']
|
||||
extra_valid = []
|
||||
for k in book['calibre_columns'].keys():
|
||||
v = book['calibre_columns'][k]
|
||||
for k, v in six.iteritems(book['calibre_columns']):
|
||||
story.setMetadata(k,v['val'])
|
||||
injectini.append('%s_label:%s'%(k,v['label']))
|
||||
extra_valid.append(k)
|
||||
if extra_valid: # if empty, there's nothing to add.
|
||||
injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
|
||||
configuration.read_file(StringIO('\n'.join(injectini)))
|
||||
configuration.readfp(StringIO('\n'.join(injectini)))
|
||||
#print("added:\n%s\n"%('\n'.join(injectini)))
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -3,9 +3,22 @@
|
|||
|
||||
[defaults]
|
||||
## [defaults] section applies to all formats and sites but may be
|
||||
## overridden at several levels. See
|
||||
## https://github.com/JimmXinu/FanFicFare/wiki/INI-File for more
|
||||
## details.
|
||||
## overridden at several levels. Example:
|
||||
|
||||
## [defaults]
|
||||
## titlepage_entries: category,genre, status
|
||||
## [www.whofic.com]
|
||||
## # overrides defaults.
|
||||
## titlepage_entries: category,genre, status,dateUpdated,rating
|
||||
## [epub]
|
||||
## # overrides defaults & site section
|
||||
## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
|
||||
## [www.whofic.com:epub]
|
||||
## # overrides defaults, site section & format section
|
||||
## titlepage_entries: category,genre, status,datePublished
|
||||
## [overrides]
|
||||
## # overrides all other sections
|
||||
## titlepage_entries: category
|
||||
|
||||
## Some sites also require the user to confirm they are adult for
|
||||
## adult content. Uncomment by removing '#' in front of is_adult.
|
||||
|
|
@ -16,32 +29,42 @@
|
|||
## want to make them all look the same? Strip them off, then add them
|
||||
## back on with add_chapter_numbers. Don't like the way it strips
|
||||
## numbers or adds them back? See chapter_title_strip_pattern and
|
||||
## chapter_title_add_pattern in defaults.ini.
|
||||
## chapter_title_add_pattern.
|
||||
#strip_chapter_numbers:true
|
||||
#add_chapter_numbers:true
|
||||
|
||||
## Add this to genre if there's more than one category.
|
||||
#add_genre_when_multi_category: Crossover
|
||||
|
||||
[epub]
|
||||
## Include images from img tags in the body and summary of stories.
|
||||
## include images from img tags in the body and summary of stories.
|
||||
## Images will be converted to jpg for size if possible. Images work
|
||||
## in epub format only. To get mobi or other format with images,
|
||||
## download as epub and use Calibre to convert.
|
||||
## true by default, uncomment and set false to not include images.
|
||||
#include_images:true
|
||||
|
||||
## If set false, the summary will have all html stripped for safety.
|
||||
## Quality level to use when converting images to jpg. Range is 0-100,
|
||||
## reasonable values likely to be in the range 70-95.
|
||||
#jpg_quality: 95
|
||||
|
||||
## If not set, the summary will have all html stripped for safety.
|
||||
## Both this and include_images must be true to get images in the
|
||||
## summary.
|
||||
## true by default, uncomment and set false to not keep summary html.
|
||||
#keep_summary_html:true
|
||||
|
||||
## If set true, and there isn't a specific cover image, the first
|
||||
## image found in the story will be made the cover image. If
|
||||
## keep_summary_html is true, images in the summary will be before any
|
||||
## If set, the first image found will be made the cover image. If
|
||||
## keep_summary_html is true, any images in summary will be before any
|
||||
## in chapters.
|
||||
## true by default, uncomment and set false to turn off
|
||||
#make_firstimage_cover:true
|
||||
|
||||
## Resize images down to width, height, preserving aspect ratio.
|
||||
## Nook size, with margin.
|
||||
#image_max_size: 580, 725
|
||||
|
||||
## Change image to grayscale, if graphics library allows, to save
|
||||
## space.
|
||||
#grayscale_images: false
|
||||
|
||||
|
||||
## Most common, I expect will be using this to save username/passwords
|
||||
## for different sites. Here are a few examples. See defaults.ini
|
||||
|
|
@ -53,6 +76,22 @@
|
|||
## default is false
|
||||
#collect_series: true
|
||||
|
||||
[ficwad.com]
|
||||
#username:YourUsername
|
||||
#password:YourPassword
|
||||
|
||||
[www.adastrafanfic.com]
|
||||
## Some sites do not require a login, but do require the user to
|
||||
## confirm they are adult for adult content.
|
||||
#is_adult:true
|
||||
|
||||
[www.twcslibrary.net]
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
#is_adult:true
|
||||
## default is false
|
||||
#collect_series: true
|
||||
|
||||
[www.fimfiction.net]
|
||||
#is_adult:true
|
||||
#fail_on_password: false
|
||||
|
|
@ -61,9 +100,8 @@
|
|||
#is_adult:true
|
||||
## tth is a little unusual--it doesn't require user/pass, but the site
|
||||
## keeps track of which chapters you've read and won't send another
|
||||
## update until it thinks you're up to date. If you set
|
||||
## username/password, FFF will login to download. Then the site
|
||||
## thinks you're up to date.
|
||||
## update until it thinks you're up to date. This way, on download,
|
||||
## it thinks you're up to date.
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
|
|
|
|||
|
|
@ -120,13 +120,12 @@ default_prefs['reject_delete_default'] = True
|
|||
|
||||
default_prefs['updatemeta'] = True
|
||||
default_prefs['bgmeta'] = False
|
||||
#default_prefs['updateepubcover'] = True # removed in favor of always True Oct 2022
|
||||
default_prefs['updateepubcover'] = False
|
||||
default_prefs['keeptags'] = False
|
||||
default_prefs['suppressauthorsort'] = False
|
||||
default_prefs['suppresstitlesort'] = False
|
||||
default_prefs['authorcase'] = False
|
||||
default_prefs['titlecase'] = False
|
||||
default_prefs['seriescase'] = False
|
||||
default_prefs['setanthologyseries'] = False
|
||||
default_prefs['mark'] = False
|
||||
default_prefs['mark_success'] = True
|
||||
|
|
@ -160,12 +159,11 @@ default_prefs['addtolistsonread'] = False
|
|||
default_prefs['autounnew'] = False
|
||||
|
||||
default_prefs['updatecalcover'] = SAVE_YES_IF_IMG
|
||||
default_prefs['covernewonly'] = False
|
||||
default_prefs['gencalcover'] = SAVE_YES_UNLESS_IMG
|
||||
default_prefs['gencalcover'] = SAVE_YES
|
||||
default_prefs['updatecover'] = False
|
||||
default_prefs['calibre_gen_cover'] = True
|
||||
default_prefs['plugin_gen_cover'] = False
|
||||
default_prefs['gcnewonly'] = True
|
||||
default_prefs['calibre_gen_cover'] = False
|
||||
default_prefs['plugin_gen_cover'] = True
|
||||
default_prefs['gcnewonly'] = False
|
||||
default_prefs['gc_site_settings'] = {}
|
||||
default_prefs['allow_gc_from_ini'] = True
|
||||
default_prefs['gc_polish_cover'] = False
|
||||
|
|
@ -183,7 +181,6 @@ default_prefs['allow_custcol_from_ini'] = True
|
|||
|
||||
default_prefs['std_cols_newonly'] = {}
|
||||
default_prefs['set_author_url'] = True
|
||||
default_prefs['set_series_url'] = True
|
||||
default_prefs['includecomments'] = False
|
||||
default_prefs['anth_comments_newonly'] = True
|
||||
|
||||
|
|
@ -198,11 +195,6 @@ default_prefs['auto_reject_from_email'] = False
|
|||
default_prefs['update_existing_only_from_email'] = False
|
||||
default_prefs['download_from_email_immediately'] = False
|
||||
|
||||
|
||||
#default_prefs['single_proc_jobs'] = True # setting and code removed
|
||||
default_prefs['site_split_jobs'] = True
|
||||
default_prefs['reconsolidate_jobs'] = True
|
||||
|
||||
def set_library_config(library_config,db,setting=PREFS_KEY_SETTINGS):
|
||||
db.prefs.set_namespaced(PREFS_NAMESPACE,
|
||||
setting,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -30,12 +30,8 @@ from .. import configurable as configurable
|
|||
|
||||
## must import each adapter here.
|
||||
|
||||
from . import base_adapter
|
||||
from . import base_efiction_adapter
|
||||
from . import adapter_test1
|
||||
from . import adapter_test2
|
||||
from . import adapter_test3
|
||||
from . import adapter_test4
|
||||
from . import adapter_fanfictionnet
|
||||
from . import adapter_fictionalleyarchiveorg
|
||||
from . import adapter_fictionpresscom
|
||||
|
|
@ -53,7 +49,9 @@ from . import adapter_archiveofourownorg
|
|||
from . import adapter_ficbooknet
|
||||
from . import adapter_midnightwhispers
|
||||
from . import adapter_ksarchivecom
|
||||
from . import adapter_archiveskyehawkecom
|
||||
from . import adapter_libraryofmoriacom
|
||||
from . import adapter_wraithbaitcom
|
||||
from . import adapter_ashwindersycophanthexcom
|
||||
from . import adapter_chaossycophanthexcom
|
||||
from . import adapter_erosnsapphosycophanthexcom
|
||||
|
|
@ -62,26 +60,37 @@ from . import adapter_occlumencysycophanthexcom
|
|||
from . import adapter_phoenixsongnet
|
||||
from . import adapter_walkingtheplankorg
|
||||
from . import adapter_dokugacom
|
||||
from . import adapter_iketernalnet
|
||||
from . import adapter_storiesofardacom
|
||||
from . import adapter_destinysgatewaycom
|
||||
from . import adapter_ncisfictioncom
|
||||
from . import adapter_fanfiktionde
|
||||
from . import adapter_ponyfictionarchivenet
|
||||
from . import adapter_themasquenet
|
||||
from . import adapter_pretendercentrecom
|
||||
from . import adapter_darksolaceorg
|
||||
from . import adapter_storyroomcom
|
||||
from . import adapter_finestoriescom
|
||||
from . import adapter_hlfictionnet
|
||||
from . import adapter_dracoandginnycom
|
||||
from . import adapter_scarvesandcoffeenet
|
||||
from . import adapter_wolverineandroguecom
|
||||
from . import adapter_merlinficdtwinscouk
|
||||
from . import adapter_thehookupzonenet
|
||||
from . import adapter_qafficcom
|
||||
from . import adapter_efpfanficnet
|
||||
from . import adapter_imagineeficcom
|
||||
from . import adapter_potterheadsanonymouscom
|
||||
from . import adapter_storiesonlinenet
|
||||
from . import adapter_trekiverseorg
|
||||
from . import adapter_literotica
|
||||
from . import adapter_voracity2eficcom
|
||||
from . import adapter_spikeluvercom
|
||||
from . import adapter_bloodshedversecom
|
||||
from . import adapter_fictionmaniatv
|
||||
from . import adapter_themaplebookshelf
|
||||
from . import adapter_sheppardweircom
|
||||
from . import adapter_samandjacknet
|
||||
from . import adapter_csiforensicscom
|
||||
from . import adapter_tgstorytimecom
|
||||
from . import adapter_forumsspacebattlescom
|
||||
from . import adapter_forumssufficientvelocitycom
|
||||
|
|
@ -90,6 +99,8 @@ from . import adapter_ninelivesarchivecom
|
|||
from . import adapter_masseffect2in
|
||||
from . import adapter_quotevcom
|
||||
from . import adapter_mcstoriescom
|
||||
from . import adapter_buffygilescom
|
||||
from . import adapter_andromedawebcom
|
||||
from . import adapter_naiceanilmenet
|
||||
from . import adapter_adultfanfictionorg
|
||||
from . import adapter_fictionhuntcom
|
||||
|
|
@ -99,48 +110,56 @@ from . import adapter_bdsmlibrarycom
|
|||
from . import adapter_asexstoriescom
|
||||
from . import adapter_gluttonyfictioncom
|
||||
from . import adapter_valentchambercom
|
||||
from . import adapter_looselugscom
|
||||
from . import adapter_wwwgiantessworldnet
|
||||
from . import adapter_lotrgficcom
|
||||
from . import adapter_sugarquillnet
|
||||
from . import adapter_starslibrarynet
|
||||
from . import adapter_fanficauthorsnet
|
||||
from . import adapter_fireflyfansnet
|
||||
from . import adapter_shriftweborgbfa
|
||||
from . import adapter_trekfanfictionnet
|
||||
from . import adapter_wwwlushstoriescom
|
||||
from . import adapter_wwwutopiastoriescom
|
||||
from . import adapter_sinfuldreamscomunicornfic
|
||||
from . import adapter_sinfuldreamscomwhisperedmuse
|
||||
from . import adapter_sinfuldreamscomwickedtemptation
|
||||
from . import adapter_asianfanficscom
|
||||
from . import adapter_mttjustoncenet
|
||||
from . import adapter_narutoficorg
|
||||
from . import adapter_starskyhutcharchivenet
|
||||
from . import adapter_swordborderlineangelcom
|
||||
from . import adapter_tasteofpoisoninkubationnet
|
||||
from . import adapter_thedelphicexpansecom
|
||||
from . import adapter_wwwaneroticstorycom
|
||||
from . import adapter_lcfanficcom
|
||||
from . import adapter_noveltrovecom
|
||||
from . import adapter_inkbunnynet
|
||||
from . import adapter_alternatehistorycom
|
||||
from . import adapter_wattpadcom
|
||||
from . import adapter_novelonlinefullcom
|
||||
from . import adapter_wwwnovelallcom
|
||||
from . import adapter_wuxiaworldco
|
||||
from . import adapter_novelupdatescc
|
||||
from . import adapter_hentaifoundrycom
|
||||
from . import adapter_mugglenetfanfictioncom
|
||||
from . import adapter_swiorgru
|
||||
from . import adapter_fanficsme
|
||||
from . import adapter_fanfictalkcom
|
||||
from . import adapter_scifistoriescom
|
||||
from . import adapter_silmarillionwritersguildorg
|
||||
from . import adapter_chireadscom
|
||||
from . import adapter_scribblehubcom
|
||||
from . import adapter_fictionlive
|
||||
from . import adapter_thesietchcom
|
||||
from . import adapter_fastnovelsnet
|
||||
from . import adapter_squidgeworldorg
|
||||
from . import adapter_novelfull
|
||||
from . import adapter_worldofxde
|
||||
from . import adapter_psychficcom
|
||||
from . import adapter_deviantartcom
|
||||
from . import adapter_merengohu
|
||||
from . import adapter_readonlymindcom
|
||||
from . import adapter_wwwsunnydaleafterdarkcom
|
||||
from . import adapter_syosetucom
|
||||
from . import adapter_kakuyomujp
|
||||
from . import adapter_fanfictionsfr
|
||||
from . import adapter_touchfluffytail
|
||||
from . import adapter_spiritfanfictioncom
|
||||
from . import adapter_superlove
|
||||
from . import adapter_cfaa
|
||||
from . import adapter_althistorycom
|
||||
|
||||
## This bit of complexity allows adapters to be added by just adding
|
||||
## importing. It eliminates the long if/else clauses we used to need
|
||||
|
|
@ -224,21 +243,6 @@ def get_section_url(url):
|
|||
## return unchanged in that case.
|
||||
return url
|
||||
|
||||
def get_url_search(url):
|
||||
'''
|
||||
For adapters that have story URLs that can change. This is
|
||||
used for searching the Calibre library by identifiers:url for
|
||||
sites (generally) that contain author or title that can
|
||||
change, but also have a unique identifier that doesn't.
|
||||
|
||||
returns a string containing a regexp, not a compiled re object.
|
||||
'''
|
||||
cls = _get_class_for(url)[0]
|
||||
if not cls:
|
||||
## still apply common processing.
|
||||
cls = base_adapter.BaseSiteAdapter
|
||||
return cls.get_url_search(url)
|
||||
|
||||
def getAdapter(config,url,anyurl=False):
|
||||
|
||||
#logger.debug("trying url:"+url)
|
||||
|
|
|
|||
|
|
@ -15,24 +15,201 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
|
||||
from .base_otw_adapter import BaseOTWAdapter
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
def getClass():
|
||||
return AdastrafanficComAdapter
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
class AdastrafanficComAdapter(BaseOTWAdapter):
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseOTWAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
self.story.setMetadata('siteabbrev','aaff')
|
||||
self.is_adult=False
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.adastrafanfic.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
addurl = "&warning=5"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
# problems with some stories, but only in calibre. I suspect
|
||||
# issues with different SGML parsers in python. This is a
|
||||
# nasty hack, but it works.
|
||||
data = data[data.index("<body"):]
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
## <meta name='description' content='<p>Description</p> ...' >
|
||||
## Summary, strangely, is in the content attr of a <meta name='description'> tag
|
||||
## which is escaped HTML. Unfortunately, we can't use it because they don't
|
||||
## escape (') chars in the desc, breakin the tag.
|
||||
#meta_desc = soup.find('meta',{'name':'description'})
|
||||
#metasoup = bs.BeautifulStoneSoup(meta_desc['content'])
|
||||
#self.story.setMetadata('description',stripHTML(metasoup))
|
||||
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ''
|
||||
while value and 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
# sometimes poorly formated desc (<p> w/o </p>) leads
|
||||
# to all labels being included.
|
||||
svalue=svalue[:svalue.find('<span class="label">')]
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warningstext = [warning.string for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
data = self.get_request(url)
|
||||
# problems with some stories, but only in calibre. I suspect
|
||||
# issues with different SGML parsers in python. This is a
|
||||
# nasty hack, but it works.
|
||||
data = data[data.index("<body"):]
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
span = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == span:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,span)
|
||||
|
||||
def getClass():
|
||||
return AdAstraFanficComSiteAdapter
|
||||
|
|
|
|||
|
|
@ -57,8 +57,8 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
# normalized story URL.(checking self.zone against list
|
||||
# removed--it was redundant w/getAcceptDomains and
|
||||
# getSiteURLPattern both)
|
||||
self._setURL('https://{0}.{1}/story.php?no={2}'.format(self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
|
||||
#self._setURL('https://' + self.zone + '.' + self.getBaseDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
|
||||
self._setURL('http://{0}.{1}/story.php?no={2}'.format(self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
|
||||
#self._setURL('http://' + self.zone + '.' + self.getBaseDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
#self.story.setMetadata('siteabbrev',self.getSiteAbbrev())
|
||||
|
|
@ -68,7 +68,9 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%B %d, %Y"
|
||||
self.dateformat = "%Y-%m-%d"
|
||||
|
||||
|
||||
|
||||
## Added because adult-fanfiction.org does send you to
|
||||
## www.adult-fanfiction.org when you go to it and it also moves
|
||||
|
|
@ -111,31 +113,79 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(self):
|
||||
return ("https://anime.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://anime2.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://bleach.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://books.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://buffy.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://cartoon.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://celeb.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://comics.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://ff.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://games.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://hp.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://inu.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://lotr.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://manga.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://movies.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://naruto.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://ne.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://original.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://tv.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://xmen.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://ygo.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "https://yuyu.adult-fanfiction.org/story.php?no=123456789")
|
||||
return ("http://anime.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://anime2.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://bleach.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://books.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://buffy.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://cartoon.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://celeb.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://comics.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://ff.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://games.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://hp.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://inu.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://lotr.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://manga.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://movies.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://naruto.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://ne.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://original.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://tv.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://xmen.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://ygo.adult-fanfiction.org/story.php?no=123456789 "
|
||||
+ "http://yuyu.adult-fanfiction.org/story.php?no=123456789")
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r'https?://(anime|anime2|bleach|books|buffy|cartoon|celeb|comics|ff|games|hp|inu|lotr|manga|movies|naruto|ne|original|tv|xmen|ygo|yuyu)\.adult-fanfiction\.org/story\.php\?no=\d+$'
|
||||
return r'http?://(anime|anime2|bleach|books|buffy|cartoon|celeb|comics|ff|games|hp|inu|lotr|manga|movies|naruto|ne|original|tv|xmen|ygo|yuyu)\.adult-fanfiction\.org/story\.php\?no=\d+$'
|
||||
|
||||
##This is not working right now, so I'm commenting it out, but leaving it for future testing
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
#def needToLoginCheck(self, data):
|
||||
##This adapter will always require a login
|
||||
# return True
|
||||
|
||||
# <form name="login" method="post" action="">
|
||||
# <div class="top">E-mail: <span id="sprytextfield1">
|
||||
# <input name="email" type="text" id="email" size="20" maxlength="255" />
|
||||
# <span class="textfieldRequiredMsg">Email is required.</span><span class="textfieldInvalidFormatMsg">Invalid E-mail.</span></span></div>
|
||||
# <div class="top">Password: <span id="sprytextfield2">
|
||||
# <input name="pass1" type="password" id="pass1" size="20" maxlength="32" />
|
||||
# <span class="textfieldRequiredMsg">password is required.</span><span class="textfieldMinCharsMsg">Minimum 8 characters8.</span><span class="textfieldMaxCharsMsg">Exceeded 32 characters.</span></span></div>
|
||||
# <div class="top"><br /> <input name="loginsubmittop" type="hidden" id="loginsubmit" value="TRUE" />
|
||||
# <input type="submit" value="Login" />
|
||||
# </div>
|
||||
# </form>
|
||||
|
||||
|
||||
##This is not working right now, so I'm commenting it out, but leaving it for future testing
|
||||
#def performLogin(self, url, soup):
|
||||
# params = {}
|
||||
|
||||
# if self.password:
|
||||
# params['email'] = self.username
|
||||
# params['pass1'] = self.password
|
||||
# else:
|
||||
# params['email'] = self.getConfig("username")
|
||||
# params['pass1'] = self.getConfig("password")
|
||||
# params['submit'] = 'Login'
|
||||
|
||||
# # copy all hidden input tags to pick up appropriate tokens.
|
||||
# for tag in soup.findAll('input',{'type':'hidden'}):
|
||||
# params[tag['name']] = tag['value']
|
||||
|
||||
# logger.debug("Will now login to URL {0} as {1} with password: {2}".format(url, params['email'],params['pass1']))
|
||||
|
||||
# d = self.post_request(url, params, usecache=False)
|
||||
# d = self.post_request(url, params, usecache=False)
|
||||
# soup = self.make_soup(d)
|
||||
|
||||
#if not (soup.find('form', {'name' : 'login'}) == None):
|
||||
# logger.info("Failed to login to URL %s as %s" % (url, params['email']))
|
||||
# raise exceptions.FailedToLogin(url,params['email'])
|
||||
# return False
|
||||
#else:
|
||||
# return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def doExtractChapterUrlsAndMetadata(self, get_cover=True):
|
||||
|
|
@ -143,97 +193,173 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
## You need to have your is_adult set to true to get this story
|
||||
if not (self.is_adult or self.getConfig("is_adult")):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
else:
|
||||
d = self.post_request('https://www.adult-fanfiction.org/globals/ajax/age-verify.php', {"verify":"1"})
|
||||
if "Age verified successfully" not in d:
|
||||
raise exceptions.FailedToDownload("Failed to Verify Age: {0}".format(d))
|
||||
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
# logger.debug(data)
|
||||
|
||||
if "The dragons running the back end of the site can not seem to find the story you are looking for." in data:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: The dragons running the back end of the site can not seem to find the story you are looking for.".format(self.zone, self.getBaseDomain()))
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
##This is not working right now, so I'm commenting it out, but leaving it for future testing
|
||||
#self.performLogin(url, soup)
|
||||
|
||||
|
||||
## Title
|
||||
## Some of the titles have a backslash on the story page, but not on the Author's page
|
||||
## So I am removing it from the title, so it can be found on the Author's page further in the code.
|
||||
## Also, some titles may have extra spaces ' ', and the search on the Author's page removes them,
|
||||
## so I have to here as well. I used multiple replaces to make sure, since I did the same below.
|
||||
h1 = soup.find('h1')
|
||||
# logger.debug("Title:%s"%h1)
|
||||
self.story.setMetadata('title',stripHTML(h1).replace('\\','').replace(' ',' ').replace(' ',' ').replace(' ',' ').strip())
|
||||
a = soup.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a).replace('\\','').replace(' ',' ').replace(' ',' ').replace(' ',' ').strip())
|
||||
|
||||
# Find the chapters from first list only
|
||||
chapters = soup.select_one('select.chapter-select').select('option')
|
||||
for chapter in chapters:
|
||||
self.add_chapter(chapter,self.url+'&chapter='+chapter['value'])
|
||||
# Find the chapters:
|
||||
chapters = soup.find('div',{'class':'dropdown-content'})
|
||||
for i, chapter in enumerate(chapters.findAll('a')):
|
||||
self.add_chapter(chapter,self.url+'&chapter='+unicode(i+1))
|
||||
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"profile.php\?id=\d+"))
|
||||
a = soup.find('a', href=re.compile(r"profile.php\?no=\d+"))
|
||||
if a == None:
|
||||
# I know that the original author of fanficfare wants to always have metadata,
|
||||
# but I posit that if the story is there, even if we can't get the metadata from the
|
||||
# author page, the story should still be able to be downloaded, which is what I've done here.
|
||||
self.story.setMetadata('authorId','000000000')
|
||||
self.story.setMetadata('authorUrl','https://www.adult-fanfiction.org')
|
||||
self.story.setMetadata('authorUrl','http://www.adult-fanfiction.org')
|
||||
self.story.setMetadata('author','Unknown')
|
||||
logger.warning('There was no author found for the story... Metadata will not be retreived.')
|
||||
self.setDescription(url,'>>>>>>>>>> No Summary Given, Unknown Author <<<<<<<<<<')
|
||||
self.setDescription(url,'>>>>>>>>>> No Summary Given <<<<<<<<<<')
|
||||
else:
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl',a['href'])
|
||||
self.story.setMetadata('author',stripHTML(a))
|
||||
|
||||
## The story page does not give much Metadata, so we go to
|
||||
## the Author's page. Except it's actually a sub-req for
|
||||
## list of author's stories for that subdomain
|
||||
author_Url = 'https://members.{0}/load-user-stories.php?subdomain={1}&uid={2}'.format(
|
||||
self.getBaseDomain(),
|
||||
self.zone,
|
||||
self.story.getMetadata('authorId'))
|
||||
##The story page does not give much Metadata, so we go to the Author's page
|
||||
|
||||
logger.debug('Getting the load-user-stories page: {0}'.format(author_Url))
|
||||
##Get the first Author page to see if there are multiple pages.
|
||||
##AFF doesn't care if the page number is larger than the actual pages,
|
||||
##it will continue to show the last page even if the variable is larger than the actual page
|
||||
author_Url = '{0}&view=story&zone={1}&page=1'.format(self.story.getMetadata('authorUrl'), self.zone)
|
||||
#author_Url = self.story.getMetadata('authorUrl')+'&view=story&zone='+self.zone+'&page=1'
|
||||
|
||||
##I'm resetting the author page to the zone for this story
|
||||
self.story.setMetadata('authorUrl',author_Url)
|
||||
|
||||
logger.debug('Getting the author page: {0}'.format(author_Url))
|
||||
adata = self.get_request(author_Url)
|
||||
|
||||
none_found = "No stories found in this category."
|
||||
if none_found in adata:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: {2}".format(self.zone, self.getBaseDomain(), none_found))
|
||||
if "The member you are looking for does not exist." in adata:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: The member you are looking for does not exist.".format(self.zone, self.getBaseDomain()))
|
||||
#raise exceptions.StoryDoesNotExist(self.zone+'.'+self.getBaseDomain() +" says: The member you are looking for does not exist.")
|
||||
|
||||
asoup = self.make_soup(adata)
|
||||
# logger.debug(asoup)
|
||||
|
||||
story_card = asoup.select_one('div.story-card:has(a[href="{0}"])'.format(url))
|
||||
# logger.debug(story_card)
|
||||
##Getting the number of pages
|
||||
pages=asoup.find('div',{'class' : 'pagination'}).findAll('li')[-1].find('a')
|
||||
if not pages == None:
|
||||
pages = pages['href'].split('=')[-1]
|
||||
else:
|
||||
pages = 0
|
||||
|
||||
## Category
|
||||
## I've only seen one category per story so far, but just in case:
|
||||
for cat in story_card.select('div.story-card-category'):
|
||||
# remove Category:, old code suggests Located: is also
|
||||
# possible, so removing by <strong>
|
||||
cat.find("strong").decompose()
|
||||
self.story.addToList('category',stripHTML(cat))
|
||||
##If there is only 1 page of stories, check it to get the Metadata,
|
||||
if pages == 0:
|
||||
a = asoup.findAll('li')
|
||||
for lc2 in a:
|
||||
if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
|
||||
break
|
||||
## otherwise go through the pages
|
||||
else:
|
||||
page=1
|
||||
i=0
|
||||
while i == 0:
|
||||
##We already have the first page, so if this is the first time through, skip getting the page
|
||||
if page != 1:
|
||||
author_Url = '{0}&view=story&zone={1}&page={2}'.format(self.story.getMetadata('authorUrl'), self.zone, unicode(page))
|
||||
logger.debug('Getting the author page: {0}'.format(author_Url))
|
||||
adata = self.get_request(author_Url)
|
||||
##This will probably never be needed, since AFF doesn't seem to care what number you put as
|
||||
## the page number, it will default to the last page, even if you use 1000, for an author
|
||||
## that only hase 5 pages of stories, but I'm keeping it in to appease Saint Justin Case (just in case).
|
||||
if "The member you are looking for does not exist." in adata:
|
||||
raise exceptions.StoryDoesNotExist("{0}.{1} says: The member you are looking for does not exist.".format(self.zone, self.getBaseDomain()))
|
||||
# we look for the li element that has the story here
|
||||
asoup = self.make_soup(adata)
|
||||
|
||||
self.setDescription(url,story_card.select_one('div.story-card-description'))
|
||||
a = asoup.findAll('li')
|
||||
for lc2 in a:
|
||||
if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$")):
|
||||
i=1
|
||||
break
|
||||
page = page + 1
|
||||
if page > int(pages):
|
||||
break
|
||||
|
||||
for tag in story_card.select('span.story-tag'):
|
||||
self.story.addToList('eroticatags',stripHTML(tag))
|
||||
|
||||
## created/updates share formatting
|
||||
for meta in story_card.select('div.story-card-meta-item span:last-child'):
|
||||
meta = stripHTML(meta)
|
||||
if 'Created: ' in meta:
|
||||
meta = meta.replace('Created: ','')
|
||||
self.story.setMetadata('datePublished', makeDate(meta, self.dateformat))
|
||||
|
||||
if 'Updated: ' in meta:
|
||||
meta = meta.replace('Updated: ','')
|
||||
self.story.setMetadata('dateUpdated', makeDate(meta, self.dateformat))
|
||||
##Split the Metadata up into a list
|
||||
##We have to change the soup type to a string, then remove the newlines, and double spaces,
|
||||
##then changes the <br/> to '-:-', which seperates the different elemeents.
|
||||
##Then we strip the HTML elements from the string.
|
||||
##There is also a double <br/>, so we have to fix that, then remove the leading and trailing '-:-'.
|
||||
##They are always in the same order.
|
||||
## EDIT 09/26/2016: Had some trouble with unicode errors... so I had to put in the decode/encode parts to fix it
|
||||
liMetadata = unicode(lc2).replace('\n','').replace('\r','').replace('\t',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ')
|
||||
liMetadata = stripHTML(liMetadata.replace(r'<br/>','-:-').replace('<!-- <br /-->','-:-'))
|
||||
liMetadata = liMetadata.strip('-:-').strip('-:-').encode('utf-8')
|
||||
for i, value in enumerate(liMetadata.decode('utf-8').split('-:-')):
|
||||
if i == 0:
|
||||
# The value for the title has been manipulated, so may not be the same as gotten at the start.
|
||||
# I'm going to use the href from the lc2 retrieved from the author's page to determine if it is correct.
|
||||
if lc2.find('a', href=re.compile(r'story.php\?no='+self.story.getMetadata('storyId')+"$"))['href'] != url:
|
||||
raise exceptions.StoryDoesNotExist('Did not find story in author story list: {0}'.format(author_Url))
|
||||
elif i == 1:
|
||||
##Get the description
|
||||
self.setDescription(url,stripHTML(value.strip()))
|
||||
else:
|
||||
# the rest of the values can be missing, so instead of hardcoding the numbers, we search for them.
|
||||
if 'Located :' in value:
|
||||
self.story.setMetadata('category',value.replace(r'>',r'>').replace(r'Located :',r'').strip())
|
||||
elif 'Category :' in value:
|
||||
# Get the Category
|
||||
self.story.setMetadata('category',value.replace(r'>',r'>').replace(r'Located :',r'').strip())
|
||||
elif 'Content Tags :' in value:
|
||||
# Get the Erotic Tags
|
||||
value = stripHTML(value.replace(r'Content Tags :',r'')).strip()
|
||||
for code in re.split(r'\s',value):
|
||||
self.story.addToList('eroticatags',code)
|
||||
elif 'Posted :' in value:
|
||||
# Get the Posted Date
|
||||
value = value.replace(r'Posted :',r'').strip()
|
||||
if value.startswith('008'):
|
||||
# It is unknown how the 200 became 008, but I'm going to change it back here
|
||||
value = value.replace('008','200')
|
||||
elif value.startswith('0000'):
|
||||
# Since the date is showing as 0000,
|
||||
# I'm going to put the memberdate here
|
||||
value = asoup.find('div',{'id':'contentdata'}).find('p').get_text(strip=True).replace('Member Since','').strip()
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
elif 'Edited :' in value:
|
||||
# Get the 'Updated' Edited date
|
||||
# AFF has the time for the Updated date, and we only want the date,
|
||||
# so we take the first 10 characters only
|
||||
value = value.replace(r'Edited :',r'').strip()[0:10]
|
||||
if value.startswith('008'):
|
||||
# It is unknown how the 200 became 008, but I'm going to change it back here
|
||||
value = value.replace('008','200')
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
elif value.startswith('0000') or '-00-' in value:
|
||||
# Since the date is showing as 0000,
|
||||
# or there is -00- in the date,
|
||||
# I'm going to put the Published date here
|
||||
self.story.setMetadata('dateUpdated', self.story.getMetadata('datPublished'))
|
||||
else:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
else:
|
||||
# This catches the blank elements, and the Review and Dragon Prints.
|
||||
# I am not interested in these, so do nothing
|
||||
zzzzzzz=0
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
|
@ -241,11 +367,10 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
|
|||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
chaptertag = soup.select_one('div.chapter-body')
|
||||
chaptertag = soup.find('div',{'class' : 'pagination'}).parent.findNext('td')
|
||||
if None == chaptertag:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: {0}! Missing required element!".format(url))
|
||||
## chapter text includes a copy of story title, author,
|
||||
## chapter title, & eroticatags specific to the chapter. Did
|
||||
## before, too.
|
||||
# Change td to a div.
|
||||
chaptertag.name='div'
|
||||
|
||||
return self.utf8FromSoup(url,chaptertag)
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2026 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import re
|
||||
|
||||
from .base_xenforo2forum_adapter import BaseXenForo2ForumAdapter
|
||||
|
||||
def getClass():
|
||||
return AltHistoryComAdapter
|
||||
|
||||
## NOTE: This is a different site than www.alternatehistory.com.
|
||||
|
||||
class AltHistoryComAdapter(BaseXenForo2ForumAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseXenForo2ForumAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ahc')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'althistory.com'
|
||||
|
||||
280
fanficfare/adapters/adapter_andromedawebcom.py
Normal file
280
fanficfare/adapters/adapter_andromedawebcom.py
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# ####### Not all lables are captured. they are not formtted correctly on the
|
||||
# ####### webpage.
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return AndromedaWebComAdapter # XXX
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class AndromedaWebComAdapter(BaseSiteAdapter): # XXX
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
# XXX Most sites don't have the /fiction part. Replace all to remove it usually.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','awc') # XXX
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d %b %Y" # XXX
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.andromeda-web.com' # XXX
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self.post_request(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&warning=2"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self.get_request(url)
|
||||
|
||||
# Since the warning text can change by warning level, let's
|
||||
# look for the warning pass url. ksarchive uses
|
||||
# &warning= -- actually, so do other sites. Must be an
|
||||
# eFiction book.
|
||||
|
||||
# fiction/viewstory.php?sid=1882&warning=4
|
||||
# fiction/viewstory.php?sid=1654&ageconsent=ok&warning=2
|
||||
#print data
|
||||
m = re.search(r"'fiction/viewstory.php\?sid=10(&warning=2)'",data)
|
||||
m = re.search(r"'fiction/viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
pagetitle = soup.find('div',{'id':'content'})
|
||||
|
||||
## Title
|
||||
a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl)
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"fiction/viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^fiction/viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('fiction/viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'class' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -18,20 +18,55 @@
|
|||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import json
|
||||
|
||||
from .base_otw_adapter import BaseOTWAdapter
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return ArchiveOfOurOwnOrgAdapter
|
||||
|
||||
class ArchiveOfOurOwnOrgAdapter(BaseOTWAdapter):
|
||||
class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseOTWAdapter.__init__(self, config, url)
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
self.addurl = ""
|
||||
|
||||
self.full_work_soup = None
|
||||
self.full_work_chapters = None
|
||||
self.use_full_work_soup = True
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
|
||||
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
m = re.match(self.getSiteURLPattern(),url)
|
||||
if m:
|
||||
self.story.setMetadata('storyId',m.group('id'))
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
|
||||
else:
|
||||
raise exceptions.InvalidStoryURL(url,
|
||||
self.getSiteDomain(),
|
||||
self.getSiteExampleURLs())
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ao3')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%Y-%b-%d"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
|
|
@ -49,21 +84,532 @@ class ArchiveOfOurOwnOrgAdapter(BaseOTWAdapter):
|
|||
return ['archiveofourown.org',
|
||||
'archiveofourown.com',
|
||||
'archiveofourown.net',
|
||||
'archiveofourown.gay',
|
||||
'download.archiveofourown.org',
|
||||
'download.archiveofourown.com',
|
||||
'download.archiveofourown.net',
|
||||
'ao3.org',
|
||||
]
|
||||
|
||||
def mod_url_request(self, url):
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/works/123456 https://"+cls.getSiteDomain()+"/collections/Some_Archive/works/123456 https://"+cls.getSiteDomain()+"/works/123456/chapters/78901"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
# https://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
|
||||
# Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
|
||||
# logger.debug(r"https?://" + r"|".join([x.replace('.','\.') for x in self.getAcceptDomains()]) + r"(/collections/[^/]+)?/works/0*(?P<id>\d+)")
|
||||
return r"https?://(" + r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()]) + r")(/collections/[^/]+)?/works/0*(?P<id>\d+)"
|
||||
|
||||
@classmethod
|
||||
def get_section_url(cls,url):
|
||||
## minimal URL used for section names in INI and reject list
|
||||
## for comparison
|
||||
# logger.debug("pre--url:%s"%url)
|
||||
## https://archiveofourown.org/works/19334905/chapters/71697933
|
||||
# http://archiveofourown.org/works/34686793/chapters/89043733
|
||||
url = re.sub(r'^https?://(.*/works/\d+).*$',r'https://\1',url)
|
||||
# logger.debug("post-url:%s"%url)
|
||||
return url
|
||||
|
||||
def mod_url_request(self, url):
|
||||
## add / to *not* replace media.archiveofourown.org
|
||||
if self.getConfig("use_archive_transformativeworks_org",False):
|
||||
return url.replace("/archiveofourown.org","/archive.transformativeworks.org")
|
||||
elif self.getConfig("use_archiveofourown_gay",False):
|
||||
return url.replace("/archiveofourown.org","/archiveofourown.gay")
|
||||
## Login
|
||||
def needToLoginCheck(self, data):
|
||||
if 'This work is only available to registered users of the Archive.' in data \
|
||||
or "The password or user name you entered doesn't match our records" in data:
|
||||
return True
|
||||
else:
|
||||
return url
|
||||
return False
|
||||
|
||||
def performLogin(self, url, data):
|
||||
|
||||
params = {}
|
||||
if self.password:
|
||||
params['user[login]'] = self.username
|
||||
params['user[password]'] = self.password
|
||||
else:
|
||||
params['user[login]'] = self.getConfig("username")
|
||||
params['user[password]'] = self.getConfig("password")
|
||||
params['user[remember_me]'] = '1'
|
||||
params['commit'] = 'Log in'
|
||||
params['utf8'] = u'\x2713' # utf8 *is* required now. hex code works better than actual character for some reason. u'✓'
|
||||
|
||||
# authenticity_token now comes from a completely separate json call.
|
||||
token_json = json.loads(self.get_request('https://' + self.getSiteDomain() + "/token_dispenser.json"))
|
||||
params['authenticity_token'] = token_json['token']
|
||||
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/users/login'
|
||||
logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['user[login]']))
|
||||
|
||||
d = self.post_request(loginUrl, params)
|
||||
|
||||
if 'href="/users/logout"' not in d :
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['user[login]']))
|
||||
raise exceptions.FailedToLogin(url,params['user[login]'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
self.addurl = "?view_adult=true"
|
||||
else:
|
||||
self.addurl=""
|
||||
|
||||
metaurl = self.url+self.addurl
|
||||
url = self.url+'/navigate'+self.addurl
|
||||
logger.info("url: "+url)
|
||||
logger.info("metaurl: "+metaurl)
|
||||
|
||||
data = self.get_request(url)
|
||||
meta = self.get_request(metaurl)
|
||||
|
||||
if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
|
||||
if self.addurl:
|
||||
## "?view_adult=true" doesn't work on base story
|
||||
## URL anymore, which means we have to
|
||||
metasoup = self.make_soup(meta)
|
||||
a = metasoup.find('a',text='Proceed')
|
||||
metaurl = 'https://'+self.host+a['href']
|
||||
meta = self.get_request(metaurl)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Sorry, we couldn't find the work you were looking for." in data:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
|
||||
# need to log in for this one, or always_login.
|
||||
if self.needToLoginCheck(data) or \
|
||||
( self.getConfig("always_login") and 'href="/users/logout"' not in data ):
|
||||
self.performLogin(url,data)
|
||||
data = self.get_request(url,usecache=False)
|
||||
meta = self.get_request(metaurl,usecache=False)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
for tag in soup.findAll('div',id='admin-banner'):
|
||||
tag.extract()
|
||||
metasoup = self.make_soup(meta)
|
||||
for tag in metasoup.findAll('div',id='admin-banner'):
|
||||
tag.extract()
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r"/works/\d+$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
if self.getConfig("always_login"):
|
||||
# deliberately using always_login instead of checking for
|
||||
# actual login so we don't have a case where these show up
|
||||
# for a user only when they get user-restricted stories.
|
||||
try:
|
||||
# is bookmarked if has update /bookmarks/ form --
|
||||
# create bookmark form uses different url
|
||||
self.story.setMetadata('bookmarked',
|
||||
None != metasoup.find('form',action=re.compile(r'^/bookmarks/')))
|
||||
self.story.extendList('bookmarktags',
|
||||
metasoup.find('input',id='bookmark_tag_string')['value'].split(', '))
|
||||
self.story.setMetadata('bookmarkprivate',
|
||||
metasoup.find('input',id='bookmark_private').has_attr('checked'))
|
||||
self.story.setMetadata('bookmarkrec',
|
||||
metasoup.find('input',id='bookmark_rec').has_attr('checked'))
|
||||
except KeyError:
|
||||
pass
|
||||
self.story.setMetadata('bookmarksummary',
|
||||
stripHTML(metasoup.find('textarea',id='bookmark_notes')))
|
||||
|
||||
if metasoup.find('img',alt='(Restricted)'):
|
||||
self.story.setMetadata('restricted','Restricted')
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/.+"))
|
||||
if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
|
||||
self.story.setMetadata('author','Anonymous')
|
||||
self.story.setMetadata('authorUrl','https://' + self.getSiteDomain() + '/')
|
||||
self.story.setMetadata('authorId','0')
|
||||
else:
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('/')[-1])
|
||||
self.story.addToList('authorUrl','https://'+self.host+a['href'])
|
||||
self.story.addToList('author',a.text)
|
||||
|
||||
byline = metasoup.find('h3',{'class':'byline'})
|
||||
if byline:
|
||||
self.story.setMetadata('byline',stripHTML(byline))
|
||||
|
||||
# byline:
|
||||
# <h3 class="byline heading">
|
||||
# Hope Roy [archived by <a href="/users/ssa_archivist/pseuds/ssa_archivist" rel="author">ssa_archivist</a>]
|
||||
# </h3>
|
||||
# stripped:"Hope Roy [archived by ssa_archivist]"
|
||||
m = re.match(r'(?P<author>.*) \[archived by ?(?P<archivist>.*)\]',stripHTML(byline))
|
||||
if( m and
|
||||
len(alist) == 1 and
|
||||
self.getConfig('use_archived_author') ):
|
||||
self.story.setMetadata('author',m.group('author'))
|
||||
|
||||
newestChapter = None
|
||||
self.newestChapterNum = None # save for comparing during update.
|
||||
# Scan all chapters to find the oldest and newest, on AO3 it's
|
||||
# possible for authors to insert new chapters out-of-order or
|
||||
# change the dates of earlier ones by editing them--That WILL
|
||||
# break epub update.
|
||||
# Find the chapters:
|
||||
chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+r"/chapters/\d+$"))
|
||||
self.story.setMetadata('numChapters',len(chapters))
|
||||
logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
|
||||
if len(chapters)==1:
|
||||
self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+chapters[0]['href'])
|
||||
else:
|
||||
for index, chapter in enumerate(chapters):
|
||||
# strip just in case there's tags, like <i> in chapter titles.
|
||||
# (2013-09-21)
|
||||
date = stripHTML(chapter.findNext('span'))[1:-1]
|
||||
chapterDate = makeDate(date,self.dateformat)
|
||||
self.add_chapter(chapter,'https://'+self.host+chapter['href'],
|
||||
{'date':chapterDate.strftime(self.getConfig("datechapter_format",self.getConfig("datePublished_format","%Y-%m-%d")))})
|
||||
if newestChapter == None or chapterDate > newestChapter:
|
||||
newestChapter = chapterDate
|
||||
self.newestChapterNum = index
|
||||
|
||||
a = metasoup.find('blockquote',{'class':'userstuff'})
|
||||
if a != None:
|
||||
a.name='div' # Change blockquote to div.
|
||||
self.setDescription(url,a)
|
||||
#self.story.setMetadata('description',a.text)
|
||||
|
||||
a = metasoup.find('dd',{'class':"rating tags"})
|
||||
if a != None:
|
||||
self.story.setMetadata('rating',stripHTML(a.text))
|
||||
|
||||
d = metasoup.find('dd',{'class':"language"})
|
||||
if d != None:
|
||||
self.story.setMetadata('language',stripHTML(d.text))
|
||||
|
||||
a = metasoup.find('dd',{'class':"fandom tags"})
|
||||
if a != None:
|
||||
fandoms = a.findAll('a',{'class':"tag"})
|
||||
for fandom in fandoms:
|
||||
self.story.addToList('fandoms',fandom.string)
|
||||
|
||||
a = metasoup.find('dd',{'class':"warning tags"})
|
||||
if a != None:
|
||||
warnings = a.findAll('a',{'class':"tag"})
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
a = metasoup.find('dd',{'class':"freeform tags"})
|
||||
if a != None:
|
||||
genres = a.findAll('a',{'class':"tag"})
|
||||
for genre in genres:
|
||||
self.story.addToList('freeformtags',genre.string)
|
||||
|
||||
a = metasoup.find('dd',{'class':"category tags"})
|
||||
if a != None:
|
||||
genres = a.findAll('a',{'class':"tag"})
|
||||
for genre in genres:
|
||||
if genre != "Gen":
|
||||
self.story.addToList('ao3categories',genre.string)
|
||||
|
||||
a = metasoup.find('dd',{'class':"character tags"})
|
||||
if a != None:
|
||||
chars = a.findAll('a',{'class':"tag"})
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
a = metasoup.find('dd',{'class':"relationship tags"})
|
||||
if a != None:
|
||||
ships = a.findAll('a',{'class':"tag"})
|
||||
for ship in ships:
|
||||
self.story.addToList('ships',ship.string)
|
||||
|
||||
a = metasoup.find('dd',{'class':"collections"})
|
||||
if a != None:
|
||||
collections = a.findAll('a')
|
||||
for collection in collections:
|
||||
self.story.addToList('collections',collection.string)
|
||||
|
||||
stats = metasoup.find('dl',{'class':'stats'})
|
||||
dt = stats.findAll('dt')
|
||||
dd = stats.findAll('dd')
|
||||
for x in range(0,len(dt)):
|
||||
label = dt[x].text
|
||||
value = dd[x].text
|
||||
|
||||
if 'Words:' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Comments:' in label:
|
||||
self.story.setMetadata('comments', value)
|
||||
|
||||
if 'Kudos:' in label:
|
||||
self.story.setMetadata('kudos', value)
|
||||
|
||||
if 'Hits:' in label:
|
||||
self.story.setMetadata('hits', value)
|
||||
|
||||
if 'Bookmarks:' in label:
|
||||
self.story.setMetadata('bookmarks', value)
|
||||
|
||||
if 'Chapters:' in label:
|
||||
self.story.setMetadata('chapterslashtotal', value)
|
||||
if value.split('/')[0] == value.split('/')[1]:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Completed' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
|
||||
# Find Series name from series URL.
|
||||
ddseries = metasoup.find('dd',{'class':"series"})
|
||||
|
||||
if ddseries:
|
||||
for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
|
||||
series_name = stripHTML(a)
|
||||
series_url = 'https://'+self.host+a['href']
|
||||
series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
|
||||
self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
|
||||
self.story.setMetadata('series%02dUrl'%i,series_url)
|
||||
if i == 0:
|
||||
self.setSeries(series_name, series_index)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
|
||||
def hookForUpdates(self,chaptercount):
|
||||
if self.newestChapterNum and self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
|
||||
logger.info("Existing epub has %s chapters\nNewest chapter is %s. Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
|
||||
self.oldchapters = self.oldchapters[:self.newestChapterNum]
|
||||
return len(self.oldchapters)
|
||||
|
||||
## Normalize chapter URLs because a) site has changed from http to
|
||||
## https and b) in case of title change. That way updates to
|
||||
## existing stories don't re-download all chapters.
|
||||
def normalize_chapterurl(self,url):
|
||||
url = re.sub(r"https?://("+self.getSiteDomain()+r"/works/\d+/chapters/\d+)(\?view_adult=true)?$",
|
||||
r"https://\1",url)
|
||||
return url
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterTextNum(self, url, index):
|
||||
## FYI: Chapter urls used to include ?view_adult=true in each
|
||||
## one. With cookiejar being passed now, that's not
|
||||
## necessary. However, there is a corner case with plugin--If
|
||||
## a user-required story is attempted after gathering metadata
|
||||
## for one that needs adult, but not user AND the user doesn't
|
||||
## enter a valid user, the is_adult cookie from before can be
|
||||
## lost.
|
||||
logger.debug('Getting chapter text for: %s index: %s' % (url,index))
|
||||
|
||||
save_chapter_soup = self.make_soup('<div class="story"></div>')
|
||||
## use the div because the full soup will also have <html><body>.
|
||||
## need save_chapter_soup for .new_tag()
|
||||
save_chapter=save_chapter_soup.find('div')
|
||||
|
||||
whole_dl_soup = chapter_dl_soup = None
|
||||
|
||||
if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and self.num_chapters() > 1:
|
||||
logger.debug("USE view_full_work")
|
||||
## Assumed view_adult=true was cookied during metadata
|
||||
if not self.full_work_soup:
|
||||
self.full_work_soup = self.make_soup(self.get_request(self.url+"?view_full_work=true"+self.addurl.replace('?','&')))
|
||||
## AO3 has had several cases now where chapter numbers
|
||||
## are missing, breaking the link between
|
||||
## <div id=chapter-##> and Chapter ##.
|
||||
## But they should all still be there and in the right
|
||||
## order, so array[index]
|
||||
self.full_work_chapters = self.full_work_soup.find_all('div',{'id':re.compile(r'chapter-\d+')})
|
||||
if len(self.full_work_chapters) != self.num_chapters():
|
||||
## sanity check just in case.
|
||||
self.use_full_work_soup = False
|
||||
self.full_work_soup = None
|
||||
logger.warning("chapter count in view_full_work(%s) disagrees with num of chapters(%s)--ending use_view_full_work"%(len(self.full_work_chapters),self.num_chapters()))
|
||||
whole_dl_soup = self.full_work_soup
|
||||
|
||||
if whole_dl_soup:
|
||||
chapter_dl_soup = self.full_work_chapters[index]
|
||||
else:
|
||||
whole_dl_soup = chapter_dl_soup = self.make_soup(self.get_request(url+self.addurl))
|
||||
if None == chapter_dl_soup:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
exclude_notes=self.getConfigList('exclude_notes')
|
||||
|
||||
def append_tag(elem,tag,string=None,classes=None):
|
||||
'''bs4 requires tags be added separately.'''
|
||||
new_tag = save_chapter_soup.new_tag(tag)
|
||||
if string:
|
||||
new_tag.string=string
|
||||
if classes:
|
||||
new_tag['class']=[classes]
|
||||
elem.append(new_tag)
|
||||
return new_tag
|
||||
|
||||
## These are the over-all work's 'Notes at the beginning'.
|
||||
## They only appear on the first chapter in individual chapter
|
||||
## pages and before chapter-1 div. Appending removes
|
||||
## headnotes from whole_dl_soup, so be sure to only do it on
|
||||
## the first chapter.
|
||||
head_notes_div = append_tag(save_chapter,'div',classes="fff_chapter_notes fff_head_notes")
|
||||
if 'authorheadnotes' not in exclude_notes and index == 0:
|
||||
headnotes = whole_dl_soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
|
||||
if headnotes != None:
|
||||
## Also include ul class='associations'.
|
||||
ulassoc = headnotes.find('ul', {'class' : "associations"})
|
||||
headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
|
||||
if headnotes != None or ulassoc != None:
|
||||
append_tag(head_notes_div,'b',"Author's Note:")
|
||||
if ulassoc != None:
|
||||
# fix relative links--all examples so far have been.
|
||||
for alink in ulassoc.find_all('a'):
|
||||
if 'http' not in alink['href']:
|
||||
alink['href']='https://' + self.getSiteDomain() + alink['href']
|
||||
head_notes_div.append(ulassoc)
|
||||
if headnotes != None:
|
||||
head_notes_div.append(headnotes)
|
||||
|
||||
## Can appear on every chapter
|
||||
if 'chaptersummary' not in exclude_notes:
|
||||
chapsumm = chapter_dl_soup.find('div', {'id' : "summary"})
|
||||
if chapsumm != None:
|
||||
chapsumm = chapsumm.find('blockquote')
|
||||
append_tag(head_notes_div,'b',"Summary for the Chapter:")
|
||||
head_notes_div.append(chapsumm)
|
||||
|
||||
## Can appear on every chapter
|
||||
if 'chapterheadnotes' not in exclude_notes:
|
||||
chapnotes = chapter_dl_soup.find('div', {'id' : "notes"})
|
||||
if chapnotes != None:
|
||||
chapnotes = chapnotes.find('blockquote')
|
||||
if chapnotes != None:
|
||||
append_tag(head_notes_div,'b',"Notes for the Chapter:")
|
||||
head_notes_div.append(chapnotes)
|
||||
|
||||
text = chapter_dl_soup.find('div', {'class' : "userstuff module"})
|
||||
chtext = text.find('h3', {'class' : "landmark heading"})
|
||||
if chtext:
|
||||
chtext.extract()
|
||||
save_chapter.append(text)
|
||||
|
||||
foot_notes_div = append_tag(save_chapter,'div',classes="fff_chapter_notes fff_foot_notes")
|
||||
## Can appear on every chapter
|
||||
if 'chapterfootnotes' not in exclude_notes:
|
||||
chapfoot = chapter_dl_soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
|
||||
if chapfoot != None:
|
||||
chapfoot = chapfoot.find('blockquote')
|
||||
append_tag(foot_notes_div,'b',"Notes for the Chapter:")
|
||||
foot_notes_div.append(chapfoot)
|
||||
|
||||
skip_on_update_tags = []
|
||||
## These are the over-all work's 'Notes at the end'.
|
||||
## They only appear on the last chapter in individual chapter
|
||||
## pages and after chapter-# div. Appending removes
|
||||
## headnotes from whole_dl_soup, so be sure to only do it on
|
||||
## the last chapter.
|
||||
if 'authorfootnotes' not in exclude_notes and index+1 == self.num_chapters():
|
||||
footnotes = whole_dl_soup.find('div', {'id' : "work_endnotes"})
|
||||
if footnotes != None:
|
||||
footnotes = footnotes.find('blockquote')
|
||||
if footnotes:
|
||||
b = append_tag(foot_notes_div,'b',"Author's Note:")
|
||||
skip_on_update_tags.append(b)
|
||||
skip_on_update_tags.append(footnotes)
|
||||
foot_notes_div.append(footnotes)
|
||||
|
||||
## It looks like 'Inspired by' links now all appear in the ul
|
||||
## class=associations tag in authorheadnotes. This code is
|
||||
## left in case I'm wrong and there are still stories with div
|
||||
## id=children inspired links at the end.
|
||||
if 'inspiredlinks' not in exclude_notes and index+1 == self.num_chapters():
|
||||
inspiredlinks = whole_dl_soup.find('div', {'id' : "children"})
|
||||
if inspiredlinks != None:
|
||||
if inspiredlinks:
|
||||
inspiredlinks.find('h3').name='b' # don't want a big h3 at the end.
|
||||
# fix relative links--all examples so far have been.
|
||||
for alink in inspiredlinks.find_all('a'):
|
||||
if 'http' not in alink['href']:
|
||||
alink['href']='https://' + self.getSiteDomain() + alink['href']
|
||||
skip_on_update_tags.append(inspiredlinks)
|
||||
foot_notes_div.append(inspiredlinks)
|
||||
|
||||
## remove empty head/food notes div(s)
|
||||
if not head_notes_div.find(True):
|
||||
head_notes_div.extract()
|
||||
if not foot_notes_div.find(True):
|
||||
foot_notes_div.extract()
|
||||
## AO3 story end notes end up in the 'last' chapter, but if
|
||||
## updated, then there's a new 'last' chapter. This option
|
||||
## applies the 'skip_on_ffdl_update' class to those tags which
|
||||
## means they will be removed during epub reading for update.
|
||||
## Results: only the last chapter will have end notes.
|
||||
## Side-effect: An 'Update Always' that doesn't add a new
|
||||
## lasts chapter will remove the end notes.
|
||||
if self.getConfig("remove_authorfootnotes_on_update"):
|
||||
for skip_tag in skip_on_update_tags:
|
||||
if skip_tag.has_attr('class'):
|
||||
skip_tag['class'].append('skip_on_ffdl_update')
|
||||
else:
|
||||
skip_tag['class']=['skip_on_ffdl_update']
|
||||
# logger.debug(skip_tag)
|
||||
|
||||
return self.utf8FromSoup(url,save_chapter)
|
||||
|
||||
def before_get_urls_from_page(self,url,normalize):
|
||||
# special stuff to log into archiveofourown.org, if possible.
|
||||
# Unlike most that show the links to 'adult' stories, but protect
|
||||
# them, AO3 doesn't even show them if not logged in. Only works
|
||||
# with saved user/pass--not going to prompt for list.
|
||||
if self.getConfig("username"):
|
||||
if self.getConfig("is_adult"):
|
||||
if '?' in url:
|
||||
addurl = "&view_adult=true"
|
||||
else:
|
||||
addurl = "?view_adult=true"
|
||||
else:
|
||||
addurl=""
|
||||
# just to get an authenticity_token.
|
||||
data = self.get_request(url+addurl)
|
||||
# login the session.
|
||||
self.performLogin(url,data)
|
||||
# get the list page with logged in session.
|
||||
|
||||
def get_series_from_page(self,url,data,normalize=False):
|
||||
'''
|
||||
This method is to make it easier for adapters to detect a
|
||||
series URL, pick out the series metadata and list of storyUrls
|
||||
to return without needing to override get_urls_from_page
|
||||
entirely.
|
||||
'''
|
||||
|
||||
if 'This work is only available to registered users of the Archive' in data:
|
||||
raise exceptions.FailedToDownload("This work is only available to registered users of the Archive -- set username/password in personal.ini under [archiveofourown.org]")
|
||||
## easiest way to get all the weird URL possibilities and stay
|
||||
## up to date with future changes.
|
||||
m = re.match(self.getSiteURLPattern().replace('/works/','/series/'),url)
|
||||
if m:
|
||||
soup = self.make_soup(data)
|
||||
retval = {}
|
||||
retval['urllist']=[ 'https://'+self.host+a['href'] for a in soup.select('h4.heading a:first-child') ]
|
||||
retval['name']=stripHTML(soup.select_one("h2.heading"))
|
||||
desc=soup.select_one("div.wrapper dd blockquote.userstuff")
|
||||
if desc:
|
||||
desc.name='div' # change blockquote to div to match stories.
|
||||
retval['desc']=desc
|
||||
stats=stripHTML(soup.select_one("dl.series dl.stats"))
|
||||
if 'Complete:Yes' in stats:
|
||||
retval['status'] = "Completed"
|
||||
elif 'Complete:No' in stats:
|
||||
retval['status'] = "In-Progress"
|
||||
return retval
|
||||
## return dict with at least {'urllist':['storyUrl','storyUrl',...]}
|
||||
## optionally 'name' and 'desc'?
|
||||
return {}
|
||||
|
|
|
|||
174
fanficfare/adapters/adapter_archiveskyehawkecom.py
Normal file
174
fanficfare/adapters/adapter_archiveskyehawkecom.py
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
|
||||
def getClass():
|
||||
return ArchiveSkyeHawkeComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ash')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%Y-%m-%d"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'archive.skyehawke.com'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['archive.skyehawke.com','www.skyehawke.com']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234 http://skyehawke.com/archive/story.php?no=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
|
||||
title=stripHTML(a).split('"')[1]
|
||||
self.story.setMetadata('title',title)
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
author = a.find('a')
|
||||
self.story.setMetadata('authorId',author['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+author['href'])
|
||||
self.story.setMetadata('author',author.string)
|
||||
|
||||
authorSoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
|
||||
|
||||
chapter=soup.find('select',{'name':'chapter'}).findAll('option')
|
||||
|
||||
for i in range(1,len(chapter)):
|
||||
ch=chapter[i]
|
||||
self.add_chapter(ch,ch['value'])
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
box=soup.find('div', {'class': "container borderridge"})
|
||||
sum=box.find('span').text
|
||||
self.setDescription(url,sum)
|
||||
|
||||
boxes=soup.findAll('div', {'class': "container bordersolid"})
|
||||
for box in boxes:
|
||||
if box.find('b') != None and box.find('b').text == "History and Story Information":
|
||||
|
||||
for b in box.findAll('b'):
|
||||
if "words" in b.nextSibling:
|
||||
self.story.setMetadata('numWords', b.text)
|
||||
if "archived" in b.previousSibling:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(b.text), self.dateformat))
|
||||
if "updated" in b.previousSibling:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(b.text), self.dateformat))
|
||||
if "fandom" in b.nextSibling:
|
||||
self.story.addToList('category', b.text)
|
||||
|
||||
for br in box.findAll('br'):
|
||||
br.replaceWith('split')
|
||||
genre=box.text.split("Genre:")[1].split("split")[0]
|
||||
if not "Unspecified" in genre:
|
||||
self.story.addToList('genre',genre)
|
||||
|
||||
|
||||
if box.find('span') != None and box.find('span').text == "WARNING":
|
||||
|
||||
rating=box.findAll('span')[1]
|
||||
rating.find('br').replaceWith('split')
|
||||
rating=rating.text.replace("This story is rated",'').split('split')[0]
|
||||
self.story.setMetadata('rating',rating)
|
||||
logger.debug(self.story.getMetadata('rating'))
|
||||
|
||||
warnings=box.find('ol')
|
||||
if warnings != None:
|
||||
warnings=warnings.text.replace(']', '').replace('[', '').split(' ')
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning)
|
||||
|
||||
|
||||
for asoup in authorSoup.findAll('div', {'class':"story bordersolid"}):
|
||||
if asoup.find('a')['href'] == 'story.php?no='+self.story.getMetadata('storyId'):
|
||||
if '[ Completed ]' in asoup.text:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
chars=asoup.findNext('div').text.split('Characters')[1].split(']')[0]
|
||||
for char in chars.split(','):
|
||||
if not "None" in char:
|
||||
self.story.addToList('characters',char)
|
||||
break
|
||||
|
||||
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div',{'class':"chapter bordersolid"}).findNext('div').findNext('div')
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -79,7 +79,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
data1 = self.get_request(self.url)
|
||||
soup1 = self.make_soup(data1)
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soup1.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
if 'Page Not Found.' in data1:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
|
|
@ -92,7 +92,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title', title.string)
|
||||
|
||||
# Author
|
||||
author = soup1.find('div',{'class':'story-info'}).find_all('div',{'class':'story-info-bl'})[1].find('a')
|
||||
author = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl'})[1].find('a')
|
||||
authorurl = author['href']
|
||||
self.story.setMetadata('author', author.string)
|
||||
self.story.setMetadata('authorUrl', authorurl)
|
||||
|
|
@ -112,7 +112,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
### add it before the rest of the pages, if any
|
||||
self.add_chapter('1', self.url)
|
||||
|
||||
chapterTable = soup1.find('div',{'class':'pages'}).find_all('a')
|
||||
chapterTable = soup1.find('div',{'class':'pages'}).findAll('a')
|
||||
|
||||
if chapterTable is not None:
|
||||
# Multi-chapter story
|
||||
|
|
@ -124,7 +124,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
|
|||
self.add_chapter(chapterTitle, chapterUrl)
|
||||
|
||||
|
||||
rated = soup1.find('div',{'class':'story-info'}).find_all('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
|
||||
rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
|
||||
self.story.setMetadata('rating',rated)
|
||||
|
||||
self.story.setMetadata('dateUpdated', makeDate('01/01/2001', '%m/%d/%Y'))
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','asph')
|
||||
|
|
@ -64,10 +64,10 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
|
|
@ -92,7 +92,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
params['intent'] = ''
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/user.php'
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
|
|
@ -130,20 +130,20 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
asoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
|
||||
|
||||
try:
|
||||
# in case link points somewhere other than the first chapter
|
||||
a = soup.find_all('option')[1]['value']
|
||||
a = soup.findAll('option')[1]['value']
|
||||
self.story.setMetadata('storyId',a.split('=',)[1])
|
||||
url = 'https://'+self.host+'/'+a
|
||||
url = 'http://'+self.host+'/'+a
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
except:
|
||||
pass
|
||||
|
||||
for info in asoup.find_all('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
|
||||
for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
|
||||
a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
if a != None:
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
|
@ -151,13 +151,13 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
if len(chapters) == 0:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
for chapter in chapters:
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href'])
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
|
|
@ -170,7 +170,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
except:
|
||||
return ""
|
||||
|
||||
cats = info.find_all('a',href=re.compile('categories.php'))
|
||||
cats = info.findAll('a',href=re.compile('categories.php'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
|
|
@ -188,7 +188,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
|
|||
## <td><span class="sb"><b>Published:</b> 04/08/2007</td>
|
||||
|
||||
## one story had <b>Updated...</b> in the description. Restrict to sub-table
|
||||
labels = info.find('table').find_all('b')
|
||||
labels = info.find('table').findAll('b')
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
|
|
|||
|
|
@ -111,17 +111,11 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
def doExtractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
url = self.url
|
||||
logger.info("url: "+url)
|
||||
soup = None
|
||||
try:
|
||||
data = self.get_request(url)
|
||||
soup = self.make_soup(data)
|
||||
except exceptions.HTTPErrorFFF as e:
|
||||
if e.status_code != 404:
|
||||
raise
|
||||
data = self.decode_data(e.data)
|
||||
data = self.get_request(url)
|
||||
|
||||
# logger.debug(data)
|
||||
if not soup or self.loginNeededCheck(data):
|
||||
soup = self.make_soup(data)
|
||||
|
||||
if self.loginNeededCheck(data):
|
||||
# always login if not already to avoid lots of headaches
|
||||
self.performLogin(url,data)
|
||||
# refresh website after logging in
|
||||
|
|
@ -146,8 +140,8 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
|
||||
# Find authorid and URL from... author url.
|
||||
mainmeta = soup.find('footer', {'class': 'main-meta'})
|
||||
alist = mainmeta.find('span', string='Author(s)')
|
||||
alist = alist.parent.find_all('a', href=re.compile(r"/profile/u/[^/]+"))
|
||||
alist = mainmeta.find('span', text='Author(s)')
|
||||
alist = alist.parent.findAll('a', href=re.compile(r"/profile/u/[^/]+"))
|
||||
for a in alist:
|
||||
self.story.addToList('authorId',a['href'].split('/')[-1])
|
||||
self.story.addToList('authorUrl','https://'+self.host+a['href'])
|
||||
|
|
@ -159,10 +153,10 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
chapters=soup.find('select',{'name':'chapter-nav'})
|
||||
hrefattr=None
|
||||
if chapters:
|
||||
chapters=chapters.find_all('option')
|
||||
chapters=chapters.findAll('option')
|
||||
hrefattr='value'
|
||||
else: # didn't find <select name='chapter-nav', look for alternative
|
||||
chapters=soup.find('div',{'class':'widget--chapters'}).find_all('a')
|
||||
chapters=soup.find('div',{'class':'widget--chapters'}).findAll('a')
|
||||
hrefattr='href'
|
||||
for index, chapter in enumerate(chapters):
|
||||
if chapter.text != 'Foreword' and 'Collapse chapters' not in chapter.text:
|
||||
|
|
@ -171,9 +165,9 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# find timestamp
|
||||
a = soup.find('span', string='Updated')
|
||||
a = soup.find('span', text='Updated')
|
||||
if a == None:
|
||||
a = soup.find('span', string='Published') # use published date if work was never updated
|
||||
a = soup.find('span', text='Published') # use published date if work was never updated
|
||||
a = a.parent.find('time')
|
||||
chapterDate = makeDate(a['datetime'],self.dateformat)
|
||||
if newestChapter == None or chapterDate > newestChapter:
|
||||
|
|
@ -181,7 +175,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
self.newestChapterNum = index
|
||||
|
||||
# story status
|
||||
a = mainmeta.find('span', string='Completed')
|
||||
a = mainmeta.find('span', text='Completed')
|
||||
if a:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
|
|
@ -200,37 +194,37 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
self.setDescription(url,a)
|
||||
|
||||
# story tags
|
||||
a = mainmeta.find('span',string='Tags')
|
||||
a = mainmeta.find('span',text='Tags')
|
||||
if a:
|
||||
tags = a.parent.find_all('a')
|
||||
tags = a.parent.findAll('a')
|
||||
for tag in tags:
|
||||
self.story.addToList('tags', tag.text)
|
||||
|
||||
# story tags
|
||||
a = mainmeta.find('span',string='Characters')
|
||||
a = mainmeta.find('span',text='Characters')
|
||||
if a:
|
||||
self.story.addToList('characters', a.nextSibling)
|
||||
|
||||
# published on
|
||||
a = soup.find('span', string='Published')
|
||||
a = soup.find('span', text='Published')
|
||||
a = a.parent.find('time')
|
||||
self.story.setMetadata('datePublished', makeDate(a['datetime'], self.dateformat))
|
||||
|
||||
# updated on
|
||||
a = soup.find('span', string='Updated')
|
||||
a = soup.find('span', text='Updated')
|
||||
if a:
|
||||
a = a.parent.find('time')
|
||||
self.story.setMetadata('dateUpdated', makeDate(a['datetime'], self.dateformat))
|
||||
|
||||
# word count
|
||||
a = soup.find('span', string='Total Word Count')
|
||||
a = soup.find('span', text='Total Word Count')
|
||||
if a:
|
||||
a = a.find_next('span')
|
||||
self.story.setMetadata('numWords', int(a.text.split()[0]))
|
||||
|
||||
# upvote, subs, and views
|
||||
a = soup.find('div',{'class':'title-meta'})
|
||||
spans = a.find_all('span', recursive=False)
|
||||
spans = a.findAll('span', recursive=False)
|
||||
self.story.setMetadata('upvotes', re.search(r'\(([^)]+)', spans[0].find('span').text).group(1))
|
||||
self.story.setMetadata('subscribers', re.search(r'\(([^)]+)', spans[1].find('span').text).group(1))
|
||||
if len(spans) > 2: # views can be private
|
||||
|
|
@ -252,39 +246,13 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
|
|||
|
||||
data = self.get_request(url)
|
||||
soup = self.make_soup(data)
|
||||
# logger.debug(data)
|
||||
|
||||
ageform = soup.select_one('form[action="/account/toggle_age"]')
|
||||
# logger.debug(ageform)
|
||||
if ageform and (self.is_adult or self.getConfig("is_adult")):
|
||||
params = {}
|
||||
params['is_of_age']=ageform.select_one('input#is_of_age')['value']
|
||||
params['current_url']=ageform.select_one('input#current_url')['value']
|
||||
params['csrf_aff_token']=ageform.select_one('input[name="csrf_aff_token"]')['value']
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/account/mark_over_18'
|
||||
logger.info("Will now toggle age to URL (%s)" % (loginUrl))
|
||||
# logger.debug(params)
|
||||
data = self.post_request(loginUrl, params)
|
||||
soup = self.make_soup(data)
|
||||
# logger.debug(data)
|
||||
|
||||
content = soup.find('div', {'id': 'user-submitted-body'})
|
||||
|
||||
if self.getConfig('inject_chapter_image'):
|
||||
logger.debug("Injecting chapter image")
|
||||
imgdiv = soup.select_one('div#bodyText div.bot-spacer')
|
||||
if imgdiv:
|
||||
content.insert(0, "\n")
|
||||
content.insert(0, imgdiv)
|
||||
content.insert(0, "\n")
|
||||
|
||||
if self.getConfig('inject_chapter_title'):
|
||||
logger.debug("Injecting full-length chapter title")
|
||||
title = soup.find('h1', {'id' : 'chapter-title'}).text
|
||||
newTitle = soup.new_tag('h3')
|
||||
newTitle.string = title
|
||||
content.insert(0, "\n")
|
||||
content.insert(0, newTitle)
|
||||
content.insert(0, "\n")
|
||||
|
||||
return self.utf8FromSoup(url,content)
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Find the chapters:
|
||||
# The update date is with the chapter links... so we will update it here as well
|
||||
for chapter in soup.find_all('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+r"&chapterid=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+r"&chapterid=\d+$")):
|
||||
value = chapter.findNext('td').findNext('td').string.replace('(added on','').replace(')','').strip()
|
||||
self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
|
||||
self.add_chapter(chapter,'https://'+self.getSiteDomain()+chapter['href'])
|
||||
|
|
@ -134,11 +134,11 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Get the MetaData
|
||||
# Erotia Tags
|
||||
tags = soup.find_all('a',href=re.compile(r'/stories/search.php\?selectedcode'))
|
||||
tags = soup.findAll('a',href=re.compile(r'/stories/search.php\?selectedcode'))
|
||||
for tag in tags:
|
||||
self.story.addToList('eroticatags',tag.text)
|
||||
|
||||
for td in soup.find_all('td'):
|
||||
for td in soup.findAll('td'):
|
||||
if len(td.text)>0:
|
||||
if 'Added on:' in td.text and '<table' not in unicode(td):
|
||||
value = td.text.replace('Added on:','').strip()
|
||||
|
|
@ -169,20 +169,20 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload("Error downloading Chapter: {0}! Missing required element!".format(url))
|
||||
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in chaptertag.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in chaptertag.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
# BDSM Library basically wraps it's own html around the document,
|
||||
# so we will be removing the script, title and meta content from the
|
||||
# storyblock
|
||||
for tag in chaptertag.find_all('head') + chaptertag.find_all('style') + chaptertag.find_all('title') + chaptertag.find_all('meta') + chaptertag.find_all('o:p') + chaptertag.find_all('link'):
|
||||
for tag in chaptertag.findAll('head') + chaptertag.findAll('style') + chaptertag.findAll('title') + chaptertag.findAll('meta') + chaptertag.findAll('o:p') + chaptertag.findAll('link'):
|
||||
tag.extract()
|
||||
|
||||
for tag in chaptertag.find_all('o:smarttagtype'):
|
||||
for tag in chaptertag.findAll('o:smarttagtype'):
|
||||
tag.name = 'span'
|
||||
|
||||
## I'm going to take the attributes off all of the tags
|
||||
## because they usually refer to the style that we removed above.
|
||||
for tag in chaptertag.find_all(True):
|
||||
for tag in chaptertag.findAll(True):
|
||||
tag.attrs = None
|
||||
|
||||
return self.utf8FromSoup(url,chaptertag)
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
|
|||
|
||||
summary_div = list_box.find('div', {'class': 'list_summary'})
|
||||
if not self.getConfig('keep_summary_html'):
|
||||
summary = ''.join(summary_div(string=True))
|
||||
summary = ''.join(summary_div(text=True))
|
||||
else:
|
||||
summary = self.utf8FromSoup(author_url, summary_div)
|
||||
|
||||
|
|
@ -157,6 +157,9 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
|
|||
|
||||
self.story.addToList('warnings', warning)
|
||||
|
||||
elif key == 'Chapters':
|
||||
self.story.setMetadata('numChapters', int(value))
|
||||
|
||||
elif key == 'Words':
|
||||
# Apparently only numChapters need to be an integer for
|
||||
# some strange reason. Remove possible ',' characters as to
|
||||
|
|
@ -171,7 +174,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
|
|||
# ugly %p(am/pm) hack moved into makeDate so other sites can use it.
|
||||
self.story.setMetadata('dateUpdated', date)
|
||||
|
||||
if self.story.getMetadataRaw('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
|
||||
if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
def getChapterText(self, url):
|
||||
|
|
|
|||
279
fanficfare/adapters/adapter_buffygilescom.py
Normal file
279
fanficfare/adapters/adapter_buffygilescom.py
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return BuffyGilesComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class BuffyGilesComAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
# XXX Most sites don't have the /efiction part. Replace all to remove it usually.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/efiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','bufg')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d/%m/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'buffygiles.velocitygrass.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self.post_request(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&warning=5"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self.get_request(url)
|
||||
|
||||
# Since the warning text can change by warning level, let's
|
||||
# look for the warning pass url. ksarchive uses
|
||||
# &warning= -- actually, so do other sites. Must be an
|
||||
# eFiction book.
|
||||
|
||||
# efiction/viewstory.php?sid=1882&warning=4
|
||||
# efiction/viewstory.php?sid=1654&ageconsent=ok&warning=5
|
||||
#print data
|
||||
m = re.search(r"'efiction/viewstory.php\?sid=542(&warning=5)'",data)
|
||||
m = re.search(r"'efiction/viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
pagetitle = soup.find('div',{'id':'pagetitle'})
|
||||
|
||||
## Title
|
||||
a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/efiction/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"efiction/viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^efiction/viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('efiction/viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
|
||||
value = labels[0].previousSibling
|
||||
svalue = ""
|
||||
|
|
@ -154,22 +154,22 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value.split(' -')[0])
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -194,7 +194,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ class ChireadsComSiteAdapter(BaseSiteAdapter):
|
|||
intro = stripHTML(info.select_one('.inform-inform-txt').span)
|
||||
self.setDescription(self.url, intro)
|
||||
|
||||
for content in soup.find_all('div', {'id': 'content'}):
|
||||
for a in content.find_all('a'):
|
||||
for content in soup.findAll('div', {'id': 'content'}):
|
||||
for a in content.findAll('a'):
|
||||
self.add_chapter(a.get_text(), a['href'])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
## Title
|
||||
## Some stories have a banner that has it's own a tag before the actual text title...
|
||||
## so I'm checking the pagetitle div for all a tags that match the criteria, then taking the last.
|
||||
a = soup.find('div',{'id':'pagetitle'}).find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))[-1]
|
||||
a = soup.find('div',{'id':'pagetitle'}).findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))[-1]
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
|
|
@ -110,7 +110,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
#self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
|
||||
self.add_chapter(chapter,'https://{0}/{1}{2}'.format(self.host, chapter['href'],addURL))
|
||||
|
|
@ -127,7 +127,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
val = labelspan.nextSibling
|
||||
value = unicode('')
|
||||
|
|
@ -149,27 +149,27 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', stripHTML(value))
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Pairing' in label:
|
||||
ships = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
for ship in ships:
|
||||
self.story.addToList('ships',ship.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -196,12 +196,12 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# this site has several links to each story.
|
||||
if a.text == 'Latest Chapter':
|
||||
if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
|
|
|
|||
220
fanficfare/adapters/adapter_csiforensicscom.py
Normal file
220
fanficfare/adapters/adapter_csiforensicscom.py
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
|
||||
def getClass():
|
||||
return CSIForensicsComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class CSIForensicsComAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','csiforensics')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d %b %Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'csi-forensics.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=5&skin=elegantcsi"
|
||||
else:
|
||||
addurl="&skin=elegantcsi"
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
# The actual text that is used to announce you need to be an
|
||||
# adult varies from site to site. Again, print data before
|
||||
# the title search to troubleshoot.
|
||||
if "This story is rated NC-17, and therefore is not suitable for minors. If you are below the age required to view such material in your locality, please return from whence you came." in data: # XXX
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
|
||||
pt = soup.find('div', {'id' : 'pagetitle'})
|
||||
a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',a.string)
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Rating
|
||||
rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
|
||||
rate = rate[rate.rindex('[')+1:rate.rindex(']')]
|
||||
self.story.setMetadata('rating', rate)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
smalldiv = soup.find('div', {'class' : 'small'})
|
||||
|
||||
|
||||
chars = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
metatext = stripHTML(smalldiv)
|
||||
|
||||
if 'Completed: Yes' in metatext:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
word=soup.find(text=re.compile("Word count:")).split(':')
|
||||
self.story.setMetadata('numWords', word[1])
|
||||
|
||||
cats = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
warnings = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=class(&)type_id=2(&)classid=\d+'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
date=soup.find('div',{'class' : 'bottom'})
|
||||
pd=date.find(text=re.compile("Published:")).string.split(': ')
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat))
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
pub=0
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Genres' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
smalldiv.extract()
|
||||
|
||||
# Summary
|
||||
summary = soup.find('div', {'class' : 'content'})
|
||||
self.setDescription(url,summary)
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
222
fanficfare/adapters/adapter_destinysgatewaycom.py
Normal file
222
fanficfare/adapters/adapter_destinysgatewaycom.py
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return DestinysGatewayComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class DestinysGatewayComAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','dgrfa')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%b %d %Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.destinysgateway.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=").replace(r"www\.",r"(www\.)?")+r"\d+$"
|
||||
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&warning=4"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while value and 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -25,7 +25,6 @@ from ..six.moves.urllib.parse import urlparse
|
|||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
from fanficfare.htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
from fanficfare.dateutils import parse_relative_date_string
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -74,126 +73,81 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
|
|||
return r'https?://www\.deviantart\.com/(?P<author>[^/]+)/art/(?P<id>[^/]+)/?'
|
||||
|
||||
def performLogin(self, url):
|
||||
if self.username and self.username != 'NoneGiven':
|
||||
username = self.username
|
||||
else:
|
||||
username = self.getConfig('username')
|
||||
|
||||
# logger.debug("\n\nusername:(%s)\n\n"%username)
|
||||
if not username:
|
||||
logger.info("Login Required for URL %s" % url)
|
||||
raise exceptions.FailedToLogin(url,username)
|
||||
|
||||
data = self.get_request_raw('https://www.deviantart.com/users/login', referer=url, usecache=False)
|
||||
data = self.get_request_raw('https://www.deviantart.com/users/login', referer=url)
|
||||
data = self.decode_data(data)
|
||||
soup = self.make_soup(data)
|
||||
params = {
|
||||
'referer': 'https://www.deviantart.com/_sisu/do/signin', # soup.find('input', {'name': 'referer'})['value'],
|
||||
'referer_type': soup.find('input', {'name': 'referer_type'})['value'],
|
||||
'referer': url,
|
||||
'csrf_token': soup.find('input', {'name': 'csrf_token'})['value'],
|
||||
'challenge': soup.find('input', {'name': 'challenge'})['value'],
|
||||
'lu_token': soup.find('input', {'name': 'lu_token'})['value'],
|
||||
'remember': 'on',
|
||||
'username': username
|
||||
}
|
||||
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/_sisu/do/step2'
|
||||
logger.debug('Will now login to deviantARt as (%s)' % username)
|
||||
|
||||
result = self.post_request(loginUrl, params, usecache=False)
|
||||
soup = self.make_soup(result)
|
||||
if not soup.find('input', {'name': 'lu_token2'}):
|
||||
logger.info("Login Failed for URL %s (no lu_token2 found)" % url)
|
||||
raise exceptions.FailedToLogin(url,username)
|
||||
|
||||
params = {
|
||||
'referer': 'https://www.deviantart.com/_sisu/do/signin', # soup.find('input', {'name': 'referer'})['value'],
|
||||
'referer_type': soup.find('input', {'name': 'referer_type'})['value'],
|
||||
'csrf_token': soup.find('input', {'name': 'csrf_token'})['value'],
|
||||
'challenge': soup.find('input', {'name': 'challenge'})['value'],
|
||||
'lu_token': soup.find('input', {'name': 'lu_token'})['value'],
|
||||
'lu_token2': soup.find('input', {'name': 'lu_token2'})['value'],
|
||||
'remember': 'on',
|
||||
'username': ''
|
||||
}
|
||||
|
||||
if self.password:
|
||||
params['username'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['username'] = self.getConfig('username')
|
||||
params['password'] = self.getConfig('password')
|
||||
|
||||
# logger.debug("\n\nparams['password']:(%s)\n\n"%params['password'])
|
||||
loginUrl = 'https://' + self.getSiteDomain() + '/_sisu/do/signin'
|
||||
logger.debug('Will now send password to deviantARt')
|
||||
logger.debug('Will now login to deviantARt as (%s)' % params['username'])
|
||||
|
||||
result = self.post_request(loginUrl, params, usecache=False)
|
||||
|
||||
if 'Log In | DeviantArt' in result:
|
||||
logger.error('Failed to login to deviantArt as %s' % username)
|
||||
raise exceptions.FailedToLogin('https://www.deviantart.com', username)
|
||||
logger.error('Failed to login to deviantArt as %s' % params['username'])
|
||||
raise exceptions.FailedToLogin('https://www.deviantart.com', params['username'])
|
||||
else:
|
||||
return True
|
||||
|
||||
def requiresLogin(self, data):
|
||||
return '</a> has limited the viewing of this artwork to members of the DeviantArt community only' in data
|
||||
|
||||
def isLoggedIn(self, data):
|
||||
return '<form id="logout-form" action="https://www.deviantart.com/users/logout" method="POST">' in data
|
||||
|
||||
def isWatchersOnly(self, data):
|
||||
return '>Watchers-Only Deviation<' in data
|
||||
return '<span>Watchers-Only Deviation</span>' in data
|
||||
|
||||
def requiresMatureContentEnabled(self, data):
|
||||
return (
|
||||
'>This content is intended for mature audiences<' in data
|
||||
or '>This deviation is intended for mature audiences<' in data
|
||||
or '>This filter hides content that may be inappropriate for some viewers<' in data
|
||||
or '>May contain sensitive content<' in data
|
||||
or '>Log in to view<' in data
|
||||
or '>This deviation has been labeled as containing themes not suitable for all deviants.<' in data
|
||||
)
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
isLoggedIn = False
|
||||
logger.debug('URL: %s', self.url)
|
||||
|
||||
data = self.get_request(self.url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
## story can require login outright, or it can show up as
|
||||
## watchers-only or mature-enabled without the same 'requires
|
||||
## login' strings.
|
||||
if self.requiresLogin(data) or ( not self.isLoggedIn(data) and
|
||||
(self.isWatchersOnly(data) or
|
||||
self.requiresMatureContentEnabled(data)) ):
|
||||
if self.requiresLogin(data):
|
||||
if self.performLogin(self.url):
|
||||
isLoggedIn = True
|
||||
data = self.get_request(self.url, usecache=False)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
## Check watchers only and mature enabled again, separately,
|
||||
## after login because they can still apply after login.
|
||||
if self.isWatchersOnly(data):
|
||||
raise exceptions.FailedToDownload(
|
||||
'Deviation is only available for watchers.' +
|
||||
'You must watch this author before you can download it.'
|
||||
)
|
||||
if self.requiresMatureContentEnabled(data):
|
||||
raise exceptions.FailedToDownload(
|
||||
'Deviation is set as mature, you must go into your account ' +
|
||||
'and enable showing of mature content.'
|
||||
)
|
||||
)
|
||||
|
||||
appurl = soup.select_one('meta[property="og:url"]')['content']
|
||||
if appurl:
|
||||
story_id = urlparse(appurl).path.lstrip('/')
|
||||
else:
|
||||
logger.debug("Looking for JS story id")
|
||||
## after login, this is only found in a JS block. Dunno why.
|
||||
## F875A309-B0DB-860E-5079-790D0FBE5668
|
||||
match = re.match(r'\\"deviationUuid\\":\\"(?P<id>[A-Z0-9-]+)\\",',data)
|
||||
if match:
|
||||
story_id = match.group('id')
|
||||
else:
|
||||
raise exceptions.FailedToDownload('Failed to find Story ID.')
|
||||
if self.requiresMatureContentEnabled(data):
|
||||
# as far as I can tell deviantArt has no way to show mature
|
||||
# content that doesn't involve logging in or using JavaScript
|
||||
if not isLoggedIn:
|
||||
self.performLogin(self.url)
|
||||
isLoggedIn = True
|
||||
data = self.get_request(self.url, usecache=False)
|
||||
soup = self.make_soup(data)
|
||||
if self.requiresMatureContentEnabled(data):
|
||||
raise exceptions.FailedToDownload(
|
||||
'Deviation is set as mature, you must go into your account ' +
|
||||
'and enable showing of mature content.'
|
||||
)
|
||||
|
||||
appurl = soup.select_one('meta[property="da:appurl"]')['content']
|
||||
story_id = urlparse(appurl).path.lstrip('/')
|
||||
self.story.setMetadata('storyId', story_id)
|
||||
|
||||
title = soup.select_one('h1').get_text()
|
||||
|
|
@ -203,12 +157,7 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
|
|||
# self.story.setMetadata('status', 'Completed')
|
||||
|
||||
pubdate = soup.select_one('time').get_text()
|
||||
|
||||
# Maybe do this better, but this works
|
||||
try:
|
||||
self.story.setMetadata('datePublished', makeDate(pubdate, '%b %d, %Y'))
|
||||
except:
|
||||
self.story.setMetadata('datePublished', parse_relative_date_string(pubdate))
|
||||
self.story.setMetadata('datePublished', makeDate(pubdate, '%b %d, %Y'))
|
||||
|
||||
# do description here if appropriate
|
||||
|
||||
|
|
@ -222,35 +171,19 @@ class DeviantArtComSiteAdapter(BaseSiteAdapter):
|
|||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from: %s', url)
|
||||
data = self.get_request(url)
|
||||
# logger.debug(data)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
# remove comments section to avoid false matches
|
||||
comments = soup.select_one('[data-hook=comments_thread]')
|
||||
if comments:
|
||||
comments.decompose()
|
||||
# previous search not always found in some stories.
|
||||
# <div id="comments"></div> inside the real containing
|
||||
# div seems more common
|
||||
commentsdiv = soup.select_one('div#comments')
|
||||
if commentsdiv:
|
||||
commentsdiv.parent.decompose()
|
||||
|
||||
# three different 'content' tags to look for.
|
||||
# This is the current in Oct 2024
|
||||
content = soup.select_one('[data-editor-viewer="1"]')
|
||||
comments.decompose()
|
||||
|
||||
content = soup.select_one('[data-id=rich-content-viewer]')
|
||||
if content is None:
|
||||
# older story? I can't find any of this style in Oct2024
|
||||
content = soup.select_one('[data-id="rich-content-viewer"]')
|
||||
|
||||
if content is None:
|
||||
# olderer story, but used by some older (2018) posts
|
||||
# older story
|
||||
content = soup.select_one('.legacy-journal')
|
||||
|
||||
if content is None:
|
||||
raise exceptions.FailedToDownload(
|
||||
'Could not find story text. Please open a bug with the URL %s' % self.url
|
||||
if content is None:
|
||||
raise exceptions.FailedToDownload(
|
||||
'Could not find story text. Please open a bug with the URL %s' % self.url
|
||||
)
|
||||
|
||||
return self.utf8FromSoup(url, content)
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
params['Submit'] = 'Submit'
|
||||
|
||||
# copy all hidden input tags to pick up appropriate tokens.
|
||||
for tag in soup.find_all('input',{'type':'hidden'}):
|
||||
for tag in soup.findAll('input',{'type':'hidden'}):
|
||||
params[tag['name']] = tag['value']
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
|
||||
|
|
@ -153,7 +153,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
chapters = soup.find('select').find_all('option')
|
||||
chapters = soup.find('select').findAll('option')
|
||||
if len(chapters)==1:
|
||||
self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1')
|
||||
else:
|
||||
|
|
@ -168,7 +168,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
|
||||
|
||||
#grab the rest of the metadata from the author's page
|
||||
for div in asoup.find_all('div'):
|
||||
for div in asoup.findAll('div'):
|
||||
nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
|
||||
if nav != None:
|
||||
break
|
||||
|
|
@ -208,7 +208,7 @@ class DokugaComAdapter(BaseSiteAdapter):
|
|||
|
||||
else:
|
||||
asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
|
||||
for div in asoup.find_all('div'):
|
||||
for div in asoup.findAll('div'):
|
||||
nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
|
||||
if nav != None:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -181,13 +181,13 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
|
||||
self.setDescription(url,content.find('blockquote'))
|
||||
|
||||
for genre in content.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
|
||||
for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
for warning in content.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
|
||||
for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
labels = content.find_all('b')
|
||||
labels = content.findAll('b')
|
||||
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
|
|
@ -208,22 +208,22 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -247,7 +247,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
# no selector found, so it's a one-chapter story.
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
allOptions = select.find_all('option', {'value' : re.compile(r'viewstory')})
|
||||
allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
|
||||
for o in allOptions:
|
||||
url = u'https://%s/%s' % ( self.getSiteDomain(),
|
||||
o['value'])
|
||||
|
|
@ -170,14 +170,14 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
if authsoup != None:
|
||||
# last author link with offset should be the 'next' link.
|
||||
authurl = u'https://%s/%s' % ( self.getSiteDomain(),
|
||||
authsoup.find_all('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
|
||||
authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
|
||||
|
||||
# Need author page for most of the metadata.
|
||||
logger.debug("fetching author page: (%s)"%authurl)
|
||||
authsoup = self.make_soup(self.get_request(authurl))
|
||||
#print("authsoup:%s"%authsoup)
|
||||
|
||||
storyas = authsoup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
|
||||
storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
|
||||
for storya in storyas:
|
||||
#print("======storya:%s"%storya)
|
||||
storyblock = storya.findParent('div',{'class':'storybloc'})
|
||||
|
|
@ -194,7 +194,7 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
# Tipo di coppia: Het | Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio | Note: OOC | Avvertimenti: Tematiche delicate<br />
|
||||
# Categoria: <a href="categories.php?catid=1&parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&a=">3</a> recensioni</div>
|
||||
|
||||
cats = noteblock.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
|
|
@ -262,7 +262,7 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
|
||||
|
|
@ -288,11 +288,11 @@ class EFPFanFicNet(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
# remove any header and 'o:p' tags.
|
||||
for tag in div.find_all("head") + div.find_all("o:p"):
|
||||
for tag in div.findAll("head") + div.findAll("o:p"):
|
||||
tag.extract()
|
||||
|
||||
# change any html and body tags to div.
|
||||
for tag in div.find_all("html") + div.find_all("body"):
|
||||
for tag in div.findAll("html") + div.findAll("body"):
|
||||
tag.name='div'
|
||||
|
||||
# remove extra bogus doctype.
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -144,7 +144,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
|
||||
value = labels[0].previousSibling
|
||||
svalue = ""
|
||||
|
|
@ -164,22 +164,22 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value.split(' -')[0])
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -204,7 +204,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -53,9 +53,6 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
#Setting the 'Zone' for each "Site"
|
||||
self.zone = self.parsedUrl.netloc.replace('.fanficauthors.net','')
|
||||
|
||||
# site change .nsns to -nsns
|
||||
self.zone = self.zone.replace('.nsns','-nsns')
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://{0}.{1}/{2}/'.format(
|
||||
self.zone, self.getBaseDomain(), self.story.getMetadata('storyId')))
|
||||
|
|
@ -82,10 +79,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
|
||||
# need both .nsns(old) and -nsns(new) because it's a domain
|
||||
# change, not just URL change.
|
||||
return ['aaran-st-vines.nsns.fanficauthors.net',
|
||||
'aaran-st-vines-nsns.fanficauthors.net',
|
||||
'abraxan.fanficauthors.net',
|
||||
'bobmin.fanficauthors.net',
|
||||
'canoncansodoff.fanficauthors.net',
|
||||
|
|
@ -101,12 +95,9 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
'jeconais.fanficauthors.net',
|
||||
'kinsfire.fanficauthors.net',
|
||||
'kokopelli.nsns.fanficauthors.net',
|
||||
'kokopelli-nsns.fanficauthors.net',
|
||||
'ladya.nsns.fanficauthors.net',
|
||||
'ladya-nsns.fanficauthors.net',
|
||||
'lorddwar.fanficauthors.net',
|
||||
'mrintel.nsns.fanficauthors.net',
|
||||
'mrintel-nsns.fanficauthors.net',
|
||||
'musings-of-apathy.fanficauthors.net',
|
||||
'ruskbyte.fanficauthors.net',
|
||||
'seelvor.fanficauthors.net',
|
||||
|
|
@ -117,7 +108,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
################################################################################################
|
||||
@classmethod
|
||||
def getSiteExampleURLs(self):
|
||||
return ("https://aaran-st-vines-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
return ("https://aaran-st-vines.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://abraxan.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://bobmin.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://canoncansodoff.fanficauthors.net/A_Story_Name/ "
|
||||
|
|
@ -132,10 +123,10 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
+ "https://jbern.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://jeconais.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://kinsfire.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://kokopelli-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://ladya-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://kokopelli.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://ladya.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://lorddwar.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://mrintel-nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://mrintel.nsns.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://musings-of-apathy.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://ruskbyte.fanficauthors.net/A_Story_Name/ "
|
||||
+ "https://seelvor.fanficauthors.net/A_Story_Name/ "
|
||||
|
|
@ -145,16 +136,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
|
||||
################################################################################################
|
||||
def getSiteURLPattern(self):
|
||||
## .nsns kept here to match both . and -
|
||||
return r'https?://(aaran-st-vines.nsns|abraxan|bobmin|canoncansodoff|chemprof|copperbadge|crys|deluded-musings|draco664|fp|frenchsession|ishtar|jbern|jeconais|kinsfire|kokopelli.nsns|ladya.nsns|lorddwar|mrintel.nsns|musings-of-apathy|ruskbyte|seelvor|tenhawk|viridian|whydoyouneedtoknow)\.fanficauthors\.net/([a-zA-Z0-9_]+)/'
|
||||
|
||||
@classmethod
|
||||
def get_section_url(cls,url):
|
||||
## only changing .nsns to -nsns and only when part of the
|
||||
## domain.
|
||||
url = url.replace('.nsns.fanficauthors.net','-nsns.fanficauthors.net')
|
||||
return url
|
||||
|
||||
################################################################################################
|
||||
def doExtractChapterUrlsAndMetadata(self, get_cover=True):
|
||||
|
||||
|
|
@ -180,7 +163,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
# Find the chapters:
|
||||
# The published and update dates are with the chapter links...
|
||||
# so we have to get them from there.
|
||||
chapters = soup.find_all('a', href=re.compile('/'+self.story.getMetadata(
|
||||
chapters = soup.findAll('a', href=re.compile('/'+self.story.getMetadata(
|
||||
'storyId')+'/([a-zA-Z0-9_]+)/'))
|
||||
|
||||
# Here we are getting the published date. It is the date the first chapter was "updated"
|
||||
|
|
@ -219,7 +202,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
## Raising AdultCheckRequired after collecting chapters gives
|
||||
## a double chapter list. So does genre, but it de-dups
|
||||
## automatically.
|
||||
if( self.story.getMetadataRaw('rating') in ['Mature','Adult Only']
|
||||
if( self.story.getMetadata('rating') == 'Mature'
|
||||
and not (self.is_adult or self.getConfig("is_adult")) ):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
|
|
@ -243,7 +226,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
if( self.story.getMetadataRaw('rating') in ['Mature','Adult Only'] and
|
||||
if( self.story.getMetadata('rating') == 'Mature' and
|
||||
(self.is_adult or self.getConfig("is_adult")) ):
|
||||
addurl = "?bypass=1"
|
||||
else:
|
||||
|
|
@ -258,8 +241,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
|
|||
"Error downloading Chapter: '{0}'! Missing required element!".format(url))
|
||||
|
||||
#Now, there are a lot of extranious tags within the story division.. so we will remove them.
|
||||
for tag in story.find_all('ul',{'class':'pager'}) + story.find_all(
|
||||
'div',{'class':'alert'}) + story.find_all('div', {'class':'btn-group'}):
|
||||
for tag in story.findAll('ul',{'class':'pager'}) + story.findAll(
|
||||
'div',{'class':'alert'}) + story.findAll('div', {'class':'btn-group'}):
|
||||
tag.extract()
|
||||
|
||||
return self.utf8FromSoup(url,story)
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
|
|||
## restrict meta searches to header.
|
||||
fichead = soup.find('div',class_='FicHead')
|
||||
def get_meta_content(title):
|
||||
val_label = fichead.find('div',string=re.compile(u'^'+title+u':'))
|
||||
val_label = fichead.find('div',string=title+u':')
|
||||
if val_label:
|
||||
return val_label.find_next('div')
|
||||
|
||||
|
|
@ -150,7 +150,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating',stripHTML(get_meta_content(u'Рейтинг')))
|
||||
|
||||
## Need to login for any rating higher than General.
|
||||
if self.story.getMetadataRaw('rating') != 'General' and self.needToLoginCheck(data):
|
||||
if self.story.getMetadata('rating') != 'General' and self.needToLoginCheck(data):
|
||||
self.performLogin(url)
|
||||
# reload after login.
|
||||
data = self.get_request(url,usecache=False)
|
||||
|
|
@ -168,7 +168,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title',stripHTML(h))
|
||||
|
||||
## author(s):
|
||||
content = get_meta_content(u'Авторы?')
|
||||
content = get_meta_content(u'Автор')
|
||||
if content:
|
||||
alist = content.find_all('a', class_='user')
|
||||
for a in alist:
|
||||
|
|
@ -181,8 +181,12 @@ class FanFicsMeAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('authorUrl','https://'+self.host)
|
||||
self.story.setMetadata('authorId','0')
|
||||
|
||||
# translator(s) in different strings
|
||||
content = get_meta_content(u'Переводчикк?и?')
|
||||
# translator(s)
|
||||
content = get_meta_content(u'Переводчик')
|
||||
if not content:
|
||||
# Переводчик vs Переводчи is 'Translator' vs 'TranslatorS'
|
||||
content = get_meta_content(u'Переводчи')
|
||||
logger.debug(content)
|
||||
if content:
|
||||
for a in content.find_all('a', class_='user'):
|
||||
self.story.addToList('translatorsId',a['href'].split('/user')[-1])
|
||||
|
|
@ -297,10 +301,6 @@ class FanFicsMeAdapter(BaseSiteAdapter):
|
|||
# grab the text for an individual chapter.
|
||||
def getChapterTextNum(self, url, index):
|
||||
logger.debug('Getting chapter text for: %s index: %s' % (url,index))
|
||||
m = re.match(r'.*&chapter=(\d+).*',url)
|
||||
if m:
|
||||
index=m.group(1)
|
||||
logger.debug("Using index(%s) from &chapter="%index)
|
||||
|
||||
chapter_div = None
|
||||
if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and self.num_chapters() > 1:
|
||||
|
|
|
|||
|
|
@ -44,8 +44,9 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self._setURL('https://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ahpfftc')
|
||||
|
|
@ -56,24 +57,24 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return [cls.getSiteDomain(),'archive.hpfanfictalk.com','fanfictalk.com']
|
||||
return [cls.getSiteDomain(),'archive.hpfanfictalk.com']
|
||||
|
||||
@classmethod
|
||||
def getConfigSections(cls):
|
||||
"Only needs to be overriden if has additional ini sections."
|
||||
return [cls.getConfigSection(),'archive.hpfanfictalk.com','fanfictalk.com']
|
||||
return [cls.getConfigSection(),'archive.hpfanfictalk.com']
|
||||
|
||||
@staticmethod # must be @stgetAcceptDomainsaticmethod, don't remove it.
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'archive.fanfictalk.com'
|
||||
return 'fanfictalk.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
return "https://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://("+r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()])+r")(/archive)?/viewstory\.php\?sid=\d+$"
|
||||
return r"https?://(archive\.hp)?"+re.escape(self.getSiteDomain())+r"(/archive)?/viewstory\.php\?sid=\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
|
@ -117,7 +118,7 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href'])
|
||||
self.add_chapter(chapter,'https://'+self.host+'/archive/'+chapter['href'])
|
||||
|
||||
# categories
|
||||
for a in soup.select("div#sort a"):
|
||||
|
|
@ -170,14 +171,14 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# Site allows stories to be in several series at once. FFF
|
||||
# isn't thrilled with that, we have series00, series01, etc.
|
||||
# Example:
|
||||
# https://archive.fanfictalk.com/viewstory.php?sid=483
|
||||
# https://fanfictalk.com/archive/viewstory.php?sid=483
|
||||
|
||||
if self.getConfig("collect_series"):
|
||||
seriesspan = soup.find('span',label='Series')
|
||||
for i, seriesa in enumerate(seriesspan.find_all('a', href=re.compile(r"viewseries\.php\?seriesid=\d+"))):
|
||||
# logger.debug(seriesa)
|
||||
series_name = stripHTML(seriesa)
|
||||
series_url = 'https://'+self.host+'/'+seriesa['href']
|
||||
series_url = 'https://'+self.host+'/archive/'+seriesa['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
|
|
@ -204,17 +205,9 @@ class FanfictalkComAdapter(BaseSiteAdapter):
|
|||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=3"
|
||||
else:
|
||||
addurl=""
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % (url+addurl))
|
||||
soup = self.make_soup(self.get_request(url+addurl))
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
|
|
|
|||
|
|
@ -93,47 +93,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
# logger.debug("post-url:%s"%url)
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def get_url_search(cls,url):
|
||||
regexp = super(getClass(), cls).get_url_search(url)
|
||||
regexp = re.sub(r"^(?P<keep>.*net/s/\d+/\d+/)(?P<urltitle>[^\$]*)?",
|
||||
r"\g<keep>(.*)",regexp)
|
||||
logger.debug(regexp)
|
||||
return regexp
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return self._get_site_url_pattern()
|
||||
|
||||
## normalized chapter URLs DO contain the story title now, but
|
||||
## normalized to current urltitle in case of title changes.
|
||||
## not actually putting urltitle on multi-chapters below, but
|
||||
## one-shots will have it, so this is still useful. normalized
|
||||
## chapter URLs do NOT contain the story title.
|
||||
def normalize_chapterurl(self,url):
|
||||
return re.sub(r"https?://(www|m)\.(?P<keep>fanfiction\.net/s/\d+/\d+/).*",
|
||||
r"https://www.\g<keep>",url)+self.urltitle
|
||||
|
||||
def get_request(self,url,usecache=True):
|
||||
## use super version if not set or isn't a chapter URL with a
|
||||
## title.
|
||||
if( not self.getConfig("try_shortened_title_urls") or
|
||||
not re.match(r"https?://www\.fanfiction\.net/s/\d+/\d+/(?P<title>[^/]+)$", url) ):
|
||||
return super(getClass(), self).get_request(url,usecache)
|
||||
|
||||
## kludgey way to attempt more than one URL variant by
|
||||
## removing title one letter at a time. Note that network and
|
||||
## open_pages_in_browser retries still happen first.
|
||||
titlelen = len(url.split('/')[-1])
|
||||
maxcut = min([4,titlelen])
|
||||
j = 0
|
||||
while j < maxcut: # should actually leave loop either by
|
||||
# return or exception raise.
|
||||
try:
|
||||
useurl = url
|
||||
if j: # j==0, full URL, then remove letters.
|
||||
useurl = url[:-j]
|
||||
return super(getClass(), self).get_request(useurl,usecache)
|
||||
except exceptions.HTTPErrorFFF as fffe:
|
||||
if j >= maxcut or 'Page not found or expired' not in unicode(fffe):
|
||||
raise
|
||||
j = j+1
|
||||
r"https://www.\g<keep>",url)
|
||||
|
||||
def doExtractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
|
||||
|
|
@ -167,7 +135,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
## the first chapter. It generates another server request and
|
||||
## doesn't seem to be needed lately, so now default it to off.
|
||||
try:
|
||||
chapcount = len(soup.find('select', { 'name' : 'chapter' } ).find_all('option'))
|
||||
chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
|
||||
# get chapter part of url.
|
||||
except:
|
||||
chapcount = 1
|
||||
|
|
@ -212,7 +180,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
## For 1, use the second link.
|
||||
## For 2, fetch the crossover page and pull the two categories from there.
|
||||
pre_links = soup.find('div',{'id':'pre_story_links'})
|
||||
categories = pre_links.find_all('a',{'class':'xcontrast_txt'})
|
||||
categories = pre_links.findAll('a',{'class':'xcontrast_txt'})
|
||||
#print("xcontrast_txt a:%s"%categories)
|
||||
if len(categories) > 1:
|
||||
# Strangely, the ones with *two* links are the
|
||||
|
|
@ -251,7 +219,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
|
||||
# for b in grayspan.find_all('button'):
|
||||
# for b in grayspan.findAll('button'):
|
||||
# b.extract()
|
||||
metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
|
||||
#logger.debug("metatext:(%s)"%metatext)
|
||||
|
|
@ -290,7 +258,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
|
||||
# Published: <span data-xutime='1384358726'>8m ago</span>
|
||||
dates = soup.find_all('span',{'data-xutime':re.compile(r'^\d+$')})
|
||||
dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
|
||||
if len(dates) > 1 :
|
||||
# updated get set to the same as published upstream if not found.
|
||||
self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
|
||||
|
|
@ -341,10 +309,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
img = soup.select_one('img.lazy.cimage')
|
||||
cover_url=img['data-original']
|
||||
except:
|
||||
## Nov 2023 - src is always "/static/images/d_60_90.jpg" now
|
||||
## Only take cover if there's data-original
|
||||
## Primary motivator is to prevent unneeded author page hits.
|
||||
pass
|
||||
img = soup.select_one('img.cimage:not(.lazy)')
|
||||
if img:
|
||||
cover_url=img['src']
|
||||
## Nov 19, 2020, ffnet lazy cover images returning 0 byte
|
||||
## files.
|
||||
logger.debug("cover_url:%s"%cover_url)
|
||||
|
||||
authimg_url = ""
|
||||
|
|
@ -395,37 +364,31 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
# no selector found, so it's a one-chapter story.
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
allOptions = select.find_all('option')
|
||||
allOptions = select.findAll('option')
|
||||
for o in allOptions:
|
||||
## title URL will be put back on chapter URL during
|
||||
## normalize_chapterurl() anyway, but also here for
|
||||
## clarity
|
||||
url = u'https://%s/s/%s/%s/%s' % ( self.getSiteDomain(),
|
||||
self.story.getMetadata('storyId'),
|
||||
o['value'],
|
||||
self.urltitle)
|
||||
url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
|
||||
self.story.getMetadata('storyId'),
|
||||
o['value'])
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
title = u"%s" % o
|
||||
title = re.sub(r'<[^>]+>','',title)
|
||||
self.add_chapter(title,url)
|
||||
|
||||
|
||||
return
|
||||
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from: %s' % (url))
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
## title URL was put back on chapter URL during
|
||||
## normalize_chapterurl()
|
||||
data = self.get_request(url)
|
||||
## AND explicitly put title URL back on chapter URL for fetch
|
||||
## *only*--normalized chapter URL does NOT have urltitle
|
||||
data = self.get_request(url+self.urltitle)
|
||||
|
||||
if "Please email this error message in full to <a href='mailto:" in data:
|
||||
if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! FanFiction.net Site Error!" % url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
## remove inline ads -- only seen with flaresolverr
|
||||
for adtag in soup.select("div.google-auto-placed"):
|
||||
adtag.decompose()
|
||||
|
||||
div = soup.find('div', {'id' : 'storytextp'})
|
||||
|
||||
if None == div:
|
||||
|
|
|
|||
|
|
@ -1,157 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2024 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
from __future__ import absolute_import
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
from fanficfare.htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def getClass():
|
||||
return FanfictionsFrSiteAdapter
|
||||
|
||||
|
||||
class FanfictionsFrSiteAdapter(BaseSiteAdapter):
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
self.story.setMetadata('siteabbrev', 'fanfictionsfr')
|
||||
self.story.setMetadata('langcode','fr')
|
||||
self.story.setMetadata('language','Français')
|
||||
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
match = re.match(self.getSiteURLPattern(), url)
|
||||
if not match:
|
||||
raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
|
||||
|
||||
story_id = match.group('id')
|
||||
self.story.setMetadata('storyId', story_id)
|
||||
fandom_name = match.group('fandom')
|
||||
|
||||
self._setURL('https://%s/fanfictions/%s/%s/chapters.html' % (self.getSiteDomain(), fandom_name, story_id))
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'www.fanfictions.fr'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return 'https://%s/fanfictions/fandom/fanfiction-id/chapters.html' % cls.getSiteDomain()
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r'https?://(?:www\.)?fanfictions\.fr/fanfictions/(?P<fandom>[^/]+)/(?P<id>[^/]+)(/chapters.html)?'
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
logger.debug('URL: %s', self.url)
|
||||
|
||||
data = self.get_request(self.url)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
# detect if the fanfiction is 'suspended' (chapters unavailable)
|
||||
alert_div = soup.find('div', id='alertInactiveFic')
|
||||
if alert_div:
|
||||
raise exceptions.FailedToDownload("Failed to download the fanfiction, most likely because it is suspended.")
|
||||
|
||||
title_element = soup.find('h1', itemprop='name')
|
||||
self.story.setMetadata('title', stripHTML(title_element))
|
||||
|
||||
author_div = soup.find('div', itemprop='author')
|
||||
author_name = stripHTML(author_div.a)
|
||||
author_id = author_div.a['href'].split('/')[-1].replace('.html', '')
|
||||
|
||||
self.story.setMetadata('author', author_name)
|
||||
self.story.setMetadata('authorId', author_id)
|
||||
|
||||
published_date_element = soup.find('span', class_='date-distance')
|
||||
published_date_text = published_date_element['data-date']
|
||||
published_date = makeDate(published_date_text, '%Y-%m-%d %H:%M:%S')
|
||||
if published_date:
|
||||
self.story.setMetadata('datePublished', published_date)
|
||||
|
||||
status_element = soup.find('p', title="Statut de la fanfiction").find('span', class_='badge')
|
||||
french_status = stripHTML(status_element)
|
||||
status_translation = {
|
||||
"En cours": "In-Progress",
|
||||
"Terminée": "Completed",
|
||||
"One-shot": "Completed",
|
||||
}
|
||||
self.story.setMetadata('status', status_translation.get(french_status, french_status))
|
||||
|
||||
genre_elements = soup.find('div', title="Format et genres").find_all('span', class_="highlightable")
|
||||
self.story.extendList('genre', [ stripHTML(genre) for genre in genre_elements[1:] ])
|
||||
|
||||
category_elements = soup.find_all('li', class_="breadcrumb-item")
|
||||
self.story.extendList('category', [ stripHTML(category) for category in category_elements[-2].find_all('a') ])
|
||||
|
||||
first_description = soup.find('p', itemprop='abstract')
|
||||
self.setDescription(self.url, first_description)
|
||||
|
||||
chapter_cards = soup.find_all(class_=['card', 'chapter'])
|
||||
|
||||
for chapter_card in chapter_cards:
|
||||
chapter_title_tag = chapter_card.find('h2')
|
||||
if chapter_title_tag:
|
||||
chapter_title = stripHTML(chapter_title_tag)
|
||||
chapter_link = 'https://'+self.getSiteDomain()+chapter_title_tag.find('a')['href']
|
||||
|
||||
# Clean up the chapter title by replacing multiple spaces and newline characters with a single space
|
||||
chapter_title = re.sub(r'\s+', ' ', chapter_title)
|
||||
|
||||
self.add_chapter(chapter_title, chapter_link)
|
||||
|
||||
last_chapter_div = chapter_cards[-1]
|
||||
updated_date_element = last_chapter_div.find('span', class_='date-distance')
|
||||
last_chapter_update_date = updated_date_element['data-date']
|
||||
date = makeDate(last_chapter_update_date, '%Y-%m-%d %H:%M:%S')
|
||||
if date:
|
||||
self.story.setMetadata('dateUpdated', date)
|
||||
|
||||
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
response, redirection_url = self.get_request_redirected(url)
|
||||
|
||||
if "telecharger_pdf.html" in redirection_url:
|
||||
with zipfile.ZipFile(io.BytesIO(response.encode('latin1'))) as z:
|
||||
# Assuming there's only one text file inside the zip
|
||||
file_list = z.namelist()
|
||||
if len(file_list) != 1:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Zip file should contain exactly one text file!" % url)
|
||||
text_filename = file_list[0]
|
||||
with z.open(text_filename) as text_file:
|
||||
# Decode the text file with windows-1252 encoding
|
||||
text = text_file.read().decode('windows-1252')
|
||||
return text.replace("\r\n", "<br>\r\n")
|
||||
else:
|
||||
soup = self.make_soup(response)
|
||||
|
||||
div_content = soup.find('div', id='readarea')
|
||||
if div_content is None:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url, div_content)
|
||||
|
|
@ -134,7 +134,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find('select').find_all('option'):
|
||||
for chapter in soup.find('select').findAll('option'):
|
||||
self.add_chapter(chapter,'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value'])
|
||||
|
||||
## title="Wörter" failed with max_zalgo:1
|
||||
|
|
@ -163,11 +163,11 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
except e:
|
||||
logger.debug("Failed to find native status:%s"%e)
|
||||
|
||||
if head.find('span',title='fertiggestellt'):
|
||||
if head.find('span',title='Fertiggestellt'):
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
elif head.find('span',title='pausiert'):
|
||||
elif head.find('span',title='Pausiert'):
|
||||
self.story.setMetadata('status', 'Paused')
|
||||
elif head.find('span',title='abgebrochen'):
|
||||
elif head.find('span',title='Abgebrochen'):
|
||||
self.story.setMetadata('status', 'Cancelled')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
|
@ -181,13 +181,13 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
|
||||
# #find metadata on the author's page
|
||||
# asoup = self.make_soup(self.get_request("https://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
|
||||
# tr=asoup.find_all('tr')
|
||||
# tr=asoup.findAll('tr')
|
||||
# for i in range(1,len(tr)):
|
||||
# a = tr[i].find('a')
|
||||
# if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
|
||||
# break
|
||||
|
||||
# td = tr[i].find_all('td')
|
||||
# td = tr[i].findAll('td')
|
||||
# self.story.addToList('category',stripHTML(td[2]))
|
||||
# self.story.setMetadata('rating', stripHTML(td[5]))
|
||||
# self.story.setMetadata('numWords', stripHTML(td[6]))
|
||||
|
|
@ -204,7 +204,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'storytext'})
|
||||
for a in div.find_all('script'):
|
||||
for a in div.findAll('script'):
|
||||
a.extract()
|
||||
|
||||
if None == div:
|
||||
|
|
|
|||
172
fanficfare/adapters/adapter_fastnovelsnet.py
Normal file
172
fanficfare/adapters/adapter_fastnovelsnet.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
####################################################################################################
|
||||
### Adapted by Rikkit on November 7. 2017
|
||||
###=================================================================================================
|
||||
### Tested with Calibre
|
||||
####################################################################################################
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
import re
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def getClass():
|
||||
''' Initializing the class '''
|
||||
return FastNovelNetAdapter
|
||||
|
||||
class FastNovelNetAdapter(BaseSiteAdapter):
|
||||
''' Adapter for FASTNOVEL.net '''
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.story.setMetadata('siteabbrev', 'fstnvl')
|
||||
|
||||
self.dateformat = '%d/%m/%Y'
|
||||
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
match = re.match(self.getSiteURLPattern(), url)
|
||||
if not match:
|
||||
raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
|
||||
|
||||
story_id = match.group('id')
|
||||
self.story.setMetadata('storyId', story_id)
|
||||
self._setURL('https://%s/%s/' % (self.getSiteDomain(), story_id))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'fastnovels.net'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return [cls.getSiteDomain(),'fastnovel.net']
|
||||
|
||||
@classmethod
|
||||
def getConfigSections(cls):
|
||||
"Only needs to be overriden if has additional ini sections."
|
||||
return [cls.getConfigSection(),'fastnovel.net']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://fastnovels.net/a-story-name-id"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
# https://fastnovels.net/ultimate-scheming-system-158/
|
||||
# also accept fastnovel.net
|
||||
return r"https?://fastnovels?\.net/(?P<id>[^/]+)"
|
||||
|
||||
## Normalized chapter URLs by changing old titlenum part to be
|
||||
## same as storyId.
|
||||
def normalize_chapterurl(self,url):
|
||||
# https://fastnovels.net/cultivation-chat-group8-29/chapter-25206.html
|
||||
return re.sub(r"\.net/.*(?P<keep>/chapter-\d+.html)",
|
||||
r".net/"+self.story.getMetadata('storyId')+r"\g<keep>",url)
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
logger.debug('URL: %s', self.url)
|
||||
|
||||
(data,rurl) = self.get_request_redirected(self.url)
|
||||
if rurl != self.url:
|
||||
match = re.match(self.getSiteURLPattern(), rurl)
|
||||
if not match:
|
||||
## shouldn't happen, but in case it does...
|
||||
raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs())
|
||||
|
||||
story_id = match.group('id')
|
||||
self.story.setMetadata('storyId', story_id)
|
||||
self._setURL('https://%s/%s/' % (self.getSiteDomain(), story_id))
|
||||
logger.debug("set to redirected url:%s"%self.url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
self.story.setMetadata('title', soup.find('h1').string)
|
||||
|
||||
for li in soup.select('.meta-data li'):
|
||||
label = li.select_one('label')
|
||||
if not label:
|
||||
continue
|
||||
|
||||
if label.string == "Author:":
|
||||
for a in li.select('a'):
|
||||
self.story.setMetadata('authorId', a["href"].split('/')[2])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+a["href"])
|
||||
self.story.setMetadata('author', a["title"])
|
||||
|
||||
if label.string == "Genre:":
|
||||
for a in li.select('a'):
|
||||
self.story.addToList('genre',a["title"])
|
||||
|
||||
if label.string == "Status:":
|
||||
if li.select_one('strong').string.strip() == "Completed":
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if label.string == "Last updated:":
|
||||
dateUpd = label.next_sibling.strip()
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(dateUpd), self.dateformat))
|
||||
|
||||
coverurl = soup.select_one('div.book-cover')["data-original"]
|
||||
if coverurl != "https://fastnovels.net/images/novel/default.jpg":
|
||||
self.setCoverImage(self.url, coverurl)
|
||||
|
||||
tags = soup.select_one('.tags')
|
||||
if tags:
|
||||
for a in tags.select("li.tag-item a"):
|
||||
self.story.addToList('tags', a["title"])
|
||||
# extract tags, because it inside description
|
||||
tags.extract()
|
||||
|
||||
# remove title from description
|
||||
soup.select_one('.film-content h3').extract()
|
||||
desc = soup.select_one('.film-content').extract()
|
||||
self.setDescription(self.url, desc)
|
||||
|
||||
|
||||
## number from end of storyId, taken this way in case it changes.
|
||||
# <input id="film_id" type="hidden" value="10667">
|
||||
film_id = soup.select_one('input#film_id')['value']
|
||||
ch_data = self.post_request('https://'+self.host+'/',
|
||||
parameters={'film_id': film_id,
|
||||
'list_chapter': '1'})
|
||||
# logger.debug(ch_data)
|
||||
ch_soup = self.make_soup(ch_data)
|
||||
# logger.debug(ch_soup)
|
||||
# for book in soup.select("#list-chapters .book"):
|
||||
# volume = book.select_one('.title a').string
|
||||
for a in ch_soup.select(".list-chapters a.chapter"):
|
||||
# title = volume + " " + stripHTML(a)
|
||||
title = stripHTML(a)
|
||||
self.add_chapter(title, 'https://' + self.host + a["href"])
|
||||
|
||||
def getChapterText(self, url):
|
||||
data = self.get_request(url)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
story = soup.select_one('#chapter-body')
|
||||
if not story:
|
||||
raise exceptions.FailedToDownload(
|
||||
"Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url, story)
|
||||
|
|
@ -15,16 +15,16 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import,unicode_literals
|
||||
# import datetime
|
||||
from __future__ import absolute_import
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
# from .. import translit
|
||||
from .. import translit
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions# as exceptions
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
|
|
@ -58,7 +58,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = u"%d %m %Y г., %H:%M"
|
||||
self.dateformat = "%d %m %Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
|
|
@ -67,33 +67,18 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/readfic/12345 https://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content https://"+cls.getSiteDomain()+"/readfic/578de1cd-a8b4-7ff1-aa49-750426508b82 https://"+cls.getSiteDomain()+"/readfic/578de1cd-a8b4-7ff1-aa49-750426508b82/94793742#part_content"
|
||||
return "https://"+cls.getSiteDomain()+"/readfic/12345 https://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/readfic/")+r"[\d\-a-zA-Z]+"
|
||||
|
||||
def performLogin(self,url,data):
|
||||
params = {}
|
||||
if self.password:
|
||||
params['login'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['login'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
|
||||
logger.debug("Try to login in as (%s)" % params['login'])
|
||||
d = self.post_request('https://' + self.getSiteDomain() + '/login_check_static',params,usecache=False)
|
||||
|
||||
if 'Войти используя аккаунт на сайте' in d:
|
||||
raise exceptions.FailedToLogin(url,params['login'])
|
||||
|
||||
return True
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/readfic/")+r"\d+"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
url=self.url
|
||||
logger.debug("URL: "+url)
|
||||
data = self.get_request(url)
|
||||
logger.debug(data)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
adult_div = soup.find('div',id='adultCoverWarning')
|
||||
|
|
@ -103,11 +88,9 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
|
||||
## Title
|
||||
try:
|
||||
a = soup.find('section',{'class':'chapter-info'}).find('h1')
|
||||
except AttributeError:
|
||||
raise exceptions.FailedToDownload("Error collecting meta: %s! Missing required element!" % url)
|
||||
a = soup.find('section',{'class':'chapter-info'}).find('h1')
|
||||
# kill '+' marks if present.
|
||||
sup = a.find('sup')
|
||||
if sup:
|
||||
|
|
@ -117,12 +100,40 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
# Find authorid and URL from... author url.
|
||||
# assume first avatar-nickname -- there can be a second marked 'beta'.
|
||||
a = soup.find('a',{'class':'creator-username'})
|
||||
a = soup.find('a',{'class':'creator-nickname'})
|
||||
self.story.setMetadata('authorId',a.text) # Author's name is unique
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
|
||||
self.story.setMetadata('author',a.text)
|
||||
logger.debug("Author: (%s)"%self.story.getMetadata('author'))
|
||||
|
||||
# Find the chapters:
|
||||
pubdate = None
|
||||
chapters = soup.find('ul', {'class' : 'list-of-fanfic-parts'})
|
||||
if chapters != None:
|
||||
for chapdiv in chapters.findAll('li', {'class':'part'}):
|
||||
chapter=chapdiv.find('a',href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+r"/\d+#part_content$"))
|
||||
churl='https://'+self.host+chapter['href']
|
||||
self.add_chapter(chapter,churl)
|
||||
|
||||
datespan = chapdiv.find('span')
|
||||
if pubdate == None and datespan:
|
||||
pubdate = translit.translit(stripHTML(datespan))
|
||||
update = translit.translit(stripHTML(datespan))
|
||||
else:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
self.story.setMetadata('numChapters',1)
|
||||
pubdate=translit.translit(stripHTML(soup.find('div',{'class':'title-area'}).find('span')))
|
||||
update=pubdate
|
||||
|
||||
logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
|
||||
|
||||
if not ',' in pubdate:
|
||||
pubdate=datetime.date.today().strftime(self.dateformat)
|
||||
if not ',' in update:
|
||||
update=datetime.date.today().strftime(self.dateformat)
|
||||
pubdate=pubdate.split(',')[0]
|
||||
update=update.split(',')[0]
|
||||
|
||||
fullmon = {"yanvarya":"01", u"января":"01",
|
||||
"fievralya":"02", u"февраля":"02",
|
||||
"marta":"03", u"марта":"03",
|
||||
|
|
@ -136,50 +147,31 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
"noyabrya":"11", u"ноября":"11",
|
||||
"diekabrya":"12", u"декабря":"12" }
|
||||
|
||||
# Find the chapters:
|
||||
pubdate = None
|
||||
chapters = soup.find('ul', {'class' : 'list-of-fanfic-parts'})
|
||||
if chapters is not None:
|
||||
for chapdiv in chapters.find_all('li', {'class':'part'}):
|
||||
chapter=chapdiv.find('a',href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+r"/\d+#part_content$"))
|
||||
churl='https://'+self.host+chapter['href']
|
||||
for (name,num) in fullmon.items():
|
||||
if name in pubdate:
|
||||
pubdate = pubdate.replace(name,num)
|
||||
if name in update:
|
||||
update = update.replace(name,num)
|
||||
|
||||
# Find the chapter dates.
|
||||
date_str = chapdiv.find('span', {'title': True})['title'].replace(u"\u202fг. в", "")
|
||||
for month_name, month_num in fullmon.items():
|
||||
date_str = date_str.replace(month_name, month_num)
|
||||
chapterdate = makeDate(date_str,self.dateformat)
|
||||
self.add_chapter(chapter,churl,
|
||||
{'date':chapterdate.strftime(self.getConfig("datechapter_format",self.getConfig("datePublished_format",self.dateformat)))})
|
||||
|
||||
if pubdate is None and chapterdate:
|
||||
pubdate = chapterdate
|
||||
update = chapterdate
|
||||
else:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
date_str = soup.find('div', {'class' : 'part-date'}).find('span', {'title': True})['title'].replace(u"\u202fг. в", "")
|
||||
for month_name, month_num in fullmon.items():
|
||||
date_str = date_str.replace(month_name, month_num)
|
||||
pubdate = update = makeDate(date_str,self.dateformat)
|
||||
|
||||
logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
|
||||
|
||||
self.story.setMetadata('dateUpdated', update)
|
||||
self.story.setMetadata('datePublished', pubdate)
|
||||
self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
|
||||
self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
|
||||
self.story.setMetadata('language','Russian')
|
||||
|
||||
dlinfo = soup.select_one('header.d-flex.flex-column.gap-12.word-break')
|
||||
## after site change, I don't see word count anywhere.
|
||||
# pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
|
||||
# pr='https://'+self.host+pr['href']
|
||||
# pr = self.make_soup(self.get_request(pr))
|
||||
# pr=pr.findAll('div', {'class' : 'part_text'})
|
||||
# i=0
|
||||
# for part in pr:
|
||||
# i=i+len(stripHTML(part).split(' '))
|
||||
# self.story.setMetadata('numWords', unicode(i))
|
||||
|
||||
series_label = dlinfo.select_one('div.description.word-break').find('strong', string='Серия:')
|
||||
logger.debug('Series: %s'%str(series_label))
|
||||
if series_label:
|
||||
series_div = series_label.find_next_sibling("div")
|
||||
# No accurate series number as for that, additional request needs to be made
|
||||
self.setSeries(stripHTML(series_div.a), 1)
|
||||
self.story.setMetadata('seriesUrl','https://' + self.getSiteDomain() + series_div.a.get('href'))
|
||||
|
||||
dlinfo = soup.find('div',{'class':'fanfic-main-info'})
|
||||
|
||||
i=0
|
||||
fandoms = dlinfo.select_one('div:not([class])').find_all('a', href=re.compile(r'/fanfiction/\w+'))
|
||||
fandoms = dlinfo.find('div').findAll('a', href=re.compile(r'/fanfiction/\w+'))
|
||||
for fandom in fandoms:
|
||||
self.story.addToList('category',fandom.string)
|
||||
i=i+1
|
||||
|
|
@ -188,16 +180,13 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
|
||||
tags = soup.find('div',{'class':'tags'})
|
||||
if tags:
|
||||
for genre in tags.find_all('a',href=re.compile(r'/tags/')):
|
||||
for genre in tags.findAll('a',href=re.compile(r'/tags/')):
|
||||
self.story.addToList('genre',stripHTML(genre))
|
||||
|
||||
logger.debug("category: (%s)"%self.story.getMetadata('category'))
|
||||
logger.debug("genre: (%s)"%self.story.getMetadata('genre'))
|
||||
|
||||
ratingdt = dlinfo.find('div',{'class':re.compile(r'badge-rating-.*')})
|
||||
self.story.setMetadata('rating', stripHTML(ratingdt.find('span')))
|
||||
|
||||
# meta=table.find_all('a', href=re.compile(r'/ratings/'))
|
||||
# meta=table.findAll('a', href=re.compile(r'/ratings/'))
|
||||
# i=0
|
||||
# for m in meta:
|
||||
# if i == 0:
|
||||
|
|
@ -210,17 +199,12 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
# elif i == 2:
|
||||
# self.story.addToList('warnings', m.find('b').text)
|
||||
|
||||
if dlinfo.find('div', {'class':'badge-status-finished'}):
|
||||
if dlinfo.find('span', {'class':'badge-status-finished'}):
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
try:
|
||||
self.story.setMetadata('universe', stripHTML(dlinfo.find('a', href=re.compile('/fandom_universe/'))))
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
paircharsdt = soup.find('strong',string='Пэйринг и персонажи:')
|
||||
paircharsdt = soup.find('strong',text='Пэйринг и персонажи:')
|
||||
# site keeps both ships and indiv chars in /pairings/ links.
|
||||
if paircharsdt:
|
||||
for paira in paircharsdt.find_next('div').find_all('a', href=re.compile(r'/pairings/')):
|
||||
|
|
@ -233,98 +217,8 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
self.story.addToList('characters',stripHTML(paira))
|
||||
|
||||
summary=soup.find('div', itemprop='description')
|
||||
if summary:
|
||||
# Fix for the text not displaying properly
|
||||
summary['class'].append('part_text')
|
||||
self.setDescription(url,summary)
|
||||
#self.story.setMetadata('description', summary.text)
|
||||
|
||||
stats = soup.find('div', {'class':'hat-actions-container'})
|
||||
targetdata = stats.find_all('span', {'class' : 'main-info'})
|
||||
for data in targetdata:
|
||||
svg_class = data.find('svg')['class'][1] if data.find('svg') else None
|
||||
value = int(stripHTML(data)) if stripHTML(data).isdigit() else 0
|
||||
|
||||
if svg_class == 'ic_thumbs-up' and value > 0:
|
||||
self.story.setMetadata('likes', value)
|
||||
#logger.debug("likes: (%s)"%self.story.getMetadata('likes'))
|
||||
elif svg_class == 'ic_bubble-dark' and value > 0:
|
||||
self.story.setMetadata('reviews', value)
|
||||
#logger.debug("reviews: (%s)"%self.story.getMetadata('reviews'))
|
||||
elif svg_class == 'ic_bookmark' and value > 0:
|
||||
self.story.setMetadata('numCollections', value)
|
||||
logger.debug("numCollections: (%s)"%self.story.getMetadata('numCollections'))
|
||||
|
||||
# Grab the amount of pages and words
|
||||
targetpages = soup.find('strong',string='Размер:').find_next('div')
|
||||
if targetpages:
|
||||
targetpages_text = re.sub(r"(?<!\,)\s| ", "", targetpages.text, flags=re.UNICODE | re.MULTILINE)
|
||||
|
||||
pages_raw = re.search(r'(\d+)(?:страницы|страниц)', targetpages_text, re.UNICODE)
|
||||
pages = int(pages_raw.group(1))
|
||||
if pages > 0:
|
||||
self.story.setMetadata('pages', pages)
|
||||
logger.debug("pages: (%s)"%self.story.getMetadata('pages'))
|
||||
|
||||
numWords_raw = re.search(r"(\d+)(?:слова|слов)", targetpages_text, re.UNICODE)
|
||||
numWords = int(numWords_raw.group(1))
|
||||
if numWords > 0:
|
||||
self.story.setMetadata('numWords', numWords)
|
||||
logger.debug("numWords: (%s)"%self.story.getMetadata('numWords'))
|
||||
|
||||
# Grab FBN Category
|
||||
class_tag = soup.select_one('div[class^="badge-with-icon direction"]').find('span', {'class' : 'badge-text'}).text
|
||||
if class_tag:
|
||||
self.story.setMetadata('classification',class_tag)
|
||||
#logger.debug("classification: (%s)"%self.story.getMetadata('classification'))
|
||||
|
||||
# Find dedication.
|
||||
ded = soup.find('div', {'class' : 'js-public-beta-dedication'})
|
||||
if ded:
|
||||
ded['class'].append('part_text')
|
||||
self.story.setMetadata('dedication',ded)
|
||||
|
||||
# Find author comment
|
||||
comm = soup.find('div', {'class' : 'js-public-beta-author-comment'})
|
||||
if comm:
|
||||
comm['class'].append('part_text')
|
||||
self.story.setMetadata('authorcomment',comm)
|
||||
|
||||
follows = stats.find('fanfic-follow-button')[':follow-count']
|
||||
if int(follows) > 0:
|
||||
self.story.setMetadata('follows', int(follows))
|
||||
logger.debug("follows: (%s)"%self.story.getMetadata('follows'))
|
||||
|
||||
# Grab the amount of awards
|
||||
numAwards = 0
|
||||
try:
|
||||
awards = soup.find('fanfic-reward-list')[':initial-fic-rewards-list']
|
||||
award_list = json.loads(awards)
|
||||
numAwards = int(len(award_list))
|
||||
# Grab the awards, but if multiple awards have the same name, only one will be kept; only an issue with hundreds of them.
|
||||
self.story.extendList('awards', [str(award['user_text']) for award in award_list])
|
||||
#logger.debug("awards (%s)"%self.story.getMetadata('awards'))
|
||||
except (TypeError, KeyError):
|
||||
logger.debug("Could not grab the awards")
|
||||
|
||||
if numAwards > 0:
|
||||
self.story.setMetadata('numAwards', numAwards)
|
||||
logger.debug("Num Awards (%s)"%self.story.getMetadata('numAwards'))
|
||||
|
||||
if get_cover:
|
||||
cover = soup.find('fanfic-cover', {'class':"jsVueComponent"})
|
||||
if cover is not None:
|
||||
self.setCoverImage(url,cover['src-original'])
|
||||
|
||||
def replace_formatting(self,tag):
|
||||
tname = tag.name
|
||||
## operating on plain text because BS4 is hard to work on
|
||||
## text with.
|
||||
## stripHTML() discards whitespace around other tags, like <i>
|
||||
txt = tag.get_text()
|
||||
txt = txt.replace("\n","<br/>")
|
||||
soup = self.make_soup("<"+tname+">"+txt+"</"+tname+">")
|
||||
return soup.find(tname)
|
||||
self.setDescription(url,summary)
|
||||
#self.story.setMetadata('description', summary.text)
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
|
@ -334,60 +228,10 @@ class FicBookNetAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
chapter = soup.find('div', {'id' : 'content'})
|
||||
if chapter is None: ## still needed?
|
||||
if chapter == None: ## still needed?
|
||||
chapter = soup.find('div', {'class' : 'public_beta_disabled'})
|
||||
|
||||
if chapter is None:
|
||||
if None == chapter:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
## ficbook uses weird CSS white-space: pre-wrap; for
|
||||
## paragraphing. Doesn't work with txt output
|
||||
if 'part_text' in chapter['class'] and self.getConfig('replace_text_formatting'):
|
||||
## copy classes, except part_text
|
||||
divclasses = chapter['class']
|
||||
divclasses.remove('part_text')
|
||||
chapter = self.replace_formatting(chapter)
|
||||
chapter['class'] = divclasses
|
||||
|
||||
exclude_notes=self.getConfigList('exclude_notes')
|
||||
if 'headnotes' not in exclude_notes:
|
||||
# Find the headnote
|
||||
head_note = soup.select_one("div.part-comment-top div.js-public-beta-comment-before")
|
||||
if head_note:
|
||||
# Create the structure for the headnote
|
||||
head_notes_div_tag = soup.new_tag('div', attrs={'class': 'fff_chapter_notes fff_head_notes'})
|
||||
head_b_tag = soup.new_tag('b')
|
||||
head_b_tag.string = 'Примечания:'
|
||||
if 'text-preline' in head_note['class'] and self.getConfig('replace_text_formatting'):
|
||||
head_blockquote_tag = self.replace_formatting(head_note)
|
||||
head_blockquote_tag.name = 'blockquote'
|
||||
else:
|
||||
head_blockquote_tag = soup.new_tag('blockquote')
|
||||
head_blockquote_tag.string = stripHTML(head_note)
|
||||
head_notes_div_tag.append(head_b_tag)
|
||||
head_notes_div_tag.append(head_blockquote_tag)
|
||||
# Prepend the headnotes to the chapter, <hr> to mimic the site
|
||||
chapter.insert(0, head_notes_div_tag)
|
||||
chapter.insert(1, soup.new_tag('hr'))
|
||||
|
||||
if 'footnotes' not in exclude_notes:
|
||||
# Find the endnote
|
||||
end_note = soup.select_one("div.part-comment-bottom div.js-public-beta-comment-after")
|
||||
if end_note:
|
||||
# Create the structure for the footnote
|
||||
end_notes_div_tag = soup.new_tag('div', attrs={'class': 'fff_chapter_notes fff_foot_notes'})
|
||||
end_b_tag = soup.new_tag('b')
|
||||
end_b_tag.string = 'Примечания:'
|
||||
if 'text-preline' in end_note['class'] and self.getConfig('replace_text_formatting'):
|
||||
end_blockquote_tag = self.replace_formatting(end_note)
|
||||
end_blockquote_tag.name = 'blockquote'
|
||||
else:
|
||||
end_blockquote_tag = soup.new_tag('blockquote')
|
||||
end_blockquote_tag.string = stripHTML(end_note)
|
||||
end_notes_div_tag.append(end_b_tag)
|
||||
end_notes_div_tag.append(end_blockquote_tag)
|
||||
# Append the endnotes to the chapter, <hr> to mimic the site
|
||||
chapter.append(soup.new_tag('hr'))
|
||||
chapter.append(end_notes_div_tag)
|
||||
|
||||
return self.utf8FromSoup(url,chapter)
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ class FictionAlleyArchiveOrgSiteAdapter(BaseSiteAdapter):
|
|||
elif key == 'Words':
|
||||
self.story.setMetadata('numWords',val)
|
||||
|
||||
summary = soup.find('dt',string='Story Summary:')
|
||||
summary = soup.find('dt',text='Story Summary:')
|
||||
if summary:
|
||||
summary = summary.find_next_sibling('dd')
|
||||
summary.name='div'
|
||||
|
|
@ -201,16 +201,16 @@ class FictionAlleyArchiveOrgSiteAdapter(BaseSiteAdapter):
|
|||
# epubutils.py
|
||||
# Yes, this still applies to fictionalley-archive.
|
||||
|
||||
for tag in chaptext.find_all('head') + chaptext.find_all('meta') + chaptext.find_all('script'):
|
||||
for tag in chaptext.findAll('head') + chaptext.findAll('meta') + chaptext.findAll('script'):
|
||||
tag.extract()
|
||||
|
||||
for tag in chaptext.find_all('body') + chaptext.find_all('html'):
|
||||
for tag in chaptext.findAll('body') + chaptext.findAll('html'):
|
||||
tag.name = 'div'
|
||||
|
||||
if self.getConfig('include_author_notes'):
|
||||
row = chaptext.find_previous_sibling('div',class_='row')
|
||||
logger.debug(row)
|
||||
andt = row.find('dt',string="Author's Note:")
|
||||
andt = row.find('dt',text="Author's Note:")
|
||||
logger.debug(andt)
|
||||
if andt:
|
||||
chaptext.insert(0,andt.parent.extract())
|
||||
|
|
|
|||
|
|
@ -235,7 +235,7 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
|
|||
# logger.debug(data)
|
||||
self.story.setMetadata('title',stripHTML(soup.find('h1',{'class':'Story__title'})))
|
||||
|
||||
summhead = soup.find('h5',string='Summary')
|
||||
summhead = soup.find('h5',text='Summary')
|
||||
self.setDescription(url,summhead.find_next('div'))
|
||||
|
||||
## author:
|
||||
|
|
@ -244,12 +244,12 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('authorUrl',autha['href'])
|
||||
self.story.setMetadata('author',autha.string)
|
||||
|
||||
updlab = soup.find('label',string='Last Updated:')
|
||||
updlab = soup.find('label',text='Last Updated:')
|
||||
if updlab:
|
||||
update = updlab.find_next('time')['datetime']
|
||||
self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
|
||||
|
||||
publab = soup.find('label',string='Published:')
|
||||
publab = soup.find('label',text='Published:')
|
||||
if publab:
|
||||
pubdate = publab.find_next('time')['datetime']
|
||||
self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
|
||||
|
|
@ -280,7 +280,7 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
|
|||
# logger.debug(meta)
|
||||
|
||||
# Find original ffnet URL
|
||||
a = soup.find('a', string="Source")
|
||||
a = soup.find('a', text="Source")
|
||||
self.story.setMetadata('origin',stripHTML(a))
|
||||
self.story.setMetadata('originUrl',a['href'])
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@
|
|||
# per-user achivement tracking with fancy achievement-get animations
|
||||
# story scripting (shows script tags visible in the text, not computed values or input fields)
|
||||
|
||||
import re
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
|
|
@ -36,8 +35,6 @@ import itertools
|
|||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# __package__ = 'fanficfare.adapters' # fixes dev issues with unknown package base
|
||||
|
||||
from .base_adapter import BaseSiteAdapter
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
|
@ -55,8 +52,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
self.story_id = self.parsedUrl.path.split('/')[3]
|
||||
self.story.setMetadata('storyId', self.story_id)
|
||||
|
||||
self.chapter_id_to_api = {}
|
||||
|
||||
# normalize URL. omits title in the url
|
||||
self._setURL("https://fiction.live/stories//{s_id}".format(s_id = self.story_id));
|
||||
|
||||
|
|
@ -70,7 +65,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
def getSiteURLPattern(self):
|
||||
# I'd like to thank regex101.com for helping me screw this up less
|
||||
return r"https?://(beta\.)?fiction\.live/[^/]*/[^/]*/([a-zA-Z0-9\-]+)(/(home)?)?$"
|
||||
return r"https?://(beta\.)?fiction\.live/[^/]*/[^/]*/([a-zA-Z0-9\-]+)(/(home)?)?"
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
|
|
@ -79,29 +74,11 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
+"https://fiction.live/Sci-fi/Example-Story-With-URL-Genre/17CharacterIDhere/ "
|
||||
+"https://fiction.live/stories/Example-Story-With-UUID/00000000-0000-4000-0000-000000000000/")
|
||||
|
||||
@classmethod
|
||||
def get_section_url(cls,url):
|
||||
## minimal URL used for section names in INI and reject list
|
||||
## for comparison
|
||||
# logger.debug("pre--url:%s"%url)
|
||||
url = re.sub(r"https?://(beta\.)?fiction\.live/[^/]*/[^/]*/(?P<id>[a-zA-Z0-9\-]+)(/(home)?)?$",r'https://fiction.live/stories//\g<id>',url)
|
||||
# logger.debug("post-url:%s"%url)
|
||||
return url
|
||||
|
||||
def parse_timestamp(self, timestamp):
|
||||
# fiction.live date format is unix-epoch milliseconds. not a good fit for fanficfare's makeDate.
|
||||
# doesn't use a timezone object and returns tz-naive datetimes. I *think* I can leave the rest to fanficfare
|
||||
return datetime.fromtimestamp(timestamp / 1000.0, None)
|
||||
|
||||
def img_url_trans(self,imgurl):
|
||||
"Apparently site changed cdn URLs for images more than once."
|
||||
# logger.debug("pre--imgurl:%s"%imgurl)
|
||||
imgurl = re.sub(r'(\w+)\.cloudfront\.net',r'cdn6.fiction.live/file/fictionlive',imgurl)
|
||||
imgurl = re.sub(r'www\.filepicker\.io/api/file/(\w+)',r'cdn4.fiction.live/fp/\1',imgurl)
|
||||
imgurl = re.sub(r'cdn[34].fiction.live/(.+)',r'cdn6.fiction.live/file/fictionlive/\1',imgurl)
|
||||
# logger.debug("post-imgurl:%s"%imgurl)
|
||||
return imgurl
|
||||
|
||||
def doExtractChapterUrlsAndMetadata(self, get_cover=True):
|
||||
|
||||
metadata_url = "https://fiction.live/api/node/{s_id}/"
|
||||
|
|
@ -173,7 +150,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
tags = data['ta'] if 'ta' in data else []
|
||||
|
||||
if (self.story.getMetadataRaw('rating') in {"nsfw", "adult"} or 'smut' in tags) and \
|
||||
if (self.story.getMetadata('rating') in {"nsfw", "adult"} or 'smut' in tags) and \
|
||||
not (self.is_adult or self.getConfig("is_adult")):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
|
|
@ -209,6 +186,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
if show_nsfw_cover_images or not nsfw_cover:
|
||||
coverUrl = data['i'][0]
|
||||
self.setCoverImage(self.url, coverUrl)
|
||||
self.story.setMetadata('cover_image', "<a href=\"" + coverUrl + "\" />") # TODO: is this needed?
|
||||
|
||||
# gonna need these later for adding details to achievement-granting links in the text
|
||||
try:
|
||||
|
|
@ -241,17 +219,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
a, b = itertools.tee(iterable, 2)
|
||||
next(b, None)
|
||||
return list(zip(a, b))
|
||||
|
||||
def map_chap_ids_to_api(chapter_ids, route_ids, times):
|
||||
for index, bounds in enumerate(times):
|
||||
start, end = bounds
|
||||
end -= 1
|
||||
chapter_url = chunkrange_url.format(s_id = data['_id'], start = start, end = end)
|
||||
self.chapter_id_to_api[chapter_ids[index]] = chapter_url
|
||||
|
||||
for route_id in route_ids:
|
||||
chapter_url = route_chunkrange_url.format(c_id = route_id)
|
||||
self.chapter_id_to_api[route_id] = chapter_url
|
||||
|
||||
## first thing to do is seperate out the appendices
|
||||
appendices, maintext, routes = [], [], []
|
||||
|
|
@ -273,25 +240,22 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
## main-text chapter extraction processing. *should* now handle all the edge cases.
|
||||
## relies on fanficfare ignoring empty chapters!
|
||||
|
||||
titles = ["Home"] + [c['title'] for c in maintext]
|
||||
chapter_ids = ['home'] + [c['id'] for c in maintext]
|
||||
times = [data['ct']] + [c['ct'] for c in maintext] + [self.most_recent_chunk + 2] # need to be 1 over, and add_url etc does -1
|
||||
times = pair(times)
|
||||
titles = [c['title'] for c in maintext]
|
||||
titles = ["Home"] + titles
|
||||
|
||||
if self.getConfig('include_appendices', True): # Add appendices after main text if desired
|
||||
titles = titles + ["Appendix: " + a['title'][9:] for a in appendices]
|
||||
chapter_ids = chapter_ids + [a['id'] for a in appendices]
|
||||
times = times + [(a['ct'], a['ct'] + 2) for a in appendices]
|
||||
|
||||
route_ids = [r['id'] for r in routes]
|
||||
|
||||
map_chap_ids_to_api(chapter_ids, route_ids, times) # Map chapter ids to API URLs for use when comparing the two
|
||||
times = [c['ct'] for c in maintext]
|
||||
times = [data['ct']] + times + [self.most_recent_chunk + 2] # need to be 1 over, and add_url etc does -1
|
||||
|
||||
# doesn't actually run without the call to list.
|
||||
list(map(add_chapter_url, titles, times))
|
||||
list(map(add_chapter_url, titles, pair(times)))
|
||||
|
||||
for a in appendices: # add appendices afterwards
|
||||
chapter_start = a['ct']
|
||||
chapter_title = "Appendix: " + a['title'][9:] # 'Appendix: ' rather than '#special' at beginning of name
|
||||
add_chapter_url(chapter_title, (chapter_start, chapter_start + 2)) # 1 msec range = this one chunk only
|
||||
|
||||
for r in routes: # add route at the end, after appendices
|
||||
route_id = r['id'] # to get route chapter content, the route id is needed, not the timestamp
|
||||
route_id = r['id'] # to get route chapter content, the route id is needed, not the timestamp
|
||||
chapter_title = "Route: " + r['title'] # 'Route: ' at beginning of name, since it's a multiroute chapter
|
||||
add_route_chapter_url(chapter_title, route_id)
|
||||
|
||||
|
|
@ -321,7 +285,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
text += "<div>" # chapter chunks aren't always well-delimited in their contents
|
||||
|
||||
# appendix chunks are mixed in with other things
|
||||
# appendix chunks are mixed in with other things
|
||||
if not getting_appendix and 't' in chunk and chunk['t'].startswith("#special"): # t = title = bookmark
|
||||
continue
|
||||
|
||||
|
|
@ -336,8 +300,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
text += "</div><br />\n"
|
||||
|
||||
## soup to repair the most egregious HTML errors.
|
||||
return self.utf8FromSoup(url,self.make_soup(text))
|
||||
return text
|
||||
|
||||
### everything from here out is chunk data handling.
|
||||
|
||||
|
|
@ -354,7 +317,8 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
if self.achievements:
|
||||
soup = self.append_achievments(soup)
|
||||
|
||||
return str(soup)
|
||||
# utf8FromSoup does important processing e.g. sanitization and imageurl extraction
|
||||
return self.utf8FromSoup(self.url, soup)
|
||||
|
||||
def add_spoiler_legends(self, soup):
|
||||
# find spoiler links and change link-anchor block to legend block
|
||||
|
|
@ -434,7 +398,7 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
# so let's just ignore non-int values here
|
||||
if not isinstance(v, int):
|
||||
continue
|
||||
if 0 <= v < len(choices):
|
||||
if 0 <= v <= len(choices):
|
||||
output[v] += 1
|
||||
return output
|
||||
|
||||
|
|
@ -518,10 +482,8 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
# now matches the site and does *not* include dicerolls as posts!
|
||||
num_votes = str(len(posts)) + " posts" if len(posts) != 0 else "be the first to post."
|
||||
|
||||
posts_title = chunk['b'] if 'b' in chunk else "Reader Posts"
|
||||
|
||||
output = ""
|
||||
output += u"<h4><span>" + posts_title + " — <small> Posting " + closed
|
||||
output += u"<h4><span>Reader Posts — <small> Posting " + closed
|
||||
output += u" — " + num_votes + "</small></span></h4>\n"
|
||||
|
||||
## so. a voter can roll with their post. these rolls are in a seperate dict, but have the **same uid**.
|
||||
|
|
@ -547,35 +509,6 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
return output
|
||||
|
||||
def normalize_chapterurl(self, url):
|
||||
if url.startswith(r'https://fiction.live/api/anonkun/chapters'):
|
||||
return url
|
||||
|
||||
pattern = None
|
||||
|
||||
if url.startswith(r'https://fiction.live/api/anonkun/route'):
|
||||
pattern = r"https?://(?:beta\.)?fiction\.live/[^/]*/[^/]*/[a-zA-Z0-9]+/routes/([a-zA-Z0-9]+)"
|
||||
elif url.startswith(r'https://fiction.live/'):
|
||||
pattern = r"https?://(?:beta\.)?fiction\.live/[^/]*/[^/]*/[a-zA-Z0-9]+/[^/]*(/[a-zA-Z0-9]+|home)"
|
||||
# regex101 rocks
|
||||
|
||||
if not pattern:
|
||||
return url
|
||||
|
||||
match = re.match(pattern, url)
|
||||
if not match:
|
||||
return url
|
||||
|
||||
chapter_id = match.group(1)
|
||||
|
||||
if chapter_id.startswith('/'):
|
||||
chapter_id = chapter_id[1:]
|
||||
|
||||
if chapter_id and chapter_id in self.chapter_id_to_api:
|
||||
return self.chapter_id_to_api[chapter_id]
|
||||
|
||||
return url
|
||||
|
||||
def format_unknown(self, chunk):
|
||||
raise NotImplementedError("Unknown chunk type ({}) in fiction.live story.".format(chunk))
|
||||
|
||||
|
|
@ -590,5 +523,5 @@ class FictionLiveAdapter(BaseSiteAdapter):
|
|||
|
||||
# TODO: support chapter urls for single-chapter / chapter-range downloads
|
||||
# complicated -- urls for getChapterText are API urls generated by add_chapters, not the public/website ones
|
||||
# in particular, may need more API reversing to figure out how to get the *end* of the chunk range
|
||||
# in particular, may need more API reversing to figure out how to get the *end* of the chunk range
|
||||
# find in 'bm' in the metadata?
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
SITE_ABBREVIATION = 'fmt'
|
||||
SITE_DOMAIN = 'fictionmania.tv'
|
||||
|
||||
BASE_URL = 'https://' + SITE_DOMAIN + '/stories/'
|
||||
BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
|
||||
READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
|
||||
DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
|
||||
|
||||
|
|
@ -40,6 +40,10 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
|
||||
self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
|
||||
|
||||
# Always single chapters, probably should use the Anthology feature to
|
||||
# merge chapters of a story
|
||||
self.story.setMetadata('numChapters', 1)
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return FictionManiaTVAdapter.SITE_DOMAIN
|
||||
|
|
@ -49,7 +53,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r'https?' + re.escape(self.BASE_URL[len('https'):]) + r'(readtextstory|readhtmlstory|readxstory|details)\.html\?storyID=\d+$'
|
||||
return r'https?' + re.escape(self.BASE_URL[len('http'):]) + r'(readtextstory|readhtmlstory|readxstory|details)\.html\?storyID=\d+$'
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
|
||||
|
|
@ -106,7 +110,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', value)
|
||||
|
||||
elif key == 'Complete':
|
||||
self.story.setMetadata('status', 'Completed' if value == 'yes' else 'In-Progress')
|
||||
self.story.setMetadata('status', 'Completed' if value == 'Complete' else 'In-Progress')
|
||||
|
||||
elif key == 'Categories':
|
||||
for element in cells[1]('a'):
|
||||
|
|
@ -163,30 +167,14 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
|
|||
# <div style="margin-left:10ex;margin-right:10ex">
|
||||
## fetching SWI version now instead of text.
|
||||
htmlurl = url.replace('readtextstory','readhtmlstory')
|
||||
## Used to find by style, but it's inconsistent now. we've seen:
|
||||
## margin-left:10ex;margin-right:10ex
|
||||
## margin-right: 5%; margin-left: 5%
|
||||
## margin-left:5%; margin-right:5%
|
||||
## margin-left:5%; margin-right:5%; background: white
|
||||
## And there's some without a <div> tag (or an unclosed div)
|
||||
## Only the comments appear to be consistent.
|
||||
beginmarker='<!--Read or display the file-->'
|
||||
endmarker='''<hr size=1 noshade>
|
||||
<!--review add read, top and bottom-->
|
||||
'''
|
||||
data = self.get_request(htmlurl)
|
||||
try:
|
||||
## if both markers are found, assume whatever is in between
|
||||
## is the chapter text.
|
||||
soup = self.make_soup(data[data.index(beginmarker):data.index(endmarker)])
|
||||
return self.utf8FromSoup(htmlurl,soup)
|
||||
except Exception as e:
|
||||
# logger.debug(e)
|
||||
# logger.debug(soup)
|
||||
soup = self.make_soup(self.get_request(htmlurl))
|
||||
div = soup.find('div',style="margin-left:10ex;margin-right:10ex")
|
||||
if div:
|
||||
return self.utf8FromSoup(htmlurl,div)
|
||||
else:
|
||||
logger.debug("Story With Images(SWI) not found, falling back to HTML.")
|
||||
|
||||
## fetching html version now instead of text.
|
||||
## Note that html and SWI pages are *not* formatted the same.
|
||||
soup = self.make_soup(self.get_request(url.replace('readtextstory','readxstory')))
|
||||
# logger.debug(soup)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
|
|
@ -47,12 +46,6 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
|
|||
def _get_site_url_pattern(cls):
|
||||
return r"https?://(www|m)?\.fictionpress\.com/s/(?P<id>\d+)(/\d+)?(/(?P<title>[^/]+))?/?$"
|
||||
|
||||
## normalized chapter URLs DO contain the story title now, but
|
||||
## normalized to current urltitle in case of title changes.
|
||||
def normalize_chapterurl(self,url):
|
||||
return re.sub(r"https?://(www|m)\.(?P<keep>fictionpress\.com/s/\d+/\d+/).*",
|
||||
r"https://www.\g<keep>",url)+self.urltitle
|
||||
|
||||
def getClass():
|
||||
return FictionPressComSiteAdapter
|
||||
|
||||
|
|
|
|||
|
|
@ -66,8 +66,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
params['username']))
|
||||
d = self.post_request(loginUrl,params,usecache=False)
|
||||
|
||||
if "Login attempt failed..." in d or \
|
||||
'<div id="error">Please enter your username and password.</div>' in d:
|
||||
if "Login attempt failed..." in d:
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['username']))
|
||||
raise exceptions.FailedToLogin(url,params['username'])
|
||||
|
|
@ -115,7 +114,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
|
||||
self.story.setMetadata('title', stripHTML(titleh4.a))
|
||||
|
||||
if 'Deleted story' in self.story.getMetadataRaw('title'):
|
||||
if 'Deleted story' in self.story.getMetadata('title'):
|
||||
raise exceptions.StoryDoesNotExist("This story was deleted. %s"%self.url)
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
|
|
@ -130,14 +129,14 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
#self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
|
||||
|
||||
# most of the meta data is here:
|
||||
metap = storydiv.find("div",{"class":"meta"})
|
||||
metap = storydiv.find("p",{"class":"meta"})
|
||||
self.story.addToList('category',metap.find("a",href=re.compile(r"^/category/\d+")).string)
|
||||
|
||||
# warnings
|
||||
# <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
|
||||
spanreq = metap.find("span",{"class":"story-warnings"})
|
||||
if spanreq: # can be no warnings.
|
||||
for a in spanreq.find_all("a"):
|
||||
for a in spanreq.findAll("a"):
|
||||
self.story.addToList('warnings',a['title'])
|
||||
|
||||
## perhaps not the most efficient way to parse this, using
|
||||
|
|
@ -187,7 +186,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
|
|||
# no list found, so it's a one-chapter story.
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
chapterlistlis = storylistul.find_all('li')
|
||||
chapterlistlis = storylistul.findAll('li')
|
||||
for chapterli in chapterlistlis:
|
||||
if "blocked" in chapterli['class']:
|
||||
# paranoia check. We should already be logged in by now.
|
||||
|
|
|
|||
|
|
@ -99,17 +99,6 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
params['username']))
|
||||
raise exceptions.FailedToLogin(url,params['username'])
|
||||
|
||||
def make_soup(self,data):
|
||||
soup = super(FimFictionNetSiteAdapter, self).make_soup(data)
|
||||
for img in soup.select('img.lazy-img, img.user_image'):
|
||||
## FimF has started a 'camo' mechanism for images that
|
||||
## gets block by CF. attr data-source is original source.
|
||||
if img.has_attr('data-source'):
|
||||
img['src'] = img['data-source']
|
||||
elif img.has_attr('data-src'):
|
||||
img['src'] = img['data-src']
|
||||
return soup
|
||||
|
||||
def doExtractChapterUrlsAndMetadata(self,get_cover=True):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
|
|
@ -117,8 +106,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
## Only needed with password protected stories, which you have
|
||||
## to have logged into in the website using this account.
|
||||
if self.getConfig("always_login"):
|
||||
self.performLogin(self.url)
|
||||
self.performLogin(self.url)
|
||||
|
||||
##---------------------------------------------------------------------------------------------------
|
||||
## Get the story's title page. Check if it exists.
|
||||
|
|
@ -151,8 +139,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata("authorId", author['href'].split('/')[2])
|
||||
self.story.setMetadata("authorUrl", "https://%s/user/%s/%s" % (self.getSiteDomain(),
|
||||
self.story.getMetadata('authorId'),
|
||||
# meta entry author can be changed by the user.
|
||||
stripHTML(author)))
|
||||
self.story.getMetadata('author')))
|
||||
|
||||
#Rating text is replaced with full words for historical compatibility after the site changed
|
||||
#on 2014-10-27
|
||||
|
|
@ -180,13 +167,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# Cover image
|
||||
if get_cover:
|
||||
storyImage = soup.select_one('div.story_container__story_image img')
|
||||
storyImage = storyContentBox.find('img', {'class':'lazy-img'})
|
||||
if storyImage:
|
||||
coverurl = storyImage['data-fullsize']
|
||||
# try setting from data-fullsize, if fails, try using data-src
|
||||
cover_set = self.setCoverImage(self.url,coverurl)[0]
|
||||
if not cover_set or cover_set.startswith("failedtoload"):
|
||||
coverurl = storyImage['src']
|
||||
if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
|
||||
coverurl = storyImage['data-src']
|
||||
self.setCoverImage(self.url,coverurl)
|
||||
|
||||
coverSource = storyImage.parent.find('a', {'class':'source'})
|
||||
|
|
@ -298,26 +284,16 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
descriptionMeta = soup.find('meta', {'property':'og:description'})
|
||||
self.story.setMetadata("short_description", stripHTML(descriptionMeta['content']))
|
||||
|
||||
# groups.
|
||||
# If there are more than X groups, there's a 'Show all' button
|
||||
# that calls for a JSON containing HTML with the full list.
|
||||
# But it doesn't work reliably with FlareSolverr.
|
||||
groupList = None
|
||||
#groups
|
||||
groupButton = soup.find('button', {'data-click':'showAll'})
|
||||
if groupButton != None and groupButton.find('i', {'class':'fa-search-plus'}):
|
||||
try:
|
||||
groupResponse = self.get_request("https://www.fimfiction.net/ajax/stories/%s/groups" % (self.story.getMetadata("storyId")))
|
||||
groupData = json.loads(groupResponse)
|
||||
groupList = self.make_soup(groupData["content"])
|
||||
except Exception as e:
|
||||
logger.warning("Collecting 'groups' (AKA 'Featured In') from JSON failed:%s"%e)
|
||||
logger.warning("Only 'groups' initially shown on the page will be collected.")
|
||||
logger.warning("This is a known issue with JSON and FlareSolverr. See #1122")
|
||||
|
||||
if not groupList:
|
||||
groupResponse = self.get_request("https://www.fimfiction.net/ajax/stories/%s/groups" % (self.story.getMetadata("storyId")))
|
||||
groupData = json.loads(groupResponse)
|
||||
groupList = self.make_soup(groupData["content"])
|
||||
else:
|
||||
groupList = soup.find('ul', {'id':'story-groups-list'})
|
||||
|
||||
if groupList:
|
||||
if not (groupList == None):
|
||||
for groupContent in groupList.find_all('a'):
|
||||
self.story.addToList("groupsUrl", 'https://'+self.host+groupContent["href"])
|
||||
groupName = groupContent.find('span', {"class":"group-name"})
|
||||
|
|
@ -328,7 +304,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
#sequels
|
||||
for header in soup.find_all('h1', {'class':'header-stories'}):
|
||||
# I don't know why using string=re.compile with find() wouldn't work, but it didn't.
|
||||
# I don't know why using text=re.compile with find() wouldn't work, but it didn't.
|
||||
if header.text.startswith('Sequels'):
|
||||
sequelContainer = header.parent
|
||||
for sequel in sequelContainer.find_all('a', {'class':'story_link'}):
|
||||
|
|
@ -408,33 +384,3 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
|
|||
# data = self.get_request(url)
|
||||
if self.getConfig("is_adult"):
|
||||
self.set_adult_cookie()
|
||||
|
||||
def get_urls_from_page(self,url,normalize):
|
||||
iterate = self.getConfig('scrape_bookshelf', default=False)
|
||||
if not re.search(r'fimfiction\.net/bookshelf/(?P<listid>.+?)/',url) or iterate == 'legacy':
|
||||
return super().get_urls_from_page(url,normalize)
|
||||
|
||||
self.before_get_urls_from_page(url,normalize)
|
||||
|
||||
final_urls = list()
|
||||
while True:
|
||||
data = self.get_request(url,usecache=True)
|
||||
soup = self.make_soup(data)
|
||||
paginator = soup.select_one('div.paginator-container > div.page_list > ul').find_all('li')
|
||||
logger.debug("Paginator: " + str(len(paginator)))
|
||||
stories_container = soup.select_one('div.content > div.two-columns > div.left').find_all('article', recursive=False)
|
||||
x = 0
|
||||
logger.debug("Container "+str(len(stories_container)))
|
||||
for story_raw in stories_container:
|
||||
x += 1
|
||||
story_url = story_raw.select_one('div.story_content_box > header.title > div > a.story_name').get('href')
|
||||
url_story = ('https://' + self.getSiteDomain() + story_url)
|
||||
#logger.debug(url_story)
|
||||
final_urls.append(url_story)
|
||||
logger.debug("Discovered %s new stories."%str(x))
|
||||
|
||||
next_button = paginator[-1].select_one('a')
|
||||
logger.debug("Next button: " + next_button.get_text())
|
||||
if next_button.get_text() or not iterate:
|
||||
return {'urllist': final_urls}
|
||||
url = ('https://' + self.getSiteDomain() + next_button.get('href'))
|
||||
|
|
|
|||
|
|
@ -24,30 +24,43 @@ logger = logging.getLogger(__name__)
|
|||
from .adapter_storiesonlinenet import StoriesOnlineNetAdapter
|
||||
|
||||
def getClass():
|
||||
return StoryRoomComAdapter
|
||||
return FineStoriesComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class StoryRoomComAdapter(StoriesOnlineNetAdapter):
|
||||
class FineStoriesComAdapter(StoriesOnlineNetAdapter):
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(cls):
|
||||
return 'stryrm'
|
||||
return 'fnst'
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'storyroom.com'
|
||||
return 'finestories.com'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['finestories.com',cls.getSiteDomain()]
|
||||
def getTheme(cls):
|
||||
## only one theme is supported.
|
||||
return "Modern"
|
||||
|
||||
@classmethod
|
||||
def getConfigSections(cls):
|
||||
"Only needs to be overriden if has additional ini sections."
|
||||
return ['finestories.com',cls.getSiteDomain()]
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Free Registration' in data \
|
||||
or "Log In" in data \
|
||||
or "Invalid Password!" in data \
|
||||
or "Invalid User Name!" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://("+r"|".join([x.replace('.',r'\.') for x in self.getAcceptDomains()])+r")/(?P<path>s|n|library)/(storyInfo.php\?id=)?(?P<id>\d+)(?P<chapter>:\d+)?(?P<title>/.+)?((;\d+)?$|(:i)?$)?"
|
||||
def getStoryMetadataFromAuthorPage(self):
|
||||
# surprisingly, the detailed page does not give enough details, so go to author's page
|
||||
story_row = self.findStoryRow('div')
|
||||
|
||||
description_element = story_row.find('div', {'class' : 'sdesc'})
|
||||
|
||||
self.parseDescriptionField(description_element)
|
||||
|
||||
misc_element = story_row.find('div', {'class' : 'misc'})
|
||||
self.parseOtherAttributes(misc_element)
|
||||
|
|
@ -93,9 +93,6 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
a = soup.find('a', href=re.compile(r"profileshow.aspx\?u="))
|
||||
self.story.setMetadata('authorId', a['href'].split('=')[1])
|
||||
if not self.story.getMetadata('authorId'):
|
||||
logger.warning("Site authorUrl missing authorId, using SiteMissingAuthorId")
|
||||
self.story.setMetadata('authorId', 'SiteMissingAuthorId')
|
||||
self.story.setMetadata('authorUrl', 'http://' +
|
||||
self.host + '/' + a['href'])
|
||||
self.story.setMetadata('author', a.string)
|
||||
|
|
@ -105,6 +102,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
|
|||
# to download them one at a time yourself. I'm also setting the status to
|
||||
# complete
|
||||
self.add_chapter(self.story.getMetadata('title'), self.url)
|
||||
self.story.setMetadata('numChapters', 1)
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
|
||||
## some stories do not have a summary listed, so I'm setting it here.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2024 FanFicFare team
|
||||
# Copyright 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -18,15 +18,15 @@
|
|||
from __future__ import absolute_import
|
||||
import re
|
||||
|
||||
from .base_xenforo2forum_adapter import BaseXenForo2ForumAdapter
|
||||
from .base_xenforoforum_adapter import BaseXenForoForumAdapter
|
||||
|
||||
def getClass():
|
||||
return QuestionablequestingComAdapter
|
||||
|
||||
class QuestionablequestingComAdapter(BaseXenForo2ForumAdapter):
|
||||
class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseXenForo2ForumAdapter.__init__(self, config, url)
|
||||
BaseXenForoForumAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','qq')
|
||||
|
|
|
|||
216
fanficfare/adapters/adapter_hlfictionnet.py
Normal file
216
fanficfare/adapters/adapter_hlfictionnet.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return HLFictionNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class HLFictionNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','hlf')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%m/%d/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'hlfiction.net'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title and author
|
||||
a = soup.find('div', {'id' : 'pagetitle'})
|
||||
|
||||
aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',aut['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+aut['href'])
|
||||
self.story.setMetadata('author',aut.string)
|
||||
aut.extract()
|
||||
|
||||
self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.find('select')
|
||||
if chapters != None:
|
||||
for chapter in chapters.findAll('option'):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])
|
||||
else:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
|
||||
|
||||
asoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
|
||||
|
||||
for list in asoup.findAll('div', {'class' : re.compile('listbox')}):
|
||||
a = list.find('a')
|
||||
if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
|
||||
break
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = list.findAll('span', {'class' : 'classification'})
|
||||
for labelspan in labels:
|
||||
label = labelspan.string
|
||||
value = labelspan.nextSibling
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while 'classification' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value[:len(value)-2])
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
for char in value.string.split(', '):
|
||||
if not 'None' in char:
|
||||
self.story.addToList('characters',char)
|
||||
|
||||
if 'Genre' in label:
|
||||
for genre in value.string.split(', '):
|
||||
if not 'None' in genre:
|
||||
self.story.addToList('genre',genre)
|
||||
|
||||
if 'Warnings' in label:
|
||||
for warning in value.string.split(', '):
|
||||
if not 'None' in warning:
|
||||
self.story.addToList('warnings',warning)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
262
fanficfare/adapters/adapter_iketernalnet.py
Normal file
262
fanficfare/adapters/adapter_iketernalnet.py
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return IkEternalNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class IkEternalNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ike')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%B %d, %Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.ik-eternal.net'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self.post_request(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&warning=1"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self.get_request(url)
|
||||
|
||||
# The actual text that is used to announce you need to be an
|
||||
# adult varies from site to site. Again, print data before
|
||||
# the title search to troubleshoot.
|
||||
|
||||
# Since the warning text can change by warning level, let's
|
||||
# look for the warning pass url. ksarchive uses
|
||||
# &warning= -- actually, so do other sites. Must be an
|
||||
# eFiction book.
|
||||
|
||||
# viewstory.php?sid=1882&warning=4
|
||||
# viewstory.php?sid=1654&ageconsent=ok&warning=5
|
||||
#print data
|
||||
#m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
asoup = soup.find('div', {'class': 'listbox'})
|
||||
for a in asoup.findAll('p'):
|
||||
a.name='br'
|
||||
labels = asoup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -161,7 +161,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -178,7 +178,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -199,22 +199,22 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -238,7 +238,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
|
|||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
28
fanficfare/adapters/adapter_inkbunnynet.py
Executable file → Normal file
28
fanficfare/adapters/adapter_inkbunnynet.py
Executable file → Normal file
|
|
@ -125,7 +125,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url,usecache=False))
|
||||
|
||||
# removing all of the scripts
|
||||
for tag in soup.find_all('script'):
|
||||
for tag in soup.findAll('script'):
|
||||
tag.extract()
|
||||
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('title', stripHTML(title))
|
||||
|
||||
# Get Author
|
||||
authortag = soup.find('table',{'class':'pooltable'}).find('a',href=re.compile(r'/gallery/|/scraps/'))
|
||||
authortag = soup.find('table',{'class':'pooltable'}).find('a',href=re.compile(r'/gallery/'))
|
||||
author = authortag['href'].split('/')[-1] # no separate ID
|
||||
self.story.setMetadata('author', author)
|
||||
self.story.setMetadata('authorId', author)
|
||||
|
|
@ -149,7 +149,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
if not self.getConfig('keep_summary_html'):
|
||||
synopsis = stripHTML(synopsis)
|
||||
|
||||
self.setDescription(url, synopsis)
|
||||
self.setDescription(url, stripHTML(synopsis))
|
||||
|
||||
#Getting Keywords/Genres
|
||||
keywords = bookdetails.find('div', {'id':'kw_scroll'}).find_next_siblings('div')[0].div.div.find_all('a')
|
||||
|
|
@ -157,11 +157,10 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
self.story.addToList('genre', stripHTML(kword))
|
||||
|
||||
# Getting the Category
|
||||
category = bookdetails.findChildren('div', recursive=False)[2].find('span', string='Type:').parent
|
||||
category.find('span').decompose()
|
||||
self.story.setMetadata('category', stripHTML(category))
|
||||
for div in bookdetails.find_all('div'):
|
||||
if 'Rating:' == stripHTML(div)[:7]:
|
||||
if 'Details' == stripHTML(div).strip():
|
||||
self.story.setMetadata('category', div.find_next_siblings('div')[0].span.next_sibling.strip())
|
||||
elif 'Rating:' == stripHTML(div).strip()[:7]:
|
||||
rating = div.span.next_sibling.strip()
|
||||
self.story.setMetadata('rating', rating)
|
||||
break
|
||||
|
|
@ -179,14 +178,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
if get_cover:
|
||||
cover_img = soup.find('img', {'id':'magicbox'})
|
||||
if cover_img:
|
||||
# image content is treated like a normal image submission
|
||||
self.setCoverImage(url, cover_img['src'])
|
||||
else:
|
||||
# image content is present, but secondary to text file
|
||||
cover_div = soup.find('div', {'class': 'content magicboxParent'})
|
||||
cover_img = cover_div.find('img', {'class':'shadowedimage'}) if cover_div else None
|
||||
if cover_img:
|
||||
self.setCoverImage(url, cover_img['src'])
|
||||
|
||||
## Save for use below
|
||||
self.soup = soup
|
||||
|
|
@ -200,11 +192,3 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
|
|||
raise exceptions.FailedToDownload("Error downloading Chapter: %s No text block found -- non-story URL?" % url)
|
||||
|
||||
return self.utf8FromSoup(url, story)
|
||||
|
||||
def before_get_urls_from_page(self,url,normalize):
|
||||
# To display the links to stories that are not available to guests.
|
||||
if self.getConfig("username") and self.getConfig("always_login"):
|
||||
# performLogin extracts token from the soup
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
self.performLogin(url, soup)
|
||||
|
|
|
|||
|
|
@ -1,213 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging, time
|
||||
logger = logging.getLogger(__name__)
|
||||
import re, json
|
||||
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six.moves import http_cookiejar as cl
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return KakuyomuJpAdapter
|
||||
|
||||
genres = {
|
||||
'FANTASY': '異世界ファンタジー',
|
||||
'ACTION': '現代ファンタジー',
|
||||
'SF': 'SF',
|
||||
'LOVE_STORY': '恋愛',
|
||||
'ROMANCE': 'ラブコメ',
|
||||
'DRAMA': '現代ドラマ',
|
||||
'HORROR': 'ホラー',
|
||||
'MYSTERY': 'ミステリー',
|
||||
'NONFICTION': 'エッセイ・ノンフィクション',
|
||||
'HISTORY': '歴史・時代・伝奇',
|
||||
'CRITICISM': '創作論・評論',
|
||||
'OTHERS': '詩・童話・その他',
|
||||
'FAN_FICTION': '二次創作',
|
||||
}
|
||||
|
||||
class KakuyomuJpAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.story.setMetadata('siteabbrev', 'kakuyomu')
|
||||
self.story.setMetadata('language', 'Japanese')
|
||||
|
||||
self.storyId = self.path.split('/')[-1]
|
||||
self.story.setMetadata('storyId', self.storyId)
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'kakuyomu.jp'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return ("https://kakuyomu.jp/works/12341234123412341234")
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"^https?://kakuyomu\.jp/works/[0-9]+$"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
data = self.get_request(self.url)
|
||||
|
||||
# Page could not be found
|
||||
if 'お探しのページは見つかりませんでした' in data:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
info = json.loads(soup.find(id='__NEXT_DATA__').contents[0])['props']['pageProps']['__APOLLO_STATE__']
|
||||
|
||||
workKey = 'Work:%s' % self.storyId
|
||||
|
||||
# Title
|
||||
self.story.setMetadata('title', info[workKey]['title'])
|
||||
|
||||
# Author
|
||||
authorKey = info[workKey]['author']['__ref']
|
||||
self.story.setMetadata('authorId', authorKey.split(':')[1])
|
||||
self.story.setMetadata('authorUrl', 'https://kakuyomu.jp/users/%s' % info[authorKey]['name'])
|
||||
self.story.setMetadata('author', info[authorKey]['activityName'])
|
||||
|
||||
# Description
|
||||
self.setDescription(self.url, info[workKey]['introduction'])
|
||||
self.story.setMetadata('catchphrase', info[workKey]['catchphrase'])
|
||||
|
||||
# Date Published and Updated
|
||||
# 2024-01-01T03:00:12Z
|
||||
self.story.setMetadata('datePublished',
|
||||
makeDate(info[workKey]['publishedAt'], '%Y-%m-%dT%H:%M:%SZ'))
|
||||
self.story.setMetadata('dateUpdated',
|
||||
makeDate(info[workKey]['editedAt'], '%Y-%m-%dT%H:%M:%SZ'))
|
||||
|
||||
# Character count
|
||||
self.story.setMetadata('numWords', info[workKey]['totalCharacterCount'])
|
||||
|
||||
# Status
|
||||
completed = info[workKey]['serialStatus'] == 'COMPLETED'
|
||||
self.story.setMetadata('status', 'Completed' if completed else 'In-Progress')
|
||||
|
||||
# Warnings
|
||||
rating = 'G'
|
||||
if info[workKey]['isCruel']:
|
||||
rating = 'R15'
|
||||
self.story.addToList('warnings', '残酷描写有り')
|
||||
if info[workKey]['isViolent']:
|
||||
rating = 'R15'
|
||||
self.story.addToList('warnings', '暴力描写有り')
|
||||
if info[workKey]['isSexual']:
|
||||
rating = 'R15'
|
||||
self.story.addToList('warnings', '性描写有り')
|
||||
|
||||
# Tags
|
||||
for tag in info[workKey]['tagLabels']:
|
||||
if re.match(r'[RrＲ].?[1１][5５]', tag) is None:
|
||||
self.story.addToList('freeformtags', tag)
|
||||
else:
|
||||
rating = 'R15'
|
||||
|
||||
# Rating
|
||||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Genre
|
||||
self.story.setMetadata('genre', genres[info[workKey]['genre']])
|
||||
|
||||
if info[workKey]['genre'] == 'FAN_FICTION':
|
||||
fandomKey = info[workKey]['fanFictionSource']['__ref']
|
||||
self.story.addToList('fandoms', info[fandomKey]['title'])
|
||||
|
||||
# Ratings, Comments, Etc.
|
||||
self.story.setMetadata('reviews', info[workKey]['reviewCount'])
|
||||
self.story.setMetadata('points', info[workKey]['totalReviewPoint'])
|
||||
self.story.setMetadata('comments', info[workKey]['totalPublicEpisodeCommentCount'])
|
||||
self.story.setMetadata('views', info[workKey]['totalReadCount'])
|
||||
self.story.setMetadata('follows', info[workKey]['totalFollowers'])
|
||||
self.story.setMetadata('collections', len(info[workKey]['publicWorkCollections']))
|
||||
self.story.setMetadata('events', info[workKey]['totalWorkContestCount'] + info[workKey]['totalUserEventCount'])
|
||||
self.story.setMetadata('published', info[workKey]['hasPublication'])
|
||||
|
||||
# visitorWorkFollowing
|
||||
# workReviewByVisitor
|
||||
|
||||
# Chapters, Episodes
|
||||
|
||||
# TOC nodes are in a list
|
||||
# each have a list of named episodes
|
||||
# each can have a named chapter
|
||||
# named chapters can be at depth 1 or 2
|
||||
# episodes might be empty (premium subscription)
|
||||
|
||||
prependSectionTitles = self.getConfig('prepend_section_titles', 'firstepisode')
|
||||
|
||||
numEpisodes = 0
|
||||
titles = []
|
||||
nestingLevel = 0
|
||||
newSection = False
|
||||
for tocNodeRef in info[workKey]['tableOfContentsV2']:
|
||||
tocNode = info[tocNodeRef['__ref']]
|
||||
|
||||
if tocNode['chapter'] is not None:
|
||||
chapter = info[tocNode['chapter']['__ref']]
|
||||
while chapter['level'] <= nestingLevel:
|
||||
titles.pop()
|
||||
nestingLevel -= 1
|
||||
titles.append(chapter['title'])
|
||||
nestingLevel = chapter['level']
|
||||
newSection = True
|
||||
else:
|
||||
titles = []
|
||||
nestingLevel = 0
|
||||
newSection = False
|
||||
|
||||
for episodeRef in tocNode['episodeUnions']:
|
||||
if not episodeRef['__ref'].startswith('EmptyEpisode'):
|
||||
numEpisodes += 1
|
||||
episode = info[episodeRef['__ref']]
|
||||
epUrl = 'https://kakuyomu.jp/works/' + self.storyId + '/episodes/' + episode['id']
|
||||
epTitle = episode['title']
|
||||
|
||||
if ((len(titles) > 0) and
|
||||
((newSection and prependSectionTitles == 'firstepisode') or
|
||||
prependSectionTitles == 'true')):
|
||||
titles.append(epTitle)
|
||||
# bracket with ZWSP to mark presence of section titles
|
||||
epTitle = u'\u200b' + u'\u3000\u200b'.join(titles)
|
||||
titles.pop()
|
||||
|
||||
self.add_chapter(epTitle, epUrl)
|
||||
newSection = False
|
||||
|
||||
logger.debug("Story: <%s>", self.story)
|
||||
return
|
||||
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Getting chapter text from <%s>' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
soup = soup.find('div', {'class':'widget-episodeBody js-episode-body'})
|
||||
if soup is None:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
soup.attrs = {'class':'episode-body'}
|
||||
|
||||
return self.utf8FromSoup(url, soup)
|
||||
|
||||
|
|
@ -144,13 +144,13 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
|
||||
# Find authorid and URL from... author urls.
|
||||
pagetitle = soup.find('div',id='pagetitle')
|
||||
for a in pagetitle.find_all('a', href=re.compile(r"viewuser.php\?uid=\d+")):
|
||||
for a in pagetitle.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")):
|
||||
self.story.addToList('authorId',a['href'].split('=')[1])
|
||||
self.story.addToList('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.addToList('author',stripHTML(a))
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -166,7 +166,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
|
@ -193,7 +193,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [stripHTML(cat) for cat in cats]
|
||||
for cat in catstext:
|
||||
# ran across one story with an empty <a href="browse.php?type=categories&catid=1"></a>
|
||||
|
|
@ -204,7 +204,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
if 'Characters' in label:
|
||||
self.story.addToList('characters','Kirk')
|
||||
self.story.addToList('characters','Spock')
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [stripHTML(char) for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',stripHTML(char))
|
||||
|
|
@ -213,7 +213,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
|
||||
genrestext = [stripHTML(genre) for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -223,7 +223,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
## has 'Story Type', which is much more what most sites
|
||||
## call genre.
|
||||
if 'Story Type' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
|
||||
genrestext = [stripHTML(genre) for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -233,21 +233,21 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warningstext = [stripHTML(warning) for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
self.story.addToList('warnings',stripHTML(warning))
|
||||
|
||||
if 'Universe' in label:
|
||||
universes = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
|
||||
universes = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
|
||||
universestext = [stripHTML(universe) for universe in universes]
|
||||
self.universe = ', '.join(universestext)
|
||||
for universe in universestext:
|
||||
self.story.addToList('universe',stripHTML(universe))
|
||||
|
||||
if 'Crossover Fandom' in label:
|
||||
crossoverfandoms = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX
|
||||
crossoverfandoms = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX
|
||||
crossoverfandomstext = [stripHTML(crossoverfandom) for crossoverfandom in crossoverfandoms]
|
||||
self.crossoverfandom = ', '.join(crossoverfandomstext)
|
||||
for crossoverfandom in crossoverfandomstext:
|
||||
|
|
@ -274,7 +274,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ from __future__ import absolute_import
|
|||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import json
|
||||
|
||||
from bs4.element import Comment
|
||||
from ..htmlcleanup import stripHTML
|
||||
|
|
@ -38,7 +37,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
#logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
|
||||
logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','litero')
|
||||
|
|
@ -48,15 +47,16 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
# where first chapter doesn't have '-ch-'.
|
||||
# Now just rely on extractChapterUrlsAndMetadata to reset
|
||||
# storyId to first chapter link.
|
||||
storyId = self.parsedUrl.path.split('/',)[2]
|
||||
|
||||
## DON'T normalize to www.literotica.com--keep for language,
|
||||
## which will be set in _setURL(url). Also, multi-chapter
|
||||
## have been keeping the language when 'normalizing' to first
|
||||
## chapter.
|
||||
url = re.sub(r"^(https?://)"+LANG_RE+r"(\.i)?",
|
||||
r"https://\2",
|
||||
r"\1\2",
|
||||
url)
|
||||
url = url.replace('/beta/','/') # to allow beta site URLs.
|
||||
url = url.replace('/beta/s/','/s/') # to allow beta site URLs.
|
||||
|
||||
## strip ?page=...
|
||||
url = re.sub(r"\?page=.*$", "", url)
|
||||
|
|
@ -66,7 +66,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%m/%d/%Y"
|
||||
self.dateformat = "%m/%d/%y"
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
|
|
@ -78,12 +78,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://www.literotica.com/s/story-title https://www.literotica.com/series/se/9999999 https://www.literotica.com/s/story-title https://www.literotica.com/i/image-or-comic-title https://www.literotica.com/p/poem-title https://portuguese.literotica.com/s/story-title https://german.literotica.com/s/story-title"
|
||||
return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
# also https://www.literotica.com/series/se/80075773
|
||||
# /s/ for story, /i/ for image/comic, /p/ for poem
|
||||
return r"https?://"+LANG_RE+r"(\.i)?\.literotica\.com/((beta/)?[sip]/([a-zA-Z0-9_-]+)|series/se/(?P<storyseriesid>[a-zA-Z0-9_-]+))"
|
||||
return r"https?://"+LANG_RE+r"(\.i)?\.literotica\.com/(beta/)?s/([a-zA-Z0-9_-]+)"
|
||||
|
||||
def _setURL(self,url):
|
||||
# logger.debug("set URL:%s"%url)
|
||||
|
|
@ -92,337 +90,260 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
lang = m.group('lang')
|
||||
if lang not in ('www','other'):
|
||||
self.story.setMetadata('language',lang.capitalize())
|
||||
# reset storyId
|
||||
self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[-1])
|
||||
# logger.debug("language:%s"%self.story.getMetadata('language'))
|
||||
|
||||
## apply clean_chapter_titles
|
||||
def add_chapter(self,chapter_title,url,othermeta={}):
|
||||
if self.getConfig("clean_chapter_titles"):
|
||||
storytitle = self.story.getMetadataRaw('title').lower()
|
||||
chapter_name_type = None
|
||||
# strip trailing ch or pt before doing the chapter clean.
|
||||
# doesn't remove from story title metadata
|
||||
storytitle = re.sub(r'^(.*?)( (ch|pt))?$',r'\1',storytitle)
|
||||
if chapter_title.lower().startswith(storytitle):
|
||||
chapter = chapter_title[len(storytitle):].strip()
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter == '':
|
||||
chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
|
||||
# Sometimes the first chapter does not have type of chapter
|
||||
if self.num_chapters() == 0:
|
||||
# logger.debug('\tChapter: first chapter without chapter type')
|
||||
chapter_name_type = None
|
||||
else:
|
||||
separater_char = chapter[0]
|
||||
# logger.debug('\tseparater_char: "%s"' % separater_char)
|
||||
chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter.lower().startswith('ch.'):
|
||||
chapter = chapter[len('ch.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Chapter %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Chapter %s' % chapter
|
||||
chapter_name_type = 'Chapter' if chapter_name_type is None else chapter_name_type
|
||||
# logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif chapter.lower().startswith('pt.'):
|
||||
chapter = chapter[len('pt.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Part %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Part %s' % chapter
|
||||
chapter_name_type = 'Part' if chapter_name_type is None else chapter_name_type
|
||||
# logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif separater_char in [":", "-"]:
|
||||
chapter_title = chapter
|
||||
# logger.debug('\tChapter: taking chapter text as whole')
|
||||
super(LiteroticaSiteAdapter, self).add_chapter(chapter_title,url,othermeta)
|
||||
def getCategories(self, soup):
|
||||
if self.getConfig("use_meta_keywords"):
|
||||
categories = soup.find("meta", {"name":"keywords"})['content'].split(',')
|
||||
categories = [c for c in categories if not self.story.getMetadata('title') in c]
|
||||
if self.story.getMetadata('author') in categories:
|
||||
categories.remove(self.story.getMetadata('author'))
|
||||
# logger.debug("Meta = %s" % categories)
|
||||
for category in categories:
|
||||
# logger.debug("\tCategory=%s" % category)
|
||||
# self.story.addToList('category', category.title())
|
||||
self.story.addToList('eroticatags', category.title())
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
"""
|
||||
In April 2024, site introduced significant changes, including
|
||||
adding a 'Story Series' page and link to it in each chapter.
|
||||
But not all stories, one-shots don't have 'Story Series'.
|
||||
|
||||
literotica has 'Story Series' & 'Story'. FFF calls them 'Story' & 'Chapters'
|
||||
See https://github.com/JimmXinu/FanFicFare/issues/1058#issuecomment-2078490037
|
||||
|
||||
So /series/se/ will be the story URL for multi chapters but
|
||||
keep individual 'chapter' URL for one-shots.
|
||||
NOTE: Some stories can have versions,
|
||||
e.g. /my-story-ch-05-version-10
|
||||
NOTE: If two stories share the same title, a running index is added,
|
||||
e.g.: /my-story-ch-02-1
|
||||
Strategy:
|
||||
* Go to author's page, search for the current story link,
|
||||
* If it's in a tr.root-story => One-part story
|
||||
* , get metadata and be done
|
||||
* If it's in a tr.sl => Chapter in series
|
||||
* Search up from there until we find a tr.ser-ttl (this is the
|
||||
story)
|
||||
* Gather metadata
|
||||
* Search down from there for all tr.sl until the next
|
||||
tr.ser-ttl, foreach
|
||||
* Chapter link is there
|
||||
"""
|
||||
logger.debug("Chapter/Story URL: <%s> " % self.url)
|
||||
|
||||
if not (self.is_adult or self.getConfig("is_adult")):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
(data,rurl) = self.get_request_redirected(self.url)
|
||||
# logger.debug(data)
|
||||
# logger.debug("Chapter/Story URL: <%s> " % self.url)
|
||||
|
||||
(data1,rurl) = self.get_request_redirected(self.url)
|
||||
## for language domains
|
||||
self._setURL(rurl)
|
||||
logger.debug("set opened url:%s"%self.url)
|
||||
soup = self.make_soup(data)
|
||||
soup1 = self.make_soup(data1)
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
if "This submission is awaiting moderator's approval" in data:
|
||||
if "This submission is awaiting moderator's approval" in data1:
|
||||
raise exceptions.StoryDoesNotExist("This submission is awaiting moderator's approval. %s"%self.url)
|
||||
|
||||
## 2025Feb - domains other than www now use different HTML.
|
||||
## Need to look for two different versions of basically
|
||||
## everything.
|
||||
|
||||
## not series URL, assumed to be a chapter. Look for Story
|
||||
## Info block of post-beta page. I don't think it should happen?
|
||||
if '/series/se' not in self.url:
|
||||
#logger.debug(data)
|
||||
## looking for /series/se URL to indicate this is a
|
||||
## chapter.
|
||||
if not soup.select_one('div.page__aside') and not soup.select_one('div.sidebar') and not soup.select_one('div[class^="_sidebar_"]'):
|
||||
raise exceptions.FailedToDownload("Missing Story Info block, Beta turned off?")
|
||||
|
||||
storyseriestag = soup.select_one('a.bn_av')
|
||||
if not storyseriestag:
|
||||
storyseriestag = soup.select_one('a[class^="_files__link_"]')
|
||||
# logger.debug("Story Series Tag:%s"%storyseriestag)
|
||||
|
||||
if storyseriestag:
|
||||
self._setURL(storyseriestag['href'])
|
||||
data = self.get_request(storyseriestag['href'])
|
||||
# logger.debug(data)
|
||||
soup = self.make_soup(data)
|
||||
# logger.debug(soup)
|
||||
else:
|
||||
logger.debug("One-shot")
|
||||
|
||||
isSingleStory = '/series/se' not in self.url
|
||||
|
||||
if not isSingleStory:
|
||||
# Normilize the url?
|
||||
state = re.findall(r"prefix\=\"/series/\",state='(.+?)'</script>", data)
|
||||
json_state = json.loads(state[0].replace("\\'","'").replace("\\\\","\\"))
|
||||
url_series_id = unicode(re.match(self.getSiteURLPattern(),self.url).group('storyseriesid'))
|
||||
json_series_id = unicode(json_state['series']['data']['id'])
|
||||
if json_series_id != url_series_id:
|
||||
res = re.sub(url_series_id, json_series_id, unicode(self.url))
|
||||
logger.debug("Normalized url: %s"%res)
|
||||
self._setURL(res)
|
||||
|
||||
## common between one-shots and multi-chapters
|
||||
# title
|
||||
self.story.setMetadata('title', stripHTML(soup.select_one('h1')))
|
||||
# logger.debug(self.story.getMetadata('title'))
|
||||
|
||||
# author
|
||||
## XXX This is still the author URL like:
|
||||
## https://www.literotica.com/stories/memberpage.php?uid=999999&page=submissions
|
||||
## because that's what's on the page. It redirects to the /authors/ page.
|
||||
## Only way I know right now to get the /authors/ is to make
|
||||
## the req and look at the redirect.
|
||||
## Should change to /authors/ if/when it starts appearing.
|
||||
## Assuming it's in the same place.
|
||||
authora = soup.find("a", class_="y_eU")
|
||||
if not authora:
|
||||
authora = soup.select_one('a[class^="_author__title"]')
|
||||
authora = soup1.find("a", class_="y_eU")
|
||||
authorurl = authora['href']
|
||||
if authorurl.startswith('//'):
|
||||
authorurl = self.parsedUrl.scheme+':'+authorurl
|
||||
# logger.debug(authora)
|
||||
# logger.debug(authorurl)
|
||||
self.story.setMetadata('author', stripHTML(authora))
|
||||
self.story.setMetadata('authorId', urlparse.parse_qs(authorurl.split('?')[1])['uid'][0])
|
||||
if authorurl.startswith('//'):
|
||||
authorurl = self.parsedUrl.scheme+':'+authorurl
|
||||
self.story.setMetadata('authorUrl', authorurl)
|
||||
if '?' in authorurl:
|
||||
self.story.setMetadata('authorId', urlparse.parse_qs(authorurl.split('?')[1])['uid'][0])
|
||||
elif '/authors/' in authorurl:
|
||||
self.story.setMetadata('authorId', authorurl.split('/')[-1])
|
||||
else: # if all else fails
|
||||
self.story.setMetadata('authorId', stripHTML(authora))
|
||||
self.story.setMetadata('author', authora.text)
|
||||
|
||||
if soup.select('div#tabpanel-tags'):
|
||||
# logger.debug("tags1")
|
||||
self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div#tabpanel-tags a.av_as') ])
|
||||
if soup.select('div[class^="_widget__tags_"]'):
|
||||
# logger.debug("tags2")
|
||||
self.story.extendList('eroticatags', [ stripHTML(t).title() for t in soup.select('div[class^="_widget__tags_"] a[class^="_tag_item_"]') ])
|
||||
# logger.debug(self.story.getList('eroticatags'))
|
||||
# get the author page
|
||||
dataAuth = self.get_request(authorurl)
|
||||
soupAuth = self.make_soup(dataAuth)
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
# logger.debug(soupAuth)
|
||||
|
||||
## look first for 'Series Introduction', then Info panel short desc
|
||||
## series can have either, so put in common code.
|
||||
desc = []
|
||||
introtag = soup.select_one('div.bp_rh')
|
||||
descdiv = soup.select_one('div#tabpanel-info div.bn_B') or \
|
||||
soup.select_one('div[class^="_tab__pane_"] div[class^="_widget__info_"]')
|
||||
if introtag and stripHTML(introtag):
|
||||
# make sure there's something in the tag.
|
||||
# logger.debug("intro %s"%introtag)
|
||||
desc.append(unicode(introtag))
|
||||
elif descdiv and stripHTML(descdiv):
|
||||
# make sure there's something in the tag.
|
||||
# logger.debug("desc %s"%descdiv)
|
||||
desc.append(unicode(descdiv))
|
||||
if not desc or self.getConfig("include_chapter_descriptions_in_summary"):
|
||||
## Only for backward compatibility with 'stories' that
|
||||
## don't have an intro or short desc.
|
||||
descriptions = []
|
||||
for i, chapterdesctag in enumerate(soup.select('p.br_rk')):
|
||||
# remove category link, but only temporarily
|
||||
a = chapterdesctag.a.extract()
|
||||
descriptions.append("%d. %s" % (i + 1, stripHTML(chapterdesctag)))
|
||||
# now put it back--it's used below
|
||||
chapterdesctag.append(a)
|
||||
desc.append(unicode("<p>"+"</p>\n<p>".join(descriptions)+"</p>"))
|
||||
## Find link to url in author's page
|
||||
## site has started using //domain.name/asdf urls remove https?: from front
|
||||
## site has started putting https back on again.
|
||||
## site is now using language specific german.lit... etc on author pages.
|
||||
## site is now back to using www.lit... etc on author pages.
|
||||
search_url_re = r"https?://"+LANG_RE+r"(\.i)?\." + re.escape(self.getSiteDomain()) + self.url[self.url.index('/s/'):]+r"$"
|
||||
logger.debug(search_url_re)
|
||||
storyLink = soupAuth.find('a', href=re.compile(search_url_re))
|
||||
# storyLink = soupAuth.find('a', href=re.compile(r'.*literotica.com/s/'+re.escape(self.story.getMetadata('storyId')) ))
|
||||
# storyLink = soupAuth.find('a', href=re.compile(r'(https?:)?'+re.escape(self.url[self.url.index(':')+1:]).replace(r'www',r'[^\.]+') ))
|
||||
# storyLink = soupAuth.find('a', href=self.url)#[self.url.index(':')+1:])
|
||||
|
||||
self.setDescription(self.url,u''.join(desc))
|
||||
if storyLink is not None:
|
||||
# pull the published date from the author page
|
||||
# default values from single link. Updated below if multiple chapter.
|
||||
# logger.debug("Found story on the author page.")
|
||||
date = storyLink.parent.parent.findAll('td')[-1].text
|
||||
self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
|
||||
self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
|
||||
|
||||
if storyLink is not None:
|
||||
urlTr = storyLink.parent.parent
|
||||
if "sl" in urlTr['class']:
|
||||
isSingleStory = False
|
||||
else:
|
||||
isSingleStory = True
|
||||
else:
|
||||
raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (self.url, authorurl))
|
||||
|
||||
if isSingleStory:
|
||||
## one-shots don't *display* date info, but they have it
|
||||
## hidden in <script>
|
||||
## shows _date_approve "date_approve":"01/31/2024"
|
||||
|
||||
## multichap also have "date_approve", but they have
|
||||
## several and they're more than just the story chapters.
|
||||
date = re.search(r'"date_approve":"(\d\d/\d\d/\d\d\d\d)"',data)
|
||||
if not date:
|
||||
date = re.search(r'date_approve:"(\d\d/\d\d/\d\d\d\d)"',data)
|
||||
if date:
|
||||
dateval = makeDate(date.group(1), self.dateformat)
|
||||
self.story.setMetadata('datePublished', dateval)
|
||||
self.story.setMetadata('dateUpdated', dateval)
|
||||
|
||||
## one-shots don't have same json data to get aver_rating
|
||||
## from below. This kludge matches the data_approve
|
||||
rateall = re.search(r'rate_all:([\d\.]+)',data)
|
||||
if rateall:
|
||||
self.story.setMetadata('averrating', '%4.2f' % float(rateall.group(1)))
|
||||
|
||||
## one-shots assumed completed.
|
||||
self.story.setMetadata('status','Completed')
|
||||
|
||||
# Add the category from the breadcumb.
|
||||
breadcrumbs = soup.find('div', id='BreadCrumbComponent')
|
||||
if not breadcrumbs:
|
||||
breadcrumbs = soup.select_one('ul[class^="_breadcrumbs_list_"]')
|
||||
if not breadcrumbs:
|
||||
# _breadcrumbs_18u7l_1
|
||||
breadcrumbs = soup.select_one('nav[class^="_breadcrumbs_"]')
|
||||
self.story.addToList('category', breadcrumbs.find_all('a')[1].string)
|
||||
|
||||
## one-shot chapter
|
||||
self.add_chapter(self.story.getMetadata('title'), self.url)
|
||||
|
||||
self.story.setMetadata('title', storyLink.text.strip('/'))
|
||||
# logger.debug('Title: "%s"' % storyLink.text.strip('/'))
|
||||
self.setDescription(authorurl, urlTr.findAll("td")[1].text)
|
||||
self.story.addToList('category', urlTr.findAll("td")[2].text)
|
||||
# self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
|
||||
date = urlTr.findAll('td')[-1].text
|
||||
self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
|
||||
self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
|
||||
self.add_chapter(storyLink.text, self.url)
|
||||
averrating = stripHTML(storyLink.parent)
|
||||
## title (0.00)
|
||||
averrating = averrating[averrating.rfind('(')+1:averrating.rfind(')')]
|
||||
try:
|
||||
self.story.setMetadata('averrating', float(averrating))
|
||||
except:
|
||||
pass
|
||||
# self.story.setMetadata('averrating',averrating)
|
||||
# parse out the list of chapters
|
||||
else:
|
||||
## Multi-chapter stories. AKA multi-part 'Story Series'.
|
||||
bn_antags = soup.select('div#tabpanel-info p.bn_an')
|
||||
# logger.debug(bn_antags)
|
||||
if bn_antags and not self.getConfig("dates_from_chapters"):
|
||||
## Use dates from series metadata unless dates_from_chapters is enabled
|
||||
dates = []
|
||||
for datetag in bn_antags[:2]:
|
||||
datetxt = stripHTML(datetag)
|
||||
# remove 'Started:' 'Updated:'
|
||||
# Assume can't use 'Started:' 'Updated:' (vs [0] or [1]) because of lang localization
|
||||
datetxt = datetxt[datetxt.index(':')+1:]
|
||||
dates.append(datetxt)
|
||||
# logger.debug(dates)
|
||||
self.story.setMetadata('datePublished', makeDate(dates[0], self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(dates[1], self.dateformat))
|
||||
seriesTr = urlTr.previousSibling
|
||||
while 'ser-ttl' not in seriesTr['class']:
|
||||
seriesTr = seriesTr.previousSibling
|
||||
m = re.match(r"^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
|
||||
self.story.setMetadata('title', m.group('title'))
|
||||
seriesTitle = m.group('title')
|
||||
|
||||
## bn_antags[2] contains "The author has completed this series." or "The author is still actively writing this series."
|
||||
## I won't be surprised if this breaks later because of lang localization
|
||||
if "completed" in stripHTML(bn_antags[-1]):
|
||||
self.story.setMetadata('status','Completed')
|
||||
else:
|
||||
self.story.setMetadata('status','In-Progress')
|
||||
## Walk the chapters
|
||||
chapterTr = seriesTr.nextSibling
|
||||
dates = []
|
||||
descriptions = []
|
||||
ratings = []
|
||||
chapters = []
|
||||
chapter_name_type = None
|
||||
while chapterTr is not None and 'sl' in chapterTr['class']:
|
||||
description = "%d. %s" % (len(descriptions)+1,stripHTML(chapterTr.findAll("td")[1]))
|
||||
description = stripHTML(chapterTr.findAll("td")[1])
|
||||
chapterLink = chapterTr.find("td", "fc").find("a")
|
||||
if self.getConfig('chapter_categories_use_all'):
|
||||
self.story.addToList('category', chapterTr.findAll("td")[2].text)
|
||||
# self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
|
||||
pub_date = makeDate(chapterTr.findAll('td')[-1].text, self.dateformat)
|
||||
dates.append(pub_date)
|
||||
chapterTr = chapterTr.nextSibling
|
||||
|
||||
## category from chapter list
|
||||
self.story.extendList('category',[ stripHTML(t) for t in soup.select('a.br_rl') ])
|
||||
chapter_title = chapterLink.text
|
||||
if self.getConfig("clean_chapter_titles"):
|
||||
# logger.debug('\tChapter Name: "%s"' % chapterLink.text)
|
||||
if chapterLink.text.lower().startswith(seriesTitle.lower()):
|
||||
chapter = chapterLink.text[len(seriesTitle):].strip()
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter == '':
|
||||
chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
|
||||
# Sometimes the first chapter does not have type of chapter
|
||||
if self.num_chapters() == 0:
|
||||
logger.debug('\tChapter: first chapter without chapter type')
|
||||
chapter_name_type = None
|
||||
else:
|
||||
separater_char = chapter[0]
|
||||
# logger.debug('\tseparater_char: "%s"' % separater_char)
|
||||
chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
|
||||
# logger.debug('\tChapter: "%s"' % chapter)
|
||||
if chapter.lower().startswith('ch.'):
|
||||
chapter = chapter[len('ch.'):].strip()
|
||||
try:
|
||||
chapter_title = 'Chapter %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Chapter %s' % chapter
|
||||
chapter_name_type = 'Chapter' if chapter_name_type is None else chapter_name_type
|
||||
logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif chapter.lower().startswith('pt.'):
|
||||
chapter = chapter[len('pt.'):]
|
||||
try:
|
||||
chapter_title = 'Part %d' % int(chapter)
|
||||
except:
|
||||
chapter_title = 'Part %s' % chapter
|
||||
chapter_name_type = 'Part' if chapter_name_type is None else chapter_name_type
|
||||
logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
elif separater_char in [":", "-"]:
|
||||
chapter_title = chapter
|
||||
logger.debug('\tChapter: taking chapter text as whole')
|
||||
|
||||
for chapteratag in soup.select('a.br_rj'):
|
||||
chapter_title = stripHTML(chapteratag)
|
||||
# logger.debug('\tChapter: "%s"' % chapteratag)
|
||||
# /series/se does include full URLs current.
|
||||
chapurl = chapteratag['href']
|
||||
# pages include full URLs.
|
||||
chapurl = chapterLink['href']
|
||||
if chapurl.startswith('//'):
|
||||
chapurl = self.parsedUrl.scheme + ':' + chapurl
|
||||
# logger.debug("Chapter URL: " + chapurl)
|
||||
self.add_chapter(chapter_title, chapurl)
|
||||
# logger.debug("Chapter Title: " + chapter_title)
|
||||
# logger.debug("Chapter description: " + description)
|
||||
chapters.append((chapter_title, chapurl, description, pub_date))
|
||||
# self.add_chapter(chapter_title, chapurl)
|
||||
numrating = stripHTML(chapterLink.parent)
|
||||
## title (0.00)
|
||||
numrating = numrating[numrating.rfind('(')+1:numrating.rfind(')')]
|
||||
try:
|
||||
ratings.append(float(numrating))
|
||||
except:
|
||||
pass
|
||||
|
||||
# <img src="https://uploads.literotica.com/series/cover/813-1695143444-desktop-x1.jpg" alt="Series cover">
|
||||
coverimg = soup.select_one('img[alt="Series cover"]')
|
||||
if coverimg:
|
||||
self.setCoverImage(self.url,coverimg['src'])
|
||||
if self.getConfig("clean_chapter_titles") \
|
||||
and chapter_name_type is not None \
|
||||
and not chapters[0][0].startswith(chapter_name_type):
|
||||
logger.debug('\tChapter: chapter_name_type="%s"' % chapter_name_type)
|
||||
logger.debug('\tChapter: first chapter="%s"' % chapters[0][0])
|
||||
logger.debug('\tChapter: first chapter number="%s"' % chapters[0][0][len('Chapter'):])
|
||||
chapters[0] = ("%s %s" % (chapter_name_type, chapters[0][0][len('Chapter'):].strip()),
|
||||
chapters[0][1],
|
||||
chapters[0][2],
|
||||
chapters[0][3]
|
||||
)
|
||||
|
||||
#### Attempting averrating from JS metadata.
|
||||
#### also alternate chapters from json
|
||||
try:
|
||||
state_start="state='"
|
||||
state_end="'</script>"
|
||||
i = data.index(state_start)
|
||||
if i:
|
||||
state = data[i+len(state_start):data.index(state_end,i)].replace("\\'","'").replace("\\\\","\\")
|
||||
if state:
|
||||
# logger.debug(state)
|
||||
json_state = json.loads(state)
|
||||
# logger.debug(json.dumps(json_state, sort_keys=True,indent=2, separators=(',', ':')))
|
||||
all_rates = []
|
||||
if 'series' in json_state:
|
||||
all_rates = [ float(x['rate_all']) for x in json_state['series']['works'] ]
|
||||
chapters = sorted(chapters, key=lambda chapter: chapter[3])
|
||||
for i, chapter in enumerate(chapters):
|
||||
self.add_chapter(chapter[0], chapter[1])
|
||||
descriptions.append("%d. %s" % (i + 1, chapter[2]))
|
||||
## Set the oldest date as publication date, the newest as update date
|
||||
dates.sort()
|
||||
self.story.setMetadata('datePublished', dates[0])
|
||||
self.story.setMetadata('dateUpdated', dates[-1])
|
||||
self.story.setMetadata('datePublished', chapters[0][3])
|
||||
self.story.setMetadata('dateUpdated', chapters[-1][3])
|
||||
## Set description to joint chapter descriptions
|
||||
self.setDescription(authorurl,"<p>"+"</p>\n<p>".join(descriptions)+"</p>")
|
||||
|
||||
## Extract dates from chapter approval dates if dates_from_chapters is enabled
|
||||
if self.getConfig("dates_from_chapters"):
|
||||
date_approvals = []
|
||||
for work in json_state['series']['works']:
|
||||
if 'date_approve' in work:
|
||||
try:
|
||||
date_approvals.append(makeDate(work['date_approve'], self.dateformat))
|
||||
except:
|
||||
pass
|
||||
if date_approvals:
|
||||
# Oldest date is published, newest is updated
|
||||
date_approvals.sort()
|
||||
self.story.setMetadata('datePublished', date_approvals[0])
|
||||
self.story.setMetadata('dateUpdated', date_approvals[-1])
|
||||
if all_rates:
|
||||
self.story.setMetadata('averrating', '%4.2f' % (sum(all_rates) / float(len(all_rates))))
|
||||
if len(ratings) > 0:
|
||||
self.story.setMetadata('averrating','%4.2f' % (sum(ratings) / float(len(ratings))))
|
||||
|
||||
## alternate chapters from JSON
|
||||
if self.num_chapters() < 1:
|
||||
logger.debug("Getting Chapters from series JSON")
|
||||
seriesid = json_state.get('series',{}).get('data',{}).get('id',None)
|
||||
if seriesid:
|
||||
logger.info("Fetching chapter data from JSON")
|
||||
logger.debug(seriesid)
|
||||
series_json = json.loads(self.get_request('https://literotica.com/api/3/series/%s/works'%seriesid))
|
||||
# logger.debug(json.dumps(series_json, sort_keys=True,indent=2, separators=(',', ':')))
|
||||
for chap in series_json:
|
||||
self.add_chapter(chap['title'], 'https://www.literotica.com/s/'+chap['url'])
|
||||
# normalize on first chapter URL.
|
||||
self._setURL(self.get_chapter(0,'url'))
|
||||
|
||||
## Collect tags from series/story page if tags_from_chapters is enabled
|
||||
if self.getConfig("tags_from_chapters"):
|
||||
self.story.extendList('eroticatags', [ unicode(t['tag']).title() for t in chap['tags'] ])
|
||||
# reset storyId to first chapter.
|
||||
self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Processing JSON failed. (%s)"%e)
|
||||
# Add the category from the breadcumb. This might duplicate a category already added.
|
||||
self.story.addToList('category', soup1.find('div', id='BreadCrumbComponent').findAll('a')[1].string)
|
||||
self.getCategories(soup1)
|
||||
|
||||
## Features removed because not supportable by new site form:
|
||||
## averrating metadata entry
|
||||
## order_chapters_by_date option
|
||||
## use_meta_keywords option
|
||||
return
|
||||
|
||||
|
||||
def getPageText(self, raw_page, url):
|
||||
logger.debug('Getting page text')
|
||||
# logger.debug('Getting page text')
|
||||
# logger.debug(soup)
|
||||
raw_page = raw_page.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
|
||||
# logger.debug("\tChapter text: %s" % raw_page)
|
||||
# logger.debug("\tChapter text: %s" % raw_page)
|
||||
page_soup = self.make_soup(raw_page)
|
||||
[comment.extract() for comment in page_soup.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
fullhtml = ""
|
||||
for aa_ht_div in page_soup.find_all('div', 'aa_ht') + page_soup.select('div[class^="_article__content_"]'):
|
||||
if aa_ht_div.div:
|
||||
html = unicode(aa_ht_div.div)
|
||||
# Strip some starting and ending tags,
|
||||
html = re.sub(r'^<div.*?>', r'', html)
|
||||
html = re.sub(r'</div>$', r'', html)
|
||||
html = re.sub(r'<p></p>$', r'', html)
|
||||
fullhtml = fullhtml + html
|
||||
# logger.debug('getPageText - fullhtml: %s' % fullhtml)
|
||||
[comment.extract() for comment in page_soup.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
story2 = page_soup.find('div', 'aa_ht').div
|
||||
# logger.debug('getPageText - story2: %s' % story2)
|
||||
|
||||
fullhtml = unicode(story2)
|
||||
# logger.debug(fullhtml)
|
||||
# Strip some starting and ending tags,
|
||||
fullhtml = re.sub(r'^<div.*?>', r'', fullhtml)
|
||||
fullhtml = re.sub(r'</div>$', r'', fullhtml)
|
||||
fullhtml = re.sub(r'<p></p>$', r'', fullhtml)
|
||||
# logger.debug('getPageText - fullhtml: %s' % fullhtml)
|
||||
return fullhtml
|
||||
|
||||
def getChapterText(self, url):
|
||||
|
|
@ -432,15 +353,9 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
raw_page = self.get_request(url)
|
||||
page_soup = self.make_soup(raw_page)
|
||||
pages = page_soup.find('div',class_='l_bH')
|
||||
if not pages:
|
||||
pages = page_soup.select_one('div._pagination_h0sum_1')
|
||||
if not pages:
|
||||
pages = page_soup.select_one('div.clearfix.panel._pagination_1400x_1')
|
||||
if not pages:
|
||||
pages = page_soup.select_one('div[class^="panel clearfix _pagination_"]')
|
||||
# logger.debug(pages)
|
||||
|
||||
fullhtml = ""
|
||||
self.getCategories(page_soup)
|
||||
chapter_description = ''
|
||||
if self.getConfig("description_in_chapter"):
|
||||
chapter_description = page_soup.find("meta", {"name" : "description"})['content']
|
||||
|
|
@ -451,10 +366,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
## look for highest numbered page, they're not all listed
|
||||
## when there are many.
|
||||
|
||||
last_page_links = pages.find_all('a', class_='l_bJ')
|
||||
if not last_page_links:
|
||||
last_page_links = pages.select('a[class^="_pagination__item_"]')
|
||||
last_page_link = last_page_links[-1]
|
||||
last_page_link = pages.find_all('a', class_='l_bJ')[-1]
|
||||
last_page_no = int(urlparse.parse_qs(last_page_link['href'].split('?')[1])['page'][0])
|
||||
# logger.debug(last_page_no)
|
||||
for page_no in range(2, last_page_no+1):
|
||||
|
|
@ -463,7 +375,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
raw_page = self.get_request(page_url)
|
||||
fullhtml += self.getPageText(raw_page, url)
|
||||
|
||||
#logger.debug(fullhtml)
|
||||
# logger.debug(fullhtml)
|
||||
page_soup = self.make_soup(fullhtml)
|
||||
fullhtml = self.utf8FromSoup(url, self.make_soup(fullhtml))
|
||||
fullhtml = chapter_description + fullhtml
|
||||
|
|
@ -471,123 +383,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
return fullhtml
|
||||
|
||||
def get_urls_from_page(self,url,normalize):
|
||||
from ..geturls import get_urls_from_html
|
||||
|
||||
## hook for logins, etc.
|
||||
self.before_get_urls_from_page(url,normalize)
|
||||
|
||||
# this way it uses User-Agent or other special settings.
|
||||
data = self.get_request(url,usecache=False)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
page_urls = get_urls_from_html(soup, url, configuration=self.configuration, normalize=normalize)
|
||||
|
||||
if not self.getConfig("fetch_stories_from_api",True):
|
||||
logger.debug('fetch_stories_from_api Not enabled')
|
||||
return {'urllist': page_urls}
|
||||
|
||||
user_story_list = re.search(r'literotica\.com/authors/.+?/lists\?listid=(?P<list_id>\d+)', url)
|
||||
fav_authors = re.search(r'literotica\.com/authors/.+?/favorites', url)
|
||||
written = re.search(r'literotica.com/authors/.+?/works/', url)
|
||||
logger.debug((bool(user_story_list), bool(fav_authors), bool(written)))
|
||||
|
||||
# If the url is not supported
|
||||
if not user_story_list and not fav_authors and not written:
|
||||
logger.debug('No supported link. %s', url)
|
||||
return {'urllist':page_urls}
|
||||
|
||||
# Grabbing the main list where chapters are contained.
|
||||
if user_story_list:
|
||||
js_story_list = re.search(r';\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]\);\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]=\{success:!\d,current_page:(?P<current_page>\d+?),last_page:(?P<last_page>\d+?),total:\d+?,per_page:\d+,(has_series:!\d)?data:\$R\[\d+?\]=\[\$R\[\d+?\]=(?P<data>.+)\}\]\}\);', data) # }] } } }); \$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]\);\$R\[\d+?]\(\$R\[\d+?\],\$R\[\d+?\]=\{sliders:
|
||||
logger.debug('user_story_list ID [%s]'%user_story_list.group('list_id'))
|
||||
else:
|
||||
js_story_list = re.search(r'\$R\[\d+?\]\(\$R\[\d+?\],\$R\[\d+?\]={current_page:(?P<current_page>\d+?),last_page:(?P<last_page>\d+?),total:\d+?,per_page:\d+,(has_series:!\d,)?data:\$R\[\d+\]=\[\$R\[\d+\]=\{(?!aim)(?P<data>.+)\}\);_\$HY\.r\[', data)
|
||||
|
||||
# In case the regex becomes outdated
|
||||
if not js_story_list:
|
||||
logger.debug('Failed to grab data from the js.')
|
||||
return {'urllist':page_urls}
|
||||
|
||||
user = None
|
||||
script_tags = soup.find_all('script')
|
||||
for script in script_tags:
|
||||
if not script.string:
|
||||
continue
|
||||
# Getting author from the js.
|
||||
user = re.search(r'_\$HY\.r\[\"AuthorQuery\[\\\"(?P<author>.+?)\\\"\]\"\]', script.string)
|
||||
if user != None:
|
||||
logger.debug("User: [%s]"%user.group('author'))
|
||||
break
|
||||
else:
|
||||
logger.debug('Failed to get a username')
|
||||
return {'urllist': page_urls}
|
||||
|
||||
# Extract the current (should be 1) and last page numbers from the js.
|
||||
logger.debug("Pages %s/%s"%(js_story_list.group('current_page'), js_story_list.group('last_page')))
|
||||
|
||||
urls = []
|
||||
# Necessary to format a proper link as there were no visible data specifying what kind of link that should be.
|
||||
cat_to_link = {'adult-comics': 'i', 'erotic-art': 'i', 'illustrated-poetry': 'p', 'erotic-audio-poetry': 'p', 'erotic-poetry': 'p', 'non-erotic-poetry': 'p'}
|
||||
stories_found = re.findall(r"category_info:\$R\[.*?type:\".+?\",pageUrl:\"(.+?)\"}.+?,type:\"(.+?)\",url:\"(.+?)\",", js_story_list.group('data'))
|
||||
for story in stories_found:
|
||||
story_category, story_type, story_url = story
|
||||
urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(story_category, 's'), story_url))
|
||||
|
||||
# Removes the duplicates
|
||||
seen = set()
|
||||
urls = [x for x in (page_urls + urls) if not (x in seen or seen.add(x))]
|
||||
logger.debug("Found [%s] stories so far."%len(urls))
|
||||
|
||||
# Sometimes the rest of the stories are burried in the js so no fetching in necessery.
|
||||
if js_story_list.group('last_page') == js_story_list.group('current_page'):
|
||||
return {'urllist': urls}
|
||||
|
||||
user = urlparse.quote(user.group(1))
|
||||
logger.debug("Escaped user: [%s]"%user)
|
||||
|
||||
if written:
|
||||
category = re.search(r"_\$HY\.r\[\"AuthorSeriesAndWorksQuery\[\\\".+?\\\",\\\"\D+?\\\",\\\"(?P<type>\D+?)\\\"\]\"\]=\$R\[\d+?\]=\$R\[\d+?\]\(\$R\[\d+?\]=\{", data)
|
||||
elif fav_authors:
|
||||
category = re.search(r"_\$HY\.r\[\"AuthorFavoriteWorksQuery\[\\\".+?\\\",\\\"(?P<type>\D+?)\\\",\d\]\"\]=\$R\[\d+?\]=\$R\[\d+?\]\(\$R\[\d+?\]={", data)
|
||||
|
||||
if not user_story_list and not category:
|
||||
logger.debug("Type of works not found")
|
||||
return {'urllist': urls}
|
||||
|
||||
last_page = int(js_story_list.group('last_page'))
|
||||
current_page = int(js_story_list.group('current_page')) + 1
|
||||
# Fetching the remaining urls from api. Can't trust the number given about the pages left from a website. Sometimes even the api returns outdated number of pages.
|
||||
while current_page <= last_page:
|
||||
i = len(urls)
|
||||
logger.debug("Pages %s/%s"%(current_page, int(last_page)))
|
||||
if fav_authors:
|
||||
jsn = self.get_request('https://literotica.com/api/3/users/{}/favorite/works?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22type%22%3A%22{}%22%2C%22withSeriesDetails%22%3Atrue%7D'.format(user, current_page, category.group('type')))
|
||||
elif user_story_list:
|
||||
jsn = self.get_request('https://literotica.com/api/3/users/{}/list/{}?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22withSeriesDetails%22%3Atrue%7D'.format(user, user_story_list.group('list_id'), current_page))
|
||||
else:
|
||||
jsn = self.get_request('https://literotica.com/api/3/users/{}/series_and_works?params=%7B%22page%22%3A{}%2C%22pageSize%22%3A50%2C%22sort%22%3A%22date%22%2C%22type%22%3A%22{}%22%2C%22listType%22%3A%22expanded%22%7D'.format(user, current_page, category.group('type')))
|
||||
|
||||
urls_data = json.loads(jsn)
|
||||
last_page = urls_data["last_page"]
|
||||
current_page = int(urls_data["current_page"]) + 1
|
||||
for story in urls_data['data']:
|
||||
#logger.debug('parts' in story)
|
||||
if story['url'] and story.get('work_count') == None:
|
||||
urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(story["category_info"]["pageUrl"], 's'), str(story['url'])))
|
||||
continue
|
||||
# Most of the time series has no url specified and contains all of the story links belonging to the series
|
||||
urls.append('https://www.literotica.com/series/se/%s'%str(story['id']))
|
||||
for series_story in story['parts']:
|
||||
urls.append('https://www.literotica.com/%s/%s'%(cat_to_link.get(series_story["category_info"]["pageUrl"], 's'), str(series_story['url'])))
|
||||
logger.debug("Found [%s] stories."%(len(urls) - i))
|
||||
|
||||
# Again removing duplicates.
|
||||
seen = set()
|
||||
urls = [x for x in urls if not (x in seen or seen.add(x))]
|
||||
|
||||
logger.debug("Found total of [%s] stories"%len(urls))
|
||||
return {'urllist':urls}
|
||||
|
||||
def getClass():
|
||||
return LiteroticaSiteAdapter
|
||||
|
|
|
|||
50
fanficfare/adapters/adapter_looselugscom.py
Normal file
50
fanficfare/adapters/adapter_looselugscom.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2018 FanFicFare team
|
||||
# Copyright 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
##################################################################################
|
||||
### Rewritten by: GComyn on November, 06, 2016
|
||||
### Original was adapter_fannation.py
|
||||
##################################################################################
|
||||
from __future__ import absolute_import
|
||||
from .base_efiction_adapter import BaseEfictionAdapter
|
||||
|
||||
class LooseLugsComAdapter(BaseEfictionAdapter):
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'www.looselugs.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(self):
|
||||
return 'looselugs'
|
||||
|
||||
@classmethod
|
||||
def getDateFormat(self):
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
return "%B %d, %Y"
|
||||
|
||||
##################################################################################
|
||||
### The Efiction Base Adapter uses the Bulk story to retrieve the metadata, but
|
||||
### on this site, the Rating is not present in the Bulk page...
|
||||
### so it is not retrieved.
|
||||
##################################################################################
|
||||
|
||||
def getClass():
|
||||
return LooseLugsComAdapter
|
||||
347
fanficfare/adapters/adapter_lotrgficcom.py
Normal file
347
fanficfare/adapters/adapter_lotrgficcom.py
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
##############################################################################
|
||||
### Adapted by GComyn
|
||||
### Completed on November, 22, 2016
|
||||
##############################################################################
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
class LOTRgficComAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.story.setMetadata('siteabbrev','lotrgfic')
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'www.lotrgfic.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
addurl = "&warning=3"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
elif "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
### Main Content for the Table Of Contents page.
|
||||
div = soup.find('div',{'id':'maincontent'})
|
||||
|
||||
divfooter = div.find('div',{'id':'footer'})
|
||||
if divfooter != None:
|
||||
divfooter.extract()
|
||||
|
||||
## Title
|
||||
a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = div.find('a', href=re.compile(r"viewuser.php"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in div.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
### Metadata is contained
|
||||
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
### This site has the metadata formatted all over the place,
|
||||
### so we have to do some very cludgy programming to get it.
|
||||
### If someone can do it better, please do so, and let us know.
|
||||
## I'm going to leave this section in, so we can get those
|
||||
## elements that are "formatted correctly".
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## the summary is not encased in a span label... so we can't do anything here.
|
||||
## I'm going to leave it here just in case.
|
||||
## Everything until the next span class='label'
|
||||
svalue = ''
|
||||
while value and 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
# sometimes poorly formated desc (<p> w/o </p>) leads
|
||||
# to all labels being included.
|
||||
svalue=svalue[:svalue.find('<span class="label">')]
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
warningstext = [warning.string for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Places' in label:
|
||||
places = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
placestext = [place.string for place in places]
|
||||
self.warning = ', '.join(placestext)
|
||||
for place in placestext:
|
||||
self.story.addToList('places',place.string)
|
||||
|
||||
if 'Times' in label:
|
||||
times = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
||||
timestext = [time.string for time in times]
|
||||
self.warning = ', '.join(timestext)
|
||||
for time in timestext:
|
||||
self.story.addToList('times',time.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
## Now we are going to cludge together the rest of the metadata
|
||||
metad = soup.findAll('p',{'class':'smaller'})
|
||||
## Categories don't have a proper label, but do use links, so...
|
||||
cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
if cat != None:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
## Characters don't have a proper label, but do use links, so...
|
||||
chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
if char != None:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
### Rating is not enclosed in a label, only in a p tag classed 'smaller' so...
|
||||
ratng = metad[0].find('strong').get_text().replace('Rated','').strip()
|
||||
self.story.setMetadata('rating', ratng)
|
||||
|
||||
## No we try to get the summary... it's not within it's own
|
||||
## dedicated tag, so we have to split some hairs..
|
||||
## This may not work every time... but I tested it with 6 stories...
|
||||
mdata = metad[0]
|
||||
while '<hr/>' not in unicode(mdata.nextSibling):
|
||||
mdata = mdata.nextSibling
|
||||
self.setDescription(url,mdata.previousSibling.previousSibling.get_text())
|
||||
|
||||
### the rest of the metadata are not in tags at all... so we have to be really cludgy.
|
||||
## we don't need the rest of them, so we get rid of all but the last one
|
||||
metad = metad[-1]
|
||||
## we also don't need any of the links in here, so we'll get rid of them as well.
|
||||
links = metad.findAll('a')
|
||||
for link in links:
|
||||
link.extract()
|
||||
## and we've already done the labels, so let's remove them
|
||||
labels = metad.findAll('span',{'class':'label'})
|
||||
for label in labels:
|
||||
label.extract()
|
||||
## now we should only have text and <br>'s... somthing like this:
|
||||
#<p class="smaller">Categories:
|
||||
#<br/>
|
||||
#Characters: , , ,
|
||||
#<br/>
|
||||
# , <br/> <br/> <br/> None<br/>
|
||||
#Challenges: None
|
||||
#<br/>
|
||||
#Series: None
|
||||
#<br/>
|
||||
#Chapters: 1    |    Word count: 200    |    Read Count: 767
|
||||
#<br/>
|
||||
#Completed: Yes    |    Updated: 04/27/13    |    Published: 04/27/13
|
||||
#<br/>
|
||||
#</p>
|
||||
## we'll have to remove the non-breaking spaces to get this to work.
|
||||
metad = unicode(metad).replace(u"\xa0",'').replace('\n','')
|
||||
for txt in metad.split('<br/>'):
|
||||
if 'Challenges:' in txt:
|
||||
txt = txt.replace('Challenges:','').strip()
|
||||
self.story.setMetadata('challenges', txt)
|
||||
elif 'Series:' in txt:
|
||||
txt = txt.replace('Series:','').strip()
|
||||
self.story.setMetadata('challenges', txt)
|
||||
elif 'Chapters:' in txt:
|
||||
for txt2 in txt.split('|'):
|
||||
txt2 = txt2.replace('\n','').strip()
|
||||
if 'Word count:' in txt2:
|
||||
txt2 = txt2.replace('Word count:','').strip()
|
||||
self.story.setMetadata('numWords', value)
|
||||
elif 'Read Count:' in txt2:
|
||||
txt2= txt2.replace('Read Count:','').strip()
|
||||
self.story.setMetadata('readings', value)
|
||||
elif 'Completed:' in txt:
|
||||
for txt2 in txt.split('|'):
|
||||
txt2 = txt2.strip()
|
||||
if 'Completed:' in txt2:
|
||||
if 'Yes' in txt2:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
elif 'Updated:' in txt2:
|
||||
txt2= txt2.replace('Updated:','').strip()
|
||||
self.story.setMetadata('dateUpdated', makeDate(txt2.strip(), "%b/%d/%y"))
|
||||
elif 'Published:' in txt2:
|
||||
txt2= txt2.replace('Published:','').strip()
|
||||
self.story.setMetadata('datePublished', makeDate(txt2.strip(), "%b/%d/%y"))
|
||||
|
||||
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
data = self.get_request(url)
|
||||
# problems with some stories, but only in calibre. I suspect
|
||||
# issues with different SGML parsers in python. This is a
|
||||
# nasty hack, but it works.
|
||||
data = data[data.index("<body"):]
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
span = soup.find('div', {'id' : 'maincontent'})
|
||||
|
||||
# Everything is encased in the maincontent section, so we have
|
||||
# to remove as much as we can systematically
|
||||
tables = span.findAll('table')
|
||||
for table in tables:
|
||||
table.extract()
|
||||
|
||||
headings = span.findAll('h3')
|
||||
for heading in headings:
|
||||
heading.extract()
|
||||
|
||||
links = span.findAll('a')
|
||||
for link in links:
|
||||
link.extract()
|
||||
|
||||
forms = span.findAll('form')
|
||||
for form in forms:
|
||||
form.extract()
|
||||
|
||||
divs = span.findAll('div')
|
||||
for div in divs:
|
||||
div.extract()
|
||||
|
||||
if None == span:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,span)
|
||||
|
||||
def getClass():
|
||||
return LOTRgficComAdapter
|
||||
|
|
@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('rating', rating)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
|
||||
value = labels[0].previousSibling
|
||||
svalue = ""
|
||||
|
|
@ -154,22 +154,22 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('numWords', value.split(' -')[0])
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
|
|
@ -194,7 +194,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
|
|||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
|
|||
self.story.extendList('authorId', [authorId])
|
||||
self.story.extendList('authorUrl', [authorUrl])
|
||||
|
||||
if not self.story.getMetadataRaw('rating'):
|
||||
if not self.story.getMetadata('rating'):
|
||||
ratingTitle = chapter.getRatingTitle()
|
||||
if ratingTitle:
|
||||
self.story.setMetadata('rating', ratingTitle)
|
||||
|
|
@ -204,6 +204,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
|
|||
self.story.setMetadata('datePublished', datePublished)
|
||||
self.story.setMetadata('dateUpdated', dateUpdated)
|
||||
self.story.setMetadata('numWords', unicode(wordCount))
|
||||
self.story.setMetadata('numChapters', len(chapters))
|
||||
|
||||
# Site-specific metadata.
|
||||
self.story.setMetadata('language', self.SITE_LANGUAGE)
|
||||
|
|
@ -677,7 +678,7 @@ class Chapter(object):
|
|||
|
||||
def _excludeEditorSignature(self, root):
|
||||
"""Exclude editor signature from within `root' element."""
|
||||
for stringNode in root.find_all(string=True):
|
||||
for textNode in root.findAll(text=True):
|
||||
if re.match(self.SIGNED_PATTERN, textNode.string):
|
||||
editorLink = textNode.findNext('a')
|
||||
if editorLink:
|
||||
|
|
|
|||
|
|
@ -64,9 +64,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
|
|||
return "https://mcstories.com/StoryTitle/ https://mcstories.com/StoryTitle/index.html https://mcstories.com/StoryTitle/StoryTitle1.html"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
## Note that this uses a regular expression *negative*
|
||||
## lookahead--story URLs *can't* have /Titles/ /Authors/ etc.
|
||||
return r"https?://(www\.)?mcstories\.com(?!/(Titles|Authors|Tags|ReadersPicks)/)/[a-zA-Z0-9_-]+/"
|
||||
return r"https?://(www\.)?mcstories\.com/([a-zA-Z0-9_-]+)/"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
"""
|
||||
|
|
@ -85,7 +83,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
|
|||
data1 = self.get_request(self.url)
|
||||
soup1 = self.make_soup(data1)
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soup1.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
if 'Page Not Found.' in data1:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
|
|
@ -163,7 +161,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
|
|||
soup1 = self.make_soup(data1)
|
||||
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soup1.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
# get story text
|
||||
story1 = soup1.find('article', id='mcstories')
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%m.%d.%Y"
|
||||
self.dateformat = "%B %d, %Y %H:%M"
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
|
|
@ -146,24 +146,21 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
|||
for (atag,aurl,name) in [ (x,x['href'],stripHTML(x)) for x in chap_p.find_all('a') ]:
|
||||
self.add_chapter(name,'https://'+self.host+aurl)
|
||||
|
||||
|
||||
# category
|
||||
# <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
|
||||
for a in soup.find_all('a',href=re.compile(r"^/fanfic/a/")):
|
||||
for a in soup.findAll('a',href=re.compile(r"^/fanfic/a/")):
|
||||
self.story.addToList('category',a.string)
|
||||
|
||||
# genre
|
||||
# <a href="/fanfic/src.php/g/567">Ranma 1/2</a>
|
||||
for a in soup.find_all('a',href=re.compile(r"^/fanfic/src.php/g/")):
|
||||
for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")):
|
||||
self.story.addToList('genre',a.string)
|
||||
|
||||
metasoup = soup.find("div",{"class":"post-meta"})
|
||||
metastr = stripHTML(metasoup)
|
||||
metahtml = unicode(metasoup)
|
||||
|
||||
self.setDescription(url, metahtml[metahtml.index('</a><br/>')+9:metahtml.index('<br/><b>')])
|
||||
metastr = stripHTML(soup.find("div",{"class":"post-meta"}))
|
||||
|
||||
# Latest Revision: February 07, 2015 15:21 PST
|
||||
m = re.match(r".*?(?:Latest Revision|Uploaded On): ?(\d\d\.\d\d\.\d\d\d\d) ?",metastr)
|
||||
m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",metastr)
|
||||
if m:
|
||||
self.story.setMetadata('dateUpdated', makeDate(m.group(1), self.dateformat))
|
||||
# site doesn't give date published on index page.
|
||||
|
|
@ -171,20 +168,19 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
|||
# self.story.setMetadata('datePublished',
|
||||
# self.story.getMetadataRaw('dateUpdated'))
|
||||
|
||||
# Words:123 or 23.1K or 1.0M
|
||||
m = re.match(r".*?\| ?Words: ?([\.\d]+)(K|M|) ?\|",metastr)
|
||||
# Words: 123456
|
||||
m = re.match(r".*?\| Words: (\d+) \|",metastr)
|
||||
if m:
|
||||
if not m.group(2):
|
||||
word_factor = 1
|
||||
elif m.group(2) == 'K':
|
||||
word_factor = 1000
|
||||
elif m.group(2) == 'M':
|
||||
word_factor = 1000000
|
||||
num_words = int(float(m.group(1))*word_factor)
|
||||
self.story.setMetadata('numWords', num_words)
|
||||
self.story.setMetadata('numWords', m.group(1))
|
||||
|
||||
# Summary: ....
|
||||
m = re.match(r".*?Summary: (.*)$",metastr)
|
||||
if m:
|
||||
self.setDescription(url, m.group(1))
|
||||
#self.story.setMetadata('description', m.group(1))
|
||||
|
||||
# completed
|
||||
m = re.match(r".*?Status: ?Completed.*?",metastr)
|
||||
m = re.match(r".*?Status: Completed.*?",metastr)
|
||||
if m:
|
||||
self.story.setMetadata('status','Completed')
|
||||
else:
|
||||
|
|
@ -202,7 +198,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
|||
# print("data:%s"%data)
|
||||
headerstr = stripHTML(soup.find('div',{'class':'post-meta'}))
|
||||
|
||||
m = re.match(r".*?Uploaded On: ?(\d\d\.\d\d\.\d\d\d\d)",headerstr)
|
||||
m = re.match(r".*?Uploaded On: ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",headerstr)
|
||||
if m:
|
||||
date = makeDate(m.group(1), self.dateformat)
|
||||
if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
|
||||
|
|
|
|||
54
fanficfare/adapters/adapter_merengohu.py
Normal file
54
fanficfare/adapters/adapter_merengohu.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2022 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
from .base_efiction_adapter import BaseEfictionAdapter
|
||||
|
||||
class MerengoHuAdapter(BaseEfictionAdapter):
|
||||
|
||||
@classmethod
|
||||
def getProtocol(self):
|
||||
return "https"
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'merengo.hu'
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(self):
|
||||
return 'merengo'
|
||||
|
||||
@classmethod
|
||||
def getDateFormat(self):
|
||||
return "%Y.%m.%d"
|
||||
|
||||
@classmethod
|
||||
def getBacktoIndex(self):
|
||||
return 'Vissza az indexhez'
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
## merengo.hu has a custom 18 consent click through
|
||||
self.get_request(self.getUrlForPhp('tizennyolc.php')+'?consent=true')
|
||||
|
||||
## Call super of extractChapterUrlsAndMetadata().
|
||||
## base_efiction leaves the soup in self.html.
|
||||
return super(MerengoHuAdapter, self).extractChapterUrlsAndMetadata()
|
||||
|
||||
def getClass():
|
||||
return MerengoHuAdapter
|
||||
|
||||
272
fanficfare/adapters/adapter_merlinficdtwinscouk.py
Normal file
272
fanficfare/adapters/adapter_merlinficdtwinscouk.py
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return MerlinFicDtwinsCoUk
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class MerlinFicDtwinsCoUk(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','mrfd')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%b %d, %Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'merlinfic.dtwins.co.uk'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self.post_request(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=4"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self.get_request(url)
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Pairing' in label:
|
||||
ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for ship in ships:
|
||||
self.story.addToList('ships',ship.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
|
|
@ -154,7 +154,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.find_all('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
|
@ -170,7 +170,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.find_all('span',{'class':'label'})
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
|
@ -191,13 +191,13 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
catstext = [cat.string for cat in cats]
|
||||
for cat in catstext:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
charstext = [char.string for char in chars]
|
||||
for char in charstext:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
|
@ -206,7 +206,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
genrestext = [genre.string for genre in genres]
|
||||
self.genre = ', '.join(genrestext)
|
||||
for genre in genrestext:
|
||||
|
|
@ -216,7 +216,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
## leaving it in. Check to make sure the type_id number
|
||||
## is correct, though--it's site specific.
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.find_all('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
|
||||
warningstext = [warning.string for warning in warnings]
|
||||
self.warning = ', '.join(warningstext)
|
||||
for warning in warningstext:
|
||||
|
|
@ -243,7 +243,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
|
|||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
storyas = seriessoup.find_all('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ class NovelFullSiteAdapter(BaseSiteAdapter):
|
|||
|
||||
self.story.setMetadata("title", soup.select_one("h3.title").text)
|
||||
|
||||
for author in soup.find("h3", string="Author:").fetchNextSiblings(
|
||||
for author in soup.find("h3", text="Author:").fetchNextSiblings(
|
||||
"a", href=re.compile("/author/")
|
||||
):
|
||||
self.story.addToList("authorId", author.text)
|
||||
|
|
@ -91,7 +91,7 @@ class NovelFullSiteAdapter(BaseSiteAdapter):
|
|||
content = soup.find(id="chapter-content")
|
||||
|
||||
# Remove chapter header if present
|
||||
chapter_header = content.find(["p", "h3"], string=re.compile(r"Chapter \d+:"))
|
||||
chapter_header = content.find(["p", "h3"], text=re.compile(r"Chapter \d+:"))
|
||||
|
||||
if chapter_header:
|
||||
chapter_header.decompose()
|
||||
|
|
|
|||
|
|
@ -189,13 +189,13 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter):
|
|||
"Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
# Some comments we will get is invalid. Remove them all.
|
||||
[comment.extract() for comment in story.find_all(string=lambda text:isinstance(text, Comment))]
|
||||
[comment.extract() for comment in story.find_all(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
# We don't need links. They have a bad css and they are not working most of times.
|
||||
[a.extract() for a in story.find_all('a')]
|
||||
|
||||
# Some tags have non-standard tag name.
|
||||
for tag in story.find_all(recursive=True):
|
||||
for tag in story.findAll(recursive=True):
|
||||
if tag.name not in HTML_TAGS:
|
||||
tag.name = 'span'
|
||||
|
||||
|
|
|
|||
149
fanficfare/adapters/adapter_noveltrovecom.py
Normal file
149
fanficfare/adapters/adapter_noveltrovecom.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
####################################################################################################
|
||||
## Adapted by GComyn on April 22, 2017
|
||||
####################################################################################################
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
import re
|
||||
import sys # ## used for debug purposes
|
||||
|
||||
# py2 vs py3 transition
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
from .. import exceptions as exceptions
|
||||
from ..htmlcleanup import stripHTML
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
####################################################################################################
|
||||
def getClass():
|
||||
return NovelTroveComSiteAdapter
|
||||
|
||||
|
||||
####################################################################################################
|
||||
class NovelTroveComSiteAdapter(BaseSiteAdapter):
|
||||
''' This is a site with 1 story per page, so no multiple chapter stories
|
||||
The date is listed (on the newer stories) as a month and a year, so I'll be adding that
|
||||
to the summary, instead of trying to transform it to a date. '''
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult = False
|
||||
|
||||
# get storyId from url
|
||||
# https://noveltrove.com/story/983/put-that-big-cock-in-me
|
||||
self.story.setMetadata('storyId', self.parsedUrl.path.split('/')[2] + '_' + self.parsedUrl.path.split('/')[3])
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','ntcom')
|
||||
|
||||
# This is a 1 story/page site, so we will initialize the variable to keep the soup
|
||||
self.html = ''
|
||||
self.endindex = []
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d %b. '%y"
|
||||
|
||||
####################################################################################################
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'noveltrove.com'
|
||||
|
||||
####################################################################################################
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/story/12345/astoryname"
|
||||
|
||||
####################################################################################################
|
||||
def getSiteURLPattern(self):
|
||||
return r"https://"+re.escape(self.getSiteDomain())+r"/story/([0-9])+/*(?P<id>[^/]+)"
|
||||
|
||||
####################################################################################################
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def doExtractChapterUrlsAndMetadata(self, get_cover=True):
|
||||
|
||||
url = self.url
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
soup = self.make_soup(data)
|
||||
|
||||
# Now go hunting for all the meta data we can get
|
||||
metablock = soup.find('div', {'class': 'title-infos'})
|
||||
|
||||
## Getting Title
|
||||
title = stripHTML(metablock.find('h1'))
|
||||
self.story.setMetadata('title', title)
|
||||
|
||||
## Getting author
|
||||
author = metablock.find('a', {'class':'author'})
|
||||
self.story.setMetadata('authorId',author['href'].split('/')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+author['href'])
|
||||
self.story.setMetadata('author',author.string)
|
||||
|
||||
## Get the categories
|
||||
for tag in metablock.find_all('a', {'class':'story-category'}):
|
||||
self.story.addToList('category',stripHTML(tag))
|
||||
|
||||
## There is no summary for these stories, so I'm going to take the first
|
||||
## 250 characters.
|
||||
synopsis = ''
|
||||
pcount = 0
|
||||
for para in soup.find('div', {'class':'body'}).find_all('p'):
|
||||
synopsis += para.get_text() + ' '
|
||||
pcount += 1
|
||||
if pcount > 10:
|
||||
break
|
||||
|
||||
synopsis = synopsis.strip()[:250] + '...'
|
||||
|
||||
self.setDescription(url, synopsis)
|
||||
|
||||
## Since this is a 1 story/page site, the published and updated dates are the same.
|
||||
dateposted = stripHTML(metablock.find('div', {'class':'date'}))
|
||||
self.story.setMetadata('datePublished', makeDate(dateposted, self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(dateposted, self.dateformat))
|
||||
|
||||
## This is a 1 story/page site, so we'll keep the soup for the getChapterText function
|
||||
## the chapterUrl and numChapters need to be set as well
|
||||
self.html = soup
|
||||
self.add_chapter(self.story.getMetadata('title'), url)
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
|
||||
## Getting the non-standard title page entries
|
||||
copyrt = soup.find('div', {'class':'copyright'}).get_text()
|
||||
self.story.setMetadata('copyright', copyrt)
|
||||
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
logger.debug('Using data that we got from: %s' % url)
|
||||
|
||||
soup = self.html
|
||||
story = soup.find('div', {'class':'body'})
|
||||
|
||||
if story == None:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,story)
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2024 FanFicFare team
|
||||
# Copyright 2020 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -15,24 +15,34 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from .base_otw_adapter import BaseOTWAdapter
|
||||
from .adapter_wuxiaworldco import WuxiaWorldCoSiteAdapter
|
||||
|
||||
def getClass():
|
||||
return SuperloveAdapter
|
||||
return NovelUpdatesCcSiteAdapter
|
||||
|
||||
class SuperloveAdapter(BaseOTWAdapter):
|
||||
class NovelUpdatesCcSiteAdapter(WuxiaWorldCoSiteAdapter):
|
||||
DATE_FORMAT = '%Y-%m-%d %H:%M'
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseOTWAdapter.__init__(self, config, url)
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','sluv')
|
||||
WuxiaWorldCoSiteAdapter.__init__(self, config, url)
|
||||
self.story.setMetadata('siteabbrev', 'nucc')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'superlove.sayitditto.net'
|
||||
return 'www.novelupdates.cc'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.novelupdates.cc','m.novelupdates.cc']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return 'https://%s/story-name' % cls.getSiteDomain()
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r'https?://(www|m)\.novelupdates\.cc/(?P<id>[^/]+)(/)?'
|
||||
|
|
@ -137,14 +137,14 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
try:
|
||||
# in case link points somewhere other than the first chapter
|
||||
a = soup.find_all('option')[1]['value']
|
||||
a = soup.findAll('option')[1]['value']
|
||||
self.story.setMetadata('storyId',a.split('=',)[1])
|
||||
url = 'http://'+self.host+'/'+a
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
except:
|
||||
pass
|
||||
|
||||
for info in asoup.find_all('table', {'class' : 'border'}):
|
||||
for info in asoup.findAll('table', {'class' : 'border'}):
|
||||
a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
if a != None:
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
|
@ -152,7 +152,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
|
||||
if len(chapters) == 0:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
else:
|
||||
|
|
@ -171,7 +171,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
except:
|
||||
return ""
|
||||
|
||||
cats = info.find_all('a',href=re.compile('categories.php'))
|
||||
cats = info.findAll('a',href=re.compile('categories.php'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
|
|
@ -188,7 +188,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
|
|||
self.setDescription(url,svalue)
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = info.find_all('b')
|
||||
labels = info.findAll('b')
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
|
|
|||
|
|
@ -93,26 +93,26 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
|
|||
chapters = soup.find('select')
|
||||
if chapters == None:
|
||||
self.add_chapter(self.story.getMetadata('title'),url)
|
||||
for b in soup.find_all('b'):
|
||||
for b in soup.findAll('b'):
|
||||
if b.text == "Updated":
|
||||
date = b.nextSibling.string.split(': ')[1].split(',')
|
||||
self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
|
||||
else:
|
||||
i = 0
|
||||
chapters = chapters.find_all('option')
|
||||
chapters = chapters.findAll('option')
|
||||
for chapter in chapters:
|
||||
self.add_chapter(chapter,'https://'+self.host+chapter['value'])
|
||||
if i == 0:
|
||||
self.story.setMetadata('storyId',chapter['value'].split('/')[3])
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).find_all('b')
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).findAll('b')
|
||||
for b in head:
|
||||
if b.text == "Updated":
|
||||
date = b.nextSibling.string.split(': ')[1].split(',')
|
||||
self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
|
||||
|
||||
if i == (len(chapters)-1):
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).find_all('b')
|
||||
head = self.make_soup(self.get_request('https://'+self.host+chapter['value'])).findAll('b')
|
||||
for b in head:
|
||||
if b.text == "Updated":
|
||||
date = b.nextSibling.string.split(': ')[1].split(',')
|
||||
|
|
@ -160,20 +160,20 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
|
|||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
chapter=self.make_soup('<div class="story"></div>')
|
||||
for p in soup.find_all(['p','blockquote']):
|
||||
for p in soup.findAll(['p','blockquote']):
|
||||
if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p):
|
||||
break
|
||||
chapter.append(p)
|
||||
|
||||
for a in chapter.find_all('div'):
|
||||
for a in chapter.findAll('div'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('table'):
|
||||
for a in chapter.findAll('table'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('script'):
|
||||
for a in chapter.findAll('script'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('form'):
|
||||
for a in chapter.findAll('form'):
|
||||
a.extract()
|
||||
for a in chapter.find_all('textarea'):
|
||||
for a in chapter.findAll('textarea'):
|
||||
a.extract()
|
||||
|
||||
|
||||
|
|
|
|||
241
fanficfare/adapters/adapter_ponyfictionarchivenet.py
Normal file
241
fanficfare/adapters/adapter_ponyfictionarchivenet.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
# py2 vs py3 transition
|
||||
from ..six import text_type as unicode
|
||||
|
||||
from .base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return PonyFictionArchiveNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
# normalized story URL.
|
||||
if "explicit" in self.parsedUrl.netloc:
|
||||
self._setURL('https://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self.dateformat = "%d/%b/%y"
|
||||
else:
|
||||
self._setURL('https://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self.dateformat = "%d %b %Y"
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','pffa')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'ponyfictionarchive.net'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "https://"+cls.getSiteDomain()+"/viewstory.php?sid=1234 https://explicit."+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"https?://(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&warning=9"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
data = self.get_request(url)
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
|
||||
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
if status: # apparently this site can have stories with neither In-Progress or Complete.
|
||||
self.story.setMetadata('status',status.string)
|
||||
|
||||
try:
|
||||
# explicit.site and .site have some differences now...
|
||||
section = soup.findAll('span', {'class' : 'General'})[1]
|
||||
self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
|
||||
|
||||
value = section.nextSibling
|
||||
svalue = ""
|
||||
while 'label' not in defaultGetattr(value,'class'):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
except:
|
||||
# find rating in data
|
||||
# <br /> • Mature • <br />
|
||||
lead = "<br /> • "
|
||||
trail = " • <br />"
|
||||
rating = data[data.index(lead)+len(lead):data.index(trail)]
|
||||
if len(rating)<20: # minor sanity check.
|
||||
self.story.setMetadata('rating',rating)
|
||||
descstr = data[data.index(trail)+len(trail):] # from desc on
|
||||
descstr = descstr[:descstr.index('<span class="label">')] # remove after desc.
|
||||
self.setDescription(url,descstr)
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'https://'+self.host+'/'+a['href']
|
||||
|
||||
seriessoup = self.make_soup(self.get_request(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self.get_request(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue