Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2026-05-09 05:21:13 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #325 from JimmXinu/python3

Browse source 
Python2/3 Dual Code
This commit is contained in:
Jim Miller 2018-08-10 13:45:47 -05:00 committed by GitHub
commit 85f28d1054
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
220 changed files with 3990 additions and 1732 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -15,6 +15,7 @@
# usually perl -pi.back -e edits.
*.back
*.bak
cleanup.sh
FanFictionDownLoader.zip

2
calibre-plugin/__init__.py
View file

@ -33,7 +33,7 @@ except NameError:
from calibre.customize import InterfaceActionBase
# pulled out from FanFicFareBase for saving in prefs.py
__version__ = (2, 28, 0)
__version__ = (2, 37, 3)
## Apparently the name for this class doesn't matter--it was still
## 'demo' for the first few versions.

52
calibre-plugin/plugin-defaults.ini
View file

@ -1,4 +1,4 @@
# Copyright 2015 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2015 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -1916,31 +1916,6 @@ comments_label:Comments
include_in_category:category,searchtags
[royalroadl.com]
extra_valid_entries:stars
#add_to_extra_titlepage_entries:,stars
## some sites include images that we don't ever want becoming the
## cover image. This lets you exclude them.
cover_exclusion_regexp:(imgur.com/dzOACJf.png|/forum/images/smilies/)
## Clear FanFiction from defaults, site has fanfictions and original fiction.
extratags:
## royalroadl.com stories sometimes have 'spoiler' blocks in
## posts. When viewed in a browser, the block is hidden until a button
## is clicked. eBook viewers can't handle that and the javascript is
## disabled. The remove_spoilers option, if uncommented, will remove
## spoiler blocks entirely.
#remove_spoilers:true
## This option if uncommented, will put a box around the spoiler
## blocks with the original spoiler button text as a label using
## fieldset and legend HTML tags. For a simple box, see the
## add_to_output_css example for [base_xenforoforum:epub].
#legend_spoilers:true
[samandjack.net]
## Some sites require login (or login for some rated stories) The
## program can prompt you, or you can save it in config. In
@ -2841,6 +2816,31 @@ extracategories:Queer as Folk
website_encodings:Windows-1252,utf8
[www.royalroad.com]
extra_valid_entries:stars
#add_to_extra_titlepage_entries:,stars
## some sites include images that we don't ever want becoming the
## cover image. This lets you exclude them.
cover_exclusion_regexp:(imgur.com/dzOACJf.png|/forum/images/smilies/)
## Clear FanFiction from defaults, site has fanfictions and original fiction.
extratags:
## royalroad.com stories sometimes have 'spoiler' blocks in
## posts. When viewed in a browser, the block is hidden until a button
## is clicked. eBook viewers can't handle that and the javascript is
## disabled. The remove_spoilers option, if uncommented, will remove
## spoiler blocks entirely.
#remove_spoilers:true
## This option if uncommented, will put a box around the spoiler
## blocks with the original spoiler button text as a label using
## fieldset and legend HTML tags. For a simple box, see the
## add_to_output_css example for [base_xenforoforum:epub].
#legend_spoilers:true
[www.scarvesandcoffee.net]
## Some sites do not require a login, but do require the user to
## confirm they are adult for adult content. In commandline version,

21
fanficfare/HtmlTagStack.py
View file

@ -1,4 +1,21 @@
# coding: utf-8
# -*- coding: utf-8 -*-
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import absolute_import
import re
import codecs
@ -54,4 +71,4 @@ def flush():
del stack[:]
def get_stack():
return stack
return stack

3
fanficfare/__init__.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Fanficdownloader team, 2016 FanFicFare team
# Copyright 2015 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import absolute_import
try:
# just a way to switch between web service and CLI/PI

319
fanficfare/adapters/__init__.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,168 +15,173 @@
# limitations under the License.
#
from __future__ import absolute_import
import os, re, sys, glob, types
from os.path import dirname, basename, normpath
import logging
import urlparse as up
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.parse import urlparse
logger = logging.getLogger(__name__)
from .. import exceptions as exceptions
from ..configurable import Configuration
from .. import configurable as configurable
## must import each adapter here.
import adapter_test1
import adapter_fanfictionnet
import adapter_fanficcastletvnet
import adapter_fictionalleyorg
import adapter_fictionpresscom
import adapter_ficwadcom
import adapter_fimfictionnet
import adapter_mediaminerorg
import adapter_potionsandsnitches
import adapter_tenhawkpresentscom
import adapter_adastrafanficcom
import adapter_tthfanficorg
import adapter_twilightednet
import adapter_whoficcom
import adapter_siyecouk
import adapter_archiveofourownorg
import adapter_ficbooknet
import adapter_nfacommunitycom
import adapter_midnightwhispers
import adapter_ksarchivecom
import adapter_archiveskyehawkecom
import adapter_squidgeorgpeja
import adapter_libraryofmoriacom
import adapter_wraithbaitcom
import adapter_dramioneorg
import adapter_ashwindersycophanthexcom
import adapter_chaossycophanthexcom
import adapter_erosnsapphosycophanthexcom
import adapter_lumossycophanthexcom
import adapter_occlumencysycophanthexcom
import adapter_phoenixsongnet
import adapter_walkingtheplankorg
import adapter_dokugacom
import adapter_iketernalnet
import adapter_storiesofardacom
import adapter_destinysgatewaycom
import adapter_ncisfictioncom
import adapter_fanfiktionde
import adapter_ponyfictionarchivenet
import adapter_ncisficcom
import adapter_nationallibrarynet
import adapter_themasquenet
import adapter_pretendercentrecom
import adapter_darksolaceorg
import adapter_finestoriescom
import adapter_hpfanficarchivecom
import adapter_twilightarchivescom
import adapter_nhamagicalworldsus
import adapter_hlfictionnet
import adapter_dracoandginnycom
import adapter_scarvesandcoffeenet
import adapter_thepetulantpoetesscom
import adapter_wolverineandroguecom
import adapter_merlinficdtwinscouk
import adapter_thehookupzonenet
import adapter_bloodtiesfancom
import adapter_qafficcom
import adapter_efpfanficnet
import adapter_potterficscom
import adapter_efictionestelielde
import adapter_imagineeficcom
import adapter_asr3slashzoneorg
import adapter_potterheadsanonymouscom
import adapter_fictionpadcom
import adapter_storiesonlinenet
import adapter_trekiverseorg
import adapter_literotica
import adapter_voracity2eficcom
import adapter_spikeluvercom
import adapter_bloodshedversecom
import adapter_nocturnallightnet
import adapter_fanfichu
import adapter_fictionmaniatv
import adapter_tolkienfanfiction
import adapter_themaplebookshelf
import adapter_fannation
import adapter_sheppardweircom
import adapter_samandjacknet
import adapter_csiforensicscom
import adapter_lotrfanfictioncom
import adapter_fhsarchivecom
import adapter_fanfictionjunkiesde
import adapter_tgstorytimecom
import adapter_itcouldhappennet
import adapter_forumsspacebattlescom
import adapter_forumssufficientvelocitycom
import adapter_forumquestionablequestingcom
import adapter_ninelivesarchivecom
import adapter_masseffect2in
import adapter_quotevcom
import adapter_mcstoriescom
import adapter_buffygilescom
import adapter_andromedawebcom
import adapter_artemisfowlcom
import adapter_naiceanilmenet
import adapter_deepinmysoulnet
import adapter_kiarepositorymujajinet
import adapter_adultfanfictionorg
import adapter_fictionhuntcom
import adapter_royalroadl
import adapter_chosentwofanficcom
import adapter_bdsmlibrarycom
import adapter_asexstoriescom
import adapter_gluttonyfictioncom
import adapter_valentchambercom
import adapter_looselugscom
import adapter_wwwgiantessworldnet
import adapter_lotrgficcom
import adapter_tomparisdormcom
import adapter_writingwhimsicalwanderingsnet
import adapter_sugarquillnet
import adapter_wwwarea52hkhnet
import adapter_starslibrarynet
import adapter_fanficauthorsnet
import adapter_fireflyfansnet
import adapter_fireflypopulliorg
import adapter_sebklainenet
import adapter_shriftweborgbfa
import adapter_trekfanfictionnet
import adapter_wuxiaworldcom
import adapter_wwwlushstoriescom
import adapter_wwwutopiastoriescom
import adapter_sinfuldreamscomunicornfic
import adapter_sinfuldreamscomwhisperedmuse
import adapter_sinfuldreamscomwickedtemptation
import adapter_asianfanficscom
import adapter_webnovelcom
import adapter_deandamagecom
import adapter_imrightbehindyoucom
import adapter_mttjustoncenet
import adapter_narutoficorg
import adapter_starskyhutcharchivenet
import adapter_swordborderlineangelcom
import adapter_tasteofpoisoninkubationnet
import adapter_thebrokenworldorg
import adapter_thedelphicexpansecom
import adapter_thundercatsfansorg
import adapter_unknowableroomorg
import adapter_www13hoursorg
import adapter_wwwaneroticstorycom
import adapter_gravitytalescom
import adapter_lcfanficcom
import adapter_noveltrovecom
import adapter_inkbunnynet
import adapter_alternatehistorycom
import adapter_wattpadcom
import adapter_lightnovelgatecom
import adapter_wwwnovelallcom
import adapter_wuxiaworldco
import adapter_harrypotterfanfictioncom
from . import base_efiction_adapter
from . import adapter_test1
from . import adapter_fanfictionnet
from . import adapter_fanficcastletvnet
from . import adapter_fictionalleyorg
from . import adapter_fictionpresscom
from . import adapter_ficwadcom
from . import adapter_fimfictionnet
from . import adapter_mediaminerorg
from . import adapter_potionsandsnitches
from . import adapter_tenhawkpresentscom
from . import adapter_adastrafanficcom
from . import adapter_tthfanficorg
from . import adapter_twilightednet
from . import adapter_whoficcom
from . import adapter_siyecouk
from . import adapter_archiveofourownorg
from . import adapter_ficbooknet
from . import adapter_nfacommunitycom
from . import adapter_midnightwhispers
from . import adapter_ksarchivecom
from . import adapter_archiveskyehawkecom
from . import adapter_squidgeorgpeja
from . import adapter_libraryofmoriacom
from . import adapter_wraithbaitcom
from . import adapter_dramioneorg
from . import adapter_ashwindersycophanthexcom
from . import adapter_chaossycophanthexcom
from . import adapter_erosnsapphosycophanthexcom
from . import adapter_lumossycophanthexcom
from . import adapter_occlumencysycophanthexcom
from . import adapter_phoenixsongnet
from . import adapter_walkingtheplankorg
from . import adapter_dokugacom
from . import adapter_iketernalnet
from . import adapter_storiesofardacom
from . import adapter_destinysgatewaycom
from . import adapter_ncisfictioncom
from . import adapter_fanfiktionde
from . import adapter_ponyfictionarchivenet
from . import adapter_ncisficcom
from . import adapter_nationallibrarynet
from . import adapter_themasquenet
from . import adapter_pretendercentrecom
from . import adapter_darksolaceorg
from . import adapter_finestoriescom
from . import adapter_hpfanficarchivecom
from . import adapter_twilightarchivescom
from . import adapter_nhamagicalworldsus
from . import adapter_hlfictionnet
from . import adapter_dracoandginnycom
from . import adapter_scarvesandcoffeenet
from . import adapter_thepetulantpoetesscom
from . import adapter_wolverineandroguecom
from . import adapter_merlinficdtwinscouk
from . import adapter_thehookupzonenet
from . import adapter_bloodtiesfancom
from . import adapter_qafficcom
from . import adapter_efpfanficnet
from . import adapter_potterficscom
from . import adapter_efictionestelielde
from . import adapter_imagineeficcom
from . import adapter_asr3slashzoneorg
from . import adapter_potterheadsanonymouscom
from . import adapter_fictionpadcom
from . import adapter_storiesonlinenet
from . import adapter_trekiverseorg
from . import adapter_literotica
from . import adapter_voracity2eficcom
from . import adapter_spikeluvercom
from . import adapter_bloodshedversecom
from . import adapter_nocturnallightnet
from . import adapter_fanfichu
from . import adapter_fictionmaniatv
from . import adapter_tolkienfanfiction
from . import adapter_themaplebookshelf
from . import adapter_fannation
from . import adapter_sheppardweircom
from . import adapter_samandjacknet
from . import adapter_csiforensicscom
from . import adapter_lotrfanfictioncom
from . import adapter_fhsarchivecom
from . import adapter_fanfictionjunkiesde
from . import adapter_tgstorytimecom
from . import adapter_itcouldhappennet
from . import adapter_forumsspacebattlescom
from . import adapter_forumssufficientvelocitycom
from . import adapter_forumquestionablequestingcom
from . import adapter_ninelivesarchivecom
from . import adapter_masseffect2in
from . import adapter_quotevcom
from . import adapter_mcstoriescom
from . import adapter_buffygilescom
from . import adapter_andromedawebcom
from . import adapter_artemisfowlcom
from . import adapter_naiceanilmenet
from . import adapter_deepinmysoulnet
from . import adapter_kiarepositorymujajinet
from . import adapter_adultfanfictionorg
from . import adapter_fictionhuntcom
from . import adapter_royalroadl
from . import adapter_chosentwofanficcom
from . import adapter_bdsmlibrarycom
from . import adapter_asexstoriescom
from . import adapter_gluttonyfictioncom
from . import adapter_valentchambercom
from . import adapter_looselugscom
from . import adapter_wwwgiantessworldnet
from . import adapter_lotrgficcom
from . import adapter_tomparisdormcom
from . import adapter_writingwhimsicalwanderingsnet
from . import adapter_sugarquillnet
from . import adapter_wwwarea52hkhnet
from . import adapter_starslibrarynet
from . import adapter_fanficauthorsnet
from . import adapter_fireflyfansnet
from . import adapter_fireflypopulliorg
from . import adapter_sebklainenet
from . import adapter_shriftweborgbfa
from . import adapter_trekfanfictionnet
from . import adapter_wuxiaworldcom
from . import adapter_wwwlushstoriescom
from . import adapter_wwwutopiastoriescom
from . import adapter_sinfuldreamscomunicornfic
from . import adapter_sinfuldreamscomwhisperedmuse
from . import adapter_sinfuldreamscomwickedtemptation
from . import adapter_asianfanficscom
from . import adapter_webnovelcom
from . import adapter_deandamagecom
from . import adapter_imrightbehindyoucom
from . import adapter_mttjustoncenet
from . import adapter_narutoficorg
from . import adapter_starskyhutcharchivenet
from . import adapter_swordborderlineangelcom
from . import adapter_tasteofpoisoninkubationnet
from . import adapter_thebrokenworldorg
from . import adapter_thedelphicexpansecom
from . import adapter_thundercatsfansorg
from . import adapter_unknowableroomorg
from . import adapter_www13hoursorg
from . import adapter_wwwaneroticstorycom
from . import adapter_gravitytalescom
from . import adapter_lcfanficcom
from . import adapter_noveltrovecom
from . import adapter_inkbunnynet
from . import adapter_alternatehistorycom
from . import adapter_wattpadcom
from . import adapter_lightnovelgatecom
from . import adapter_wwwnovelallcom
from . import adapter_wuxiaworldco
from . import adapter_harrypotterfanfictioncom
## This bit of complexity allows adapters to be added by just adding
## importing. It eliminates the long if/else clauses we used to need
@ -187,9 +192,11 @@ __class_list = []
__domain_map = {}
def imports():
out = []
for name, val in globals().items():
if isinstance(val, types.ModuleType):
yield val.__name__
out.append(val.__name__)
return out
for x in imports():
if "fanficfare.adapters.adapter_" in x:
@ -223,7 +230,7 @@ def getNormalStoryURL(url):
def getNormalStoryURLSite(url):
# print("getNormalStoryURLSite:%s"%url)
if not getNormalStoryURL.__dummyconfig:
getNormalStoryURL.__dummyconfig = Configuration(["test1.com"],"EPUB",lightweight=True)
getNormalStoryURL.__dummyconfig = configurable.Configuration(["test1.com"],"EPUB",lightweight=True)
# pulling up an adapter is pretty low over-head. If
# it fails, it's a bad url.
try:
@ -297,7 +304,7 @@ def _get_class_for(url):
if not "#post-" in fixedurl:
fixedurl = re.sub(r"#.*$","",fixedurl)
parsedUrl = up.urlparse(fixedurl)
parsedUrl = urlparse(fixedurl)
domain = parsedUrl.netloc.lower()
if( domain != parsedUrl.netloc ):
fixedurl = fixedurl.replace(parsedUrl.netloc,domain)

15
fanficfare/adapters/adapter_adastrafanficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,20 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six import string_types as basestring
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
@ -73,7 +76,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

23
fanficfare/adapters/adapter_adultfanfictionorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# -- coding: utf-8 --
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -17,19 +17,22 @@
################################################################################
### Written by GComyn
################################################################################
from __future__ import absolute_import
from __future__ import unicode_literals
import time
import logging
logger = logging.getLogger(__name__)
import re
import sys
import urllib2
from bs4 import UnicodeDammit
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
################################################################################
@ -199,7 +202,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist("Code: 404. {0}".format(url))
elif e.code == 410:
@ -232,7 +235,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
# Find the chapters:
chapters = soup.find('div',{'class':'dropdown-content'})
for i, chapter in enumerate(chapters.findAll('a')):
self.add_chapter(chapter,self.url+'&chapter='+str(i+1))
self.add_chapter(chapter,self.url+'&chapter='+unicode(i+1))
# Find authorid and URL from... author url.
@ -265,7 +268,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
logger.debug('Getting the author page: {0}'.format(author_Url))
try:
adata = self._fetchUrl(author_Url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code in 404:
raise exceptions.StoryDoesNotExist("Author Page: Code: 404. {0}".format(author_Url))
elif e.code == 410:
@ -299,11 +302,11 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
while i == 0:
##We already have the first page, so if this is the first time through, skip getting the page
if page != 1:
author_Url = '{0}&view=story&zone={1}&page={2}'.format(self.story.getMetadata('authorUrl'), self.zone, str(page))
author_Url = '{0}&view=story&zone={1}&page={2}'.format(self.story.getMetadata('authorUrl'), self.zone, unicode(page))
logger.debug('Getting the author page: {0}'.format(author_Url))
try:
adata = self._fetchUrl(author_Url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code in 404:
raise exceptions.StoryDoesNotExist("Author Page: Code: 404. {0}".format(author_Url))
elif e.code == 410:
@ -334,7 +337,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
##There is also a double <br/>, so we have to fix that, then remove the leading and trailing '-:-'.
##They are always in the same order.
## EDIT 09/26/2016: Had some trouble with unicode errors... so I had to put in the decode/encode parts to fix it
liMetadata = str(lc2).decode('utf-8').replace('\n','').replace('\r','').replace('\t',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ')
liMetadata = unicode(lc2).replace('\n','').replace('\r','').replace('\t',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ')
liMetadata = stripHTML(liMetadata.replace(r'<br/>','-:-').replace('<!-- <br /-->','-:-'))
liMetadata = liMetadata.strip('-:-').strip('-:-').encode('utf-8')
for i, value in enumerate(liMetadata.decode('utf-8').split('-:-')):

5
fanficfare/adapters/adapter_alternatehistorycom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,7 +15,8 @@
# limitations under the License.
#
from adapter_forumquestionablequestingcom import QuestionablequestingComAdapter
from __future__ import absolute_import
from .adapter_forumquestionablequestingcom import QuestionablequestingComAdapter
def getClass():
return WWWAlternatehistoryComAdapter

17
fanficfare/adapters/adapter_andromedawebcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,17 +18,18 @@
# ####### webpage.
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return AndromedaWebComAdapter # XXX
@ -125,7 +126,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -159,7 +160,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

13
fanficfare/adapters/adapter_archiveofourownorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,20 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import json
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ArchiveOfOurOwnOrgAdapter
@ -155,7 +158,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
raise exceptions.AdultCheckRequired(self.url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_archiveskyehawkecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,18 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
@ -80,7 +81,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_artemisfowlcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,17 +18,18 @@
# ####### webpage.
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ArtemisFowlComAdapter # XXX
@ -125,7 +126,7 @@ class ArtemisFowlComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -159,7 +160,7 @@ class ArtemisFowlComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

18
fanficfare/adapters/adapter_asexstoriescom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,13 +15,10 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import urlparse
import time
import os
from bs4.element import Comment
@ -29,7 +26,12 @@ from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
import sys
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ASexStoriesComAdapter
@ -84,7 +86,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
soup1 = self.make_soup(data1)
#strip comments from soup
[comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -133,7 +135,7 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
self.add_chapter(chapterTitle, chapterUrl)
rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'class':'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
self.story.setMetadata('rating',rated)
self.story.setMetadata('dateUpdated', makeDate('01/01/2001', '%m/%d/%Y'))

15
fanficfare/adapters/adapter_ashwindersycophanthexcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return AshwinderSycophantHexComAdapter
@ -115,7 +116,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

12
fanficfare/adapters/adapter_asianfanficscom.py
View file

@ -1,15 +1,17 @@
# -*- coding: utf-8 -*-
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return AsianFanFicsComAdapter
@ -125,7 +127,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_asr3slashzoneorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return Asr3SlashzoneOrgAdapter
@ -85,7 +86,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -105,7 +106,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_bdsmlibrarycom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -23,6 +23,7 @@
### Fixed the removal of the extra tags from some of the stories and
### removed the attributes from the paragraph and span tags
###########################################################################
from __future__ import absolute_import
'''
This works, but some of the stories have abysmal formatting, so it would
probably need to be edited for reading.
@ -50,15 +51,17 @@ import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
import sys
import urlparse
from bs4 import Comment
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return BDSMLibraryComSiteAdapter
@ -110,7 +113,7 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(self.url)
soup = self.make_soup(data)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -132,7 +135,7 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(self.url)
soup = self.make_soup(data)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

12
fanficfare/adapters/adapter_bloodshedversecom.py
View file

@ -1,7 +1,6 @@
from __future__ import absolute_import
from datetime import timedelta
import re
import urllib2
import urlparse
import logging
logger = logging.getLogger(__name__)
@ -9,7 +8,12 @@ logger = logging.getLogger(__name__)
from bs4 import BeautifulSoup
from ..htmlcleanup import stripHTML
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions
@ -47,7 +51,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
if exception:
try:
data = self._fetchUrl(url, parameters)
except urllib2.HTTPError:
except HTTPError:
raise exception(self.url)
# Just let self._fetchUrl throw the exception, don't catch and
# customize it.

16
fanficfare/adapters/adapter_bloodtiesfancom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,19 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from bs4.element import Tag
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
# By virtue of being recent and requiring both is_adult and user/pass,
# adapter_fanficcastletvnet.py is the best choice for learning to
@ -150,7 +152,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -184,7 +186,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_buffygilescom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return BuffyGilesComAdapter
@ -123,7 +124,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -157,7 +158,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_chaossycophanthexcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ChaosSycophantHexComAdapter
@ -88,7 +89,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

13
fanficfare/adapters/adapter_chosentwofanficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,18 +16,21 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import sys
from bs4.element import Comment
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ChosenTwoFanFicArchiveAdapter
@ -85,7 +88,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_csiforensicscom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,18 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
@ -87,7 +88,7 @@ class CSIForensicsComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

5
fanficfare/adapters/adapter_darksolaceorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,10 +15,11 @@
# limitations under the License.
#
from __future__ import absolute_import
from ..htmlcleanup import stripHTML
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class DarkSolaceOrgAdapter(BaseEfictionAdapter):

5
fanficfare/adapters/adapter_deandamagecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,10 +15,11 @@
# limitations under the License.
#
from __future__ import absolute_import
from ..htmlcleanup import stripHTML
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class DeanDamageComSiteAdapter(BaseEfictionAdapter):

17
fanficfare/adapters/adapter_deepinmysoulnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return DeepInMySoulNetAdapter ## XXX
@ -123,7 +124,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -157,7 +158,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_destinysgatewaycom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return DestinysGatewayComAdapter
@ -88,7 +89,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -108,7 +109,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_dokugacom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,18 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return DokugaComAdapter
@ -126,7 +127,7 @@ class DokugaComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_dracoandginnycom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return DracoAndGinnyComAdapter
@ -122,7 +123,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -147,7 +148,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

14
fanficfare/adapters/adapter_dramioneorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,19 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from bs4.element import Tag
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return DramioneOrgAdapter
@ -122,7 +124,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_efictionestelielde.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return EfictionEstelielDeAdapter
@ -79,7 +80,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_efpfanficnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return EFPFanFicNet
@ -110,7 +111,7 @@ class EFPFanFicNet(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ErosnSapphoSycophantHexComAdapter
@ -88,7 +89,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -108,7 +109,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

13
fanficfare/adapters/adapter_fanficauthorsnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# -- coding: utf-8 --
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,19 +18,22 @@
### Adapted by GComyn - November 26, 2016
###
####################################################################################################
from __future__ import absolute_import
from __future__ import unicode_literals
import time
import logging
logger = logging.getLogger(__name__)
import re
import sys
import urllib2
from bs4 import UnicodeDammit, Comment
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
####################################################################################################
def getClass():
@ -157,7 +160,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url+'index/', params, usecache=False)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist("Code: 404. {0}".format(url))
elif e.code == 410:

17
fanficfare/adapters/adapter_fanficcastletvnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
# In general an 'adapter' needs to do these five things:
@ -138,7 +139,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -163,7 +164,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

44
fanficfare/adapters/adapter_fanfichu.py
View file

@ -1,6 +1,6 @@
# coding=utf-8
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,11 +15,15 @@
# limitations under the License.
#
from __future__ import absolute_import
import re
import urllib2
import urlparse
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six import ensure_text
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from base_adapter import BaseSiteAdapter, makeDate
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions
@ -61,7 +65,7 @@ class FanficHuAdapter(BaseSiteAdapter):
if exception:
try:
data = self._fetchUrl(url, parameters)
except urllib2.HTTPError:
except HTTPError:
raise exception(self.url)
# Just let self._fetchUrl throw the exception, don't catch and
# customize it.
@ -84,7 +88,7 @@ class FanficHuAdapter(BaseSiteAdapter):
def extractChapterUrlsAndMetadata(self):
soup = self._customized_fetch_url(self.url + '&i=1')
if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
if ensure_text(soup.title.string).strip(u' :') == u'írta':
raise exceptions.StoryDoesNotExist(self.url)
chapter_options = soup.find('form', action='viewstory.php').select('option')
@ -140,46 +144,46 @@ class FanficHuAdapter(BaseSiteAdapter):
while index < len(cells):
cell = cells[index]
key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
key = ensure_text(cell.b.string).strip(u':')
try:
value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
except AttributeError:
value = ensure_text(cells[index+1].string)
except:
value = None
if key == 'Kategória':
if key == u'Kategória':
for anchor in cells[index+1]('a'):
self.story.addToList('category', anchor.string)
elif key == 'Szereplõk':
elif key == u'Szereplõk':
if cells[index+1].string:
for name in cells[index+1].string.split(', '):
self.story.addToList('character', name)
elif key == 'Korhatár':
elif key == u'Korhatár':
if value != 'nem korhatáros':
self.story.setMetadata('rating', value)
elif key == 'Figyelmeztetések':
elif key == u'Figyelmeztetések':
for b_tag in cells[index+1]('b'):
self.story.addToList('warnings', b_tag.string)
elif key == 'Jellemzõk':
elif key == u'Jellemzõk':
for genre in cells[index+1].string.split(', '):
self.story.addToList('genre', genre)
elif key == 'Fejezetek':
elif key == u'Fejezetek':
self.story.setMetadata('numChapters', int(value))
elif key == 'Megjelenés':
elif key == u'Megjelenés':
self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
elif key == 'Frissítés':
elif key == u'Frissítés':
self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
elif key == 'Szavak':
elif key == u'Szavak':
self.story.setMetadata('numWords', value)
elif key == 'Befejezett':
elif key == u'Befejezett':
self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
index += 2

15
fanficfare/adapters/adapter_fanfictionjunkiesde.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
# By virtue of being recent and requiring both is_adult and user/pass,
# adapter_fanficcastletvnet.py is the best choice for learning to
@ -147,7 +148,7 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

16
fanficfare/adapters/adapter_fanfictionnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,21 @@
# limitations under the License.
#
from __future__ import absolute_import
from datetime import datetime
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from urllib import unquote_plus
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
from base_adapter import BaseSiteAdapter, makeDate
from .base_adapter import BaseSiteAdapter, makeDate
ffnetgenres=["Adventure", "Angst", "Crime", "Drama", "Family", "Fantasy", "Friendship", "General",
"Horror", "Humor", "Hurt-Comfort", "Mystery", "Parody", "Poetry", "Romance", "Sci-Fi",
@ -100,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
data = self._fetchUrl(url)
#logger.debug("\n===================\n%s\n===================\n"%data)
soup = self.make_soup(data)
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(url)
else:
@ -135,7 +139,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
and "This request takes too long to process, it is timed out by the server." not in newdata:
logger.debug('=======Found newer chapter: %s' % tryurl)
soup = self.make_soup(newdata)
except urllib2.HTTPError as e:
except HTTPError as e:
if e.code == 503:
raise e
except Exception as e:

16
fanficfare/adapters/adapter_fanfiktionde.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,19 +15,21 @@
# limitations under the License.
#
from __future__ import absolute_import
import time
import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return FanFiktionDeAdapter
@ -118,7 +120,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -164,7 +166,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
self.story.extendList('genre',genres[:genres.index(' / ')].split(', '))
self.story.setMetadata('rating', genres[genres.index(' / ')+3:])
self.story.addToList('category',stripHTML(soup.find('span',id='ffcbox-story-topic-1')).split(' / ')[2])
self.story.addToList('category',stripHTML(soup.find('span',id='ffcbox-story-topic-1')).split('/')[2].strip())
try:
self.story.setMetadata('native_status', head.find_all('span',{'class':'titled-icon'})[3]['title'])

5
fanficfare/adapters/adapter_fannation.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,8 +16,9 @@
#
# Software: eFiction
from __future__ import absolute_import
import re
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class FanNationAdapter(BaseEfictionAdapter):

5
fanficfare/adapters/adapter_fhsarchivecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,8 +16,9 @@
#
# Software: eFiction
from __future__ import absolute_import
import re
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class FHSArchiveComAdapter(BaseEfictionAdapter):

13
fanficfare/adapters/adapter_ficbooknet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,19 +15,22 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import datetime
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from .. import translit
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
@ -77,7 +80,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_fictionalleyorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,18 +15,19 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
@ -80,7 +81,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
try:
data = self._postFetchWithIAmOld(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

13
fanficfare/adapters/adapter_fictionhuntcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,15 +15,18 @@
# limitations under the License.
#
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class FictionHuntComSiteAdapter(BaseSiteAdapter):
@ -68,7 +71,7 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
url = self.url
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.meta)
else:

11
fanficfare/adapters/adapter_fictionmaniatv.py
View file

@ -1,8 +1,11 @@
from __future__ import absolute_import
import re
import urllib2
import urlparse
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from base_adapter import BaseSiteAdapter, makeDate
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
@ -44,7 +47,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
if exception:
try:
data = self._fetchUrl(url, parameters)
except urllib2.HTTPError:
except HTTPError:
raise exception(self.url)
# Just let self._fetchUrl throw the exception, don't catch and
# customize it.

14
fanficfare/adapters/adapter_fictionpadcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,19 +15,21 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import time
import json
#from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class FictionPadSiteAdapter(BaseSiteAdapter):
@ -123,7 +125,7 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
data = data[:data.rindex(";")]
data = data.replace('tables:','"tables":')
tables = json.loads(data)['tables']
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(url)
else:

11
fanficfare/adapters/adapter_fictionpresscom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,15 +15,16 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import time
# py2 vs py3 transition
from ..six import text_type as unicode
## They're from the same people and pretty much identical.
from adapter_fanfictionnet import FanFictionNetSiteAdapter
from .adapter_fanfictionnet import FanFictionNetSiteAdapter
class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):

17
fanficfare/adapters/adapter_ficwadcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,18 +15,19 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import time
import httplib, urllib
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class FicwadComSiteAdapter(BaseSiteAdapter):
@ -96,7 +97,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
if "<h4>Featured Story</h4>" in data:
raise exceptions.StoryDoesNotExist(self.url)
soup = self.make_soup(data)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -118,7 +119,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
self._setURL(url)
try:
soup = self.make_soup(self._fetchUrl(url))
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

14
fanficfare/adapters/adapter_fimfictionnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,19 +15,23 @@
# limitations under the License.
#
from __future__ import absolute_import
import time
from datetime import date, datetime
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import cookielib as cl
import json
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from ..six.moves import http_cookiejar as cl
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return FimFictionNetSiteAdapter
@ -97,7 +101,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
data = self.do_fix_blockquotes(self._fetchUrl(self.url,
usecache=(not self.is_adult)))
soup = self.make_soup(data)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

9
fanficfare/adapters/adapter_finestoriescom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,9 +15,14 @@
# limitations under the License.
#
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
from adapter_storiesonlinenet import StoriesOnlineNetAdapter
# py2 vs py3 transition
from ..six import text_type as unicode
from .adapter_storiesonlinenet import StoriesOnlineNetAdapter
def getClass():
return FineStoriesComAdapter

14
fanficfare/adapters/adapter_fireflyfansnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -17,14 +17,16 @@
####################################################################################################
# Adapted by GComyn - December 10, 2016
####################################################################################################
from __future__ import absolute_import
''' This adapter will download the stories from the www.fireflyfans.net forum pages '''
import logging
import re
import sys
import time
import urllib2
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from base_adapter import BaseSiteAdapter, makeDate
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
@ -79,7 +81,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -131,7 +133,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
# which is usualy FireFly on this site, but I'm going to get them
# anyway.a
category = soup.find('span', {'id': 'MainContent_txtItemDetails'})
category = stripHTML(str(category).replace(b"\xc2\xa0", ' '))
category = stripHTML(unicode(category).replace(u"\xa0", u' '))
metad = category.split(' ')
for meta in metad:
if ":" in meta:

30
fanficfare/adapters/adapter_fireflypopulliorg.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -21,13 +21,17 @@
###=================================================================================================
### I have started to use lines of # on the line just before a function so they are easier to find.
####################################################################################################
from __future__ import absolute_import
''' This adapter scrapes the metadata and chapter text from stories on firefly.populli.org '''
import logging
import re
import urllib2
import sys
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
@ -97,7 +101,7 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
'''
try:
page_data = self._fetchUrl(page)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('404 error: {}'.format(page))
else:
@ -125,7 +129,8 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
if not title:
raise exceptions.StoryDoesNotExist('Cannot find title on the page {}'.format(url))
self.story.setMetadata('title', stripHTML(soup.find('h2')))
rawtitle = stripHTML(soup.find('h2'))
self.story.setMetadata('title', rawtitle)
# This site has the entire story on one page, so we will be using the normalized URL as
# the chapterUrl and the Title as the chapter Title
@ -149,7 +154,7 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
if ',' in mdata:
self.story.setMetadata('coauthor', ', '.join(mdata.split(',')[1:]).strip())
mdata = mdata.split(',')[0]
# print mdata
# self.story.getMetadata('coauthor')
# sys.exit()
@ -180,13 +185,16 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
if stories:
for story in stories:
# There alot of nbsp's (non broken spaces) in here, so I'm going to remove them
# I'm also getting rid of the bold tags and the nextline characters to make it
# I'm also getting rid of the bold tags and the nextline characters to make it
# easier to get the information below
story = repr(story).replace(b'\\xa0', '').replace(' ',' ').replace(
story = repr(story).replace(u'\\xa0', '').replace(' ',' ').replace(
'<b>','').replace('</b>','').replace(r'\n','')
story = self.make_soup(story).find('p')
story_a = story.find('a')
title = self.story.getMetadata('title').split('-')[0].strip()
# not sure why this split is here, but it caused
# problems when title_chapter_range_pattern
# introduces a '-', so save rawtitle --JM
title = rawtitle.split('-')[0].strip()
if story_a.get_text() == title:
story_found = True
break
@ -265,7 +273,7 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
else:
## This should catch anything else, and shouldn't ever really be gotten
# to, but I'm going to have it print out in the debugger, just in case
logger.debug('Metadata not caught: %s' % str(meta))
logger.debug('Metadata not caught: %s' % unicode(meta))
zzzzzzzz = 0
elif label == 'characters':
self.story.setMetadata('characters', value)
@ -315,7 +323,7 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
# the end of the section, which has alot of extraneous things, then adding my own div
# wrapper, recreating the soup, then getting that div from the soup again, before sending to
# the writers.
story = repr(story).replace(b'\\xa0', '').replace(' ',' ').replace(r'\n','').strip()
story = repr(story).replace(u'\\xa0', '').replace(' ',' ').replace(r'\n','').strip()
story = story[12:]
story = story[:story.find('<p align="center" class="comments">Please <')]
story = '<div class="chaptertext">' + story + '</div>'

5
fanficfare/adapters/adapter_forumquestionablequestingcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,10 +15,11 @@
# limitations under the License.
#
from __future__ import absolute_import
import re
from ..htmlcleanup import stripHTML
from base_xenforoforum_adapter import BaseXenForoForumAdapter
from .base_xenforoforum_adapter import BaseXenForoForumAdapter
def getClass():
return QuestionablequestingComAdapter

5
fanficfare/adapters/adapter_forumsspacebattlescom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,9 +15,10 @@
# limitations under the License.
#
from __future__ import absolute_import
import re
from base_xenforoforum_adapter import BaseXenForoForumAdapter
from .base_xenforoforum_adapter import BaseXenForoForumAdapter
def getClass():
return ForumsSpacebattlesComAdapter

3
fanficfare/adapters/adapter_forumssufficientvelocitycom.py
View file

@ -15,9 +15,10 @@
# limitations under the License.
#
from __future__ import absolute_import
import re
from base_xenforoforum_adapter import BaseXenForoForumAdapter
from .base_xenforoforum_adapter import BaseXenForoForumAdapter
def getClass():
return ForumsSufficientVelocityComAdapter

7
fanficfare/adapters/adapter_gluttonyfictioncom.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2015 FanFicFare team
# Copyright 2016 FanFicFare team
# Copyright 2018 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -21,7 +21,8 @@
### Rewritten by: GComyn on November, 06, 2016
### Original was adapter_fannation.py
##################################################################################
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
class GluttonyFictionComAdapter(BaseEfictionAdapter):

12
fanficfare/adapters/adapter_gravitytalescom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,10 +18,10 @@
## Adapted by GComyn on April 21, 2017
####################################################################################################
from __future__ import absolute_import
import logging
import re
import time
import urllib2
from datetime import datetime
logger = logging.getLogger(__name__)
@ -37,7 +37,11 @@ except ImportError:
# logger.warn('No version of feedparser module available, falling back to naive published and updated date')
feedparser = None
from base_adapter import BaseSiteAdapter
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
@ -93,7 +97,7 @@ class GravityTalesComSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('Error 404: {0}'.format(self.url))
else:

14
fanficfare/adapters/adapter_harrypotterfanfictioncom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,16 +15,18 @@
# limitations under the License.
#
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
@ -69,7 +71,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_hlfictionnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return HLFictionNetAdapter
@ -78,7 +79,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

14
fanficfare/adapters/adapter_hpfanficarchivecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,19 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from bs4.element import Comment
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return HPFanficArchiveComAdapter
@ -78,7 +80,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

16
fanficfare/adapters/adapter_iketernalnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,16 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return IkEternalNetAdapter
@ -121,7 +123,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -159,7 +161,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_imagineeficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return ImagineEFicComAdapter
@ -122,7 +123,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -147,7 +148,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

5
fanficfare/adapters/adapter_imrightbehindyoucom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,10 +15,11 @@
# limitations under the License.
#
from __future__ import absolute_import
from ..htmlcleanup import stripHTML
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class ImRightBehindYouComSiteAdapter(BaseEfictionAdapter):

13
fanficfare/adapters/adapter_inkbunnynet.py
View file

@ -18,20 +18,23 @@
# Adapted by GComyn on April 24, 2017
# Updated by GComyn on June 11, 2018
from __future__ import absolute_import
import logging
import re
import sys
import urllib2
from datetime import datetime, timedelta
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
from ..dateutils import UNIX_EPOCHE
UNIX_EPOCHE = datetime.fromtimestamp(0)
logger = logging.getLogger(__name__)
def getClass():
return InkBunnyNetSiteAdapter
@ -122,7 +125,7 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('Error 404: {0}'.format(self.url))
else:

5
fanficfare/adapters/adapter_itcouldhappennet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,8 +16,9 @@
#
# Software: eFiction
from __future__ import absolute_import
import re
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class ItCouldHappenNetSiteAdapter(BaseEfictionAdapter):

17
fanficfare/adapters/adapter_kiarepositorymujajinet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return KiaRepositoryMujajiNetAdapter ## XXX
@ -123,7 +124,7 @@ class KiaRepositoryMujajiNetAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -157,7 +158,7 @@ class KiaRepositoryMujajiNetAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_ksarchivecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
# Search for XXX comments--that's where things are most likely to need changing.
@ -101,7 +102,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -134,7 +135,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

13
fanficfare/adapters/adapter_lcfanficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,15 +18,18 @@
## Adapted by GComyn on April 22, 2017
####################################################################################################
from __future__ import absolute_import
import logging
import json
import re
import sys # ## used for debug purposes
import time
import urllib2
import datetime
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
@ -96,7 +99,7 @@ class LCFanFicComSiteAdapter(BaseSiteAdapter):
url = self.url
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('Error 404: {0}'.format(self.url))
else:

5
fanficfare/adapters/adapter_libraryofmoriacom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,7 +16,8 @@
#
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
class LibraryOfMoriaComAdapter(BaseEfictionAdapter):

11
fanficfare/adapters/adapter_lightnovelgatecom.py
View file

@ -19,12 +19,15 @@
### Tested with Calibre
####################################################################################################
from __future__ import absolute_import
import logging
import re
import urllib2
import urlparse
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from base_adapter import BaseSiteAdapter, makeDate
from .base_adapter import BaseSiteAdapter, makeDate
from bs4 import Comment
from ..htmlcleanup import removeEntities, stripHTML, fix_excess_space
@ -95,7 +98,7 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('404 error: {}'.format(url))
else:

19
fanficfare/adapters/adapter_literotica.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,18 +15,21 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import urlparse
from bs4.element import Comment
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class LiteroticaSiteAdapter(BaseSiteAdapter):
@ -132,7 +135,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
soup1 = self.make_soup(data1)
#strip comments from soup
[comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, Comment))]
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code in [404, 410]:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -157,7 +160,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
#strip comments from soup
[comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, Comment))]
# logger.debug(soupAuth)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code in [404, 410]:
raise exceptions.StoryDoesNotExist(authorurl)
else:
@ -343,7 +346,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
chapter_description = '<p><b>Description:</b> %s</p><hr />' % chapter_description
fullhtml += self.getPageText(raw_page, url)
if pages:
for page_no in xrange(2, len(page_nums) + 1):
for page_no in range(2, len(page_nums) + 1):
page_url = url + "?page=%s" % page_no
logger.debug("page_url= %s" % page_url)
raw_page = self._fetchUrl(page_url)

7
fanficfare/adapters/adapter_looselugscom.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# Copyright 2015 FanFicFare team
# Copyright 2016 FanFicFare team
# Copyright 2018 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -21,7 +21,8 @@
### Rewritten by: GComyn on November, 06, 2016
### Original was adapter_fannation.py
##################################################################################
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
class LooseLugsComAdapter(BaseEfictionAdapter):

5
fanficfare/adapters/adapter_lotrfanfictioncom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,7 +16,8 @@
#
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
class TheLOTRFanFictionSiteAdapter(BaseEfictionAdapter):

18
fanficfare/adapters/adapter_lotrgficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,17 +18,19 @@
### Adapted by GComyn
### Completed on November, 22, 2016
##############################################################################
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class LOTRgficComAdapter(BaseSiteAdapter):
@ -79,7 +81,7 @@ class LOTRgficComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -253,7 +255,7 @@ class LOTRgficComAdapter(BaseSiteAdapter):
## dedicated tag, so we have to split some hairs..
## This may not work every time... but I tested it with 6 stories...
mdata = metad[0]
while '<hr/>' not in str(mdata.nextSibling):
while '<hr/>' not in unicode(mdata.nextSibling):
mdata = mdata.nextSibling
self.setDescription(url,mdata.previousSibling.previousSibling.get_text())
@ -284,7 +286,7 @@ class LOTRgficComAdapter(BaseSiteAdapter):
#<br/>
#</p>
## we'll have to remove the non-breaking spaces to get this to work.
metad = str(metad).replace(b"\xc2\xa0",'').replace('\n','')
metad = unicode(metad).replace(u"\xa0",'').replace('\n','')
for txt in metad.split('<br/>'):
if 'Challenges:' in txt:
txt = txt.replace('Challenges:','').strip()

15
fanficfare/adapters/adapter_lumossycophanthexcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return LumosSycophantHexComAdapter
@ -88,7 +89,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_masseffect2in.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,15 +15,21 @@
# limitations under the License.
#
from __future__ import absolute_import
import bs4
import datetime
import logging
import re
import urllib2
from itertools import takewhile
from ..htmlcleanup import removeEntities, stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from ..six.moves import zip as izip
from .base_adapter import BaseSiteAdapter, makeDate
_logger = logging.getLogger(__name__)
@ -114,7 +120,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
try:
startingChapter = self._makeChapter(self.url)
except urllib2.HTTPError, error:
except HTTPError as error:
if error.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
raise
@ -198,7 +204,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
self.add_chapter(chapterTitle, url)
except ParsingError, error:
except ParsingError as error:
raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))
# Some metadata are handled separately due to format conversions.
@ -700,7 +706,6 @@ def _getLargestCommonPrefix(*args):
"""Returns largest common prefix of all unicode arguments, ignoring case.
:rtype : unicode
"""
from itertools import takewhile, izip
toLower = lambda xs: map(lambda x: x.lower(), xs)
allSame = lambda xs: len(set(toLower(xs))) == 1
return u''.join([i[0] for i in takewhile(allSame, izip(*args))])

16
fanficfare/adapters/adapter_mcstoriescom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,20 +15,22 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
import urlparse
import time
import os
from bs4.element import Comment
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class MCStoriesComSiteAdapter(BaseSiteAdapter):
@ -86,7 +88,7 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
soup1 = self.make_soup(data1)
#strip comments from soup
[comment.extract() for comment in soup1.find_all(text=lambda text:isinstance(text, Comment))]
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

16
fanficfare/adapters/adapter_mediaminerorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,19 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
@ -108,7 +110,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url) # w/o trailing / gets 'chapter list' page even for one-shots.
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
logger.error("404 on %s"%url)
raise exceptions.StoryDoesNotExist(self.url)
@ -120,7 +122,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
## title:
## <h1 id="post-title">A, A' Fan Fiction &#10095; Mmmmm</h1>
titletext = stripHTML(soup.find("h1",{"id":"post-title"}))
titletext = unicode(stripHTML(soup.find("h1",{"id":"post-title"})))
titletext = titletext[titletext.index(u'❯')+2:]
# print("title:(%s)"%titletext)
self.story.setMetadata('title',titletext)

17
fanficfare/adapters/adapter_merlinficdtwinscouk.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return MerlinFicDtwinsCoUk
@ -122,7 +123,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -147,7 +148,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_midnightwhispers.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
# Search for XXX comments--that's where things are most likely to need changing.
@ -106,7 +107,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -139,7 +140,7 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

5
fanficfare/adapters/adapter_mttjustoncenet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,10 +15,11 @@
# limitations under the License.
#
from __future__ import absolute_import
from ..htmlcleanup import stripHTML
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class MTTJustOnceNetSiteAdapter(BaseEfictionAdapter):

5
fanficfare/adapters/adapter_naiceanilmenet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016 FanFicFare team
# Copyright 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,7 +16,8 @@
#
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
class NaiceaNilmeNetAdapter(BaseEfictionAdapter):

5
fanficfare/adapters/adapter_narutoficorg.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,10 +15,11 @@
# limitations under the License.
#
from __future__ import absolute_import
from ..htmlcleanup import stripHTML
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class NarutoFicOrgSiteAdapter(BaseEfictionAdapter):

15
fanficfare/adapters/adapter_nationallibrarynet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return NationalLibraryNetAdapter
@ -83,7 +84,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_ncisficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return NCISFicComAdapter
@ -82,7 +83,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

5
fanficfare/adapters/adapter_ncisfictioncom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,7 +16,8 @@
#
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
class NCISFictionComAdapter(BaseEfictionAdapter):

17
fanficfare/adapters/adapter_nfacommunitycom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
# Search for XXX comments--that's where things are most likely to need changing.
@ -101,7 +102,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -134,7 +135,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

5
fanficfare/adapters/adapter_nhamagicalworldsus.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,7 +16,8 @@
#
# Software: eFiction
from base_efiction_adapter import BaseEfictionAdapter
from __future__ import absolute_import
from .base_efiction_adapter import BaseEfictionAdapter
# Class name has to be unique. Our convention is camel case the
# sitename with Adapter at the end. www is skipped.

5
fanficfare/adapters/adapter_ninelivesarchivecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team
# Copyright 2015 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,8 +16,9 @@
#
# Software: eFiction
from __future__ import absolute_import
import re
from base_efiction_adapter import BaseEfictionAdapter
from .base_efiction_adapter import BaseEfictionAdapter
class NineLivesAdapter(BaseEfictionAdapter):

13
fanficfare/adapters/adapter_nocturnallightnet.py
View file

@ -1,12 +1,15 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import re
import urllib2
import urlparse
from bs4.element import Tag
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib import parse as urlparse
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions
@ -47,7 +50,7 @@ class NocturnalLightNetAdapter(BaseSiteAdapter):
if exception:
try:
data = self._fetchUrl(url, parameters)
except urllib2.HTTPError:
except HTTPError:
raise exception(self.url)
# Just let self._fetchUrl throw the exception, don't catch and
# customize it.

15
fanficfare/adapters/adapter_noveltrovecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2014 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,15 +18,18 @@
## Adapted by GComyn on April 22, 2017
####################################################################################################
from __future__ import absolute_import
import logging
import json
import re
import sys # ## used for debug purposes
import time
import urllib2
import datetime
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
from .. import exceptions as exceptions
from ..htmlcleanup import stripHTML
@ -88,7 +91,7 @@ class NovelTroveComSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('Error 404: {0}'.format(self.url))
else:
@ -98,7 +101,7 @@ class NovelTroveComSiteAdapter(BaseSiteAdapter):
soup = self.make_soup(data)
# Now go hunting for all the meta data we can get
metablock = soup.find('div', {'class', 'title-infos'})
metablock = soup.find('div', {'class': 'title-infos'})
## Getting Title
title = stripHTML(metablock.find('h1'))

15
fanficfare/adapters/adapter_occlumencysycophanthexcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return OcclumencySycophantHexComAdapter
@ -114,7 +115,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

14
fanficfare/adapters/adapter_phoenixsongnet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,19 @@
# limitations under the License.
#
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2, urllib, cookielib
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return PhoenixSongNetAdapter
@ -111,7 +113,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
if self.getConfig('force_login'):
self.performLogin(url)
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_ponyfictionarchivenet.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return PonyFictionArchiveNetAdapter
@ -92,7 +93,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -113,7 +114,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

15
fanficfare/adapters/adapter_potionsandsnitches.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2011 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,18 +16,19 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
@ -65,7 +66,7 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

14
fanficfare/adapters/adapter_potterficscom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,17 +15,19 @@
# limitations under the License.
#
from __future__ import absolute_import
import datetime
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter
# This function is called by the downloader in all adapter_*.py files
# in this dir to register the adapter class. So it needs to be
@ -144,7 +146,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_potterheadsanonymouscom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2013 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return PotterHeadsAnonymousComAdapter
@ -122,7 +123,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -156,7 +157,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_pretendercentrecom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return PretenderCenterComAdapter
@ -91,7 +92,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -111,7 +112,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

17
fanficfare/adapters/adapter_qafficcom.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2012 Fanficdownloader team, 2017 FanFicFare team
# Copyright 2012 Fanficdownloader team, 2018 FanFicFare team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,17 +16,18 @@
#
# Software: eFiction
import time
from __future__ import absolute_import
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
# py2 vs py3 transition
from ..six import text_type as unicode
from ..six.moves.urllib.error import HTTPError
from .base_adapter import BaseSiteAdapter, makeDate
def getClass():
return QafFicComAdapter
@ -88,7 +89,7 @@ class QafFicComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
@ -108,7 +109,7 @@ class QafFicComAdapter(BaseSiteAdapter):
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:

Some files were not shown because too many files have changed in this diff Show more