diff --git a/fanficfare/adapters/adapter_adastrafanficcom.py b/fanficfare/adapters/adapter_adastrafanficcom.py index 87cc1659..a4a39a17 100644 --- a/fanficfare/adapters/adapter_adastrafanficcom.py +++ b/fanficfare/adapters/adapter_adastrafanficcom.py @@ -75,7 +75,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): logger.debug("URL: "+url) try: - data = self._fetchUrl(url) + data = self.get_request(url) except HTTPError as e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) @@ -194,7 +194,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): series_url = 'http://'+self.host+'/'+a['href'] # use BeautifulSoup HTML parser to make everything easier to find. - seriessoup = self.make_soup(self._fetchUrl(series_url)) + seriessoup = self.make_soup(self.get_request(series_url)) storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) i=1 for a in storyas: @@ -213,7 +213,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): logger.debug('Getting chapter text from: %s' % url) - data = self._fetchUrl(url) + data = self.get_request(url) # problems with some stories, but only in calibre. I suspect # issues with different SGML parsers in python. This is a # nasty hack, but it works. diff --git a/fanficfare/adapters/adapter_adultfanfictionorg.py b/fanficfare/adapters/adapter_adultfanfictionorg.py index ee1433bc..94085c9f 100644 --- a/fanficfare/adapters/adapter_adultfanfictionorg.py +++ b/fanficfare/adapters/adapter_adultfanfictionorg.py @@ -201,7 +201,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter): logger.debug("URL: "+url) try: - data = self._fetchUrl(url) + data = self.get_request(url) except HTTPError as e: if e.code == 404: raise exceptions.StoryDoesNotExist("Code: 404. {0}".format(url)) @@ -267,7 +267,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter): logger.debug('Getting the author page: {0}'.format(author_Url)) try: - adata = self._fetchUrl(author_Url) + adata = self.get_request(author_Url) except HTTPError as e: if e.code in 404: raise exceptions.StoryDoesNotExist("Author Page: Code: 404. {0}".format(author_Url)) @@ -305,7 +305,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter): author_Url = '{0}&view=story&zone={1}&page={2}'.format(self.story.getMetadata('authorUrl'), self.zone, unicode(page)) logger.debug('Getting the author page: {0}'.format(author_Url)) try: - adata = self._fetchUrl(author_Url) + adata = self.get_request(author_Url) except HTTPError as e: if e.code in 404: raise exceptions.StoryDoesNotExist("Author Page: Code: 404. {0}".format(author_Url)) @@ -398,7 +398,7 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter): #Since each chapter is on 1 page, we don't need to do anything special, just get the content of the page. logger.debug('Getting chapter text from: %s' % url) - soup = self.make_soup(self._fetchUrl(url)) + soup = self.make_soup(self.get_request(url)) chaptertag = soup.find('div',{'class' : 'pagination'}).parent.findNext('td') if None == chaptertag: raise exceptions.FailedToDownload("Error downloading Chapter: {0}! Missing required element!".format(url)) diff --git a/fanficfare/adapters/adapter_andromedawebcom.py b/fanficfare/adapters/adapter_andromedawebcom.py index f871f537..6e34f70b 100644 --- a/fanficfare/adapters/adapter_andromedawebcom.py +++ b/fanficfare/adapters/adapter_andromedawebcom.py @@ -125,7 +125,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX logger.debug("URL: "+url) try: - data = self._fetchUrl(url) + data = self.get_request(url) except HTTPError as e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) @@ -135,7 +135,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX if self.needToLoginCheck(data): # need to log in for this one. self.performLogin(url) - data = self._fetchUrl(url) + data = self.get_request(url) # Since the warning text can change by warning level, let's # look for the warning pass url. ksarchive uses @@ -159,7 +159,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX logger.debug("URL 2nd try: "+url) try: - data = self._fetchUrl(url) + data = self.get_request(url) except HTTPError as e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) @@ -267,7 +267,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX series_url = 'http://'+self.host+'/'+a['href'] # use BeautifulSoup HTML parser to make everything easier to find. - seriessoup = self.make_soup(self._fetchUrl(series_url)) + seriessoup = self.make_soup(self.get_request(series_url)) storyas = seriessoup.findAll('a', href=re.compile(r'^fiction/viewstory.php\?sid=\d+$')) i=1 for a in storyas: @@ -286,7 +286,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX logger.debug('Getting chapter text from: %s' % url) - soup = self.make_soup(self._fetchUrl(url)) + soup = self.make_soup(self.get_request(url)) div = soup.find('div', {'class' : 'story'}) diff --git a/fanficfare/adapters/adapter_archiveofourownorg.py b/fanficfare/adapters/adapter_archiveofourownorg.py index b5c40614..5066cb44 100644 --- a/fanficfare/adapters/adapter_archiveofourownorg.py +++ b/fanficfare/adapters/adapter_archiveofourownorg.py @@ -124,7 +124,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): params['utf8'] = u'\x2713' # utf8 *is* required now. hex code works better than actual character for some reason. u'✓' # authenticity_token now comes from a completely separate json call. - token_json = json.loads(self._fetchUrl('https://' + self.getSiteDomain() + "/token_dispenser.json")) + token_json = json.loads(self.get_request('https://' + self.getSiteDomain() + "/token_dispenser.json")) params['authenticity_token'] = token_json['token'] loginUrl = 'https://' + self.getSiteDomain() + '/users/login' @@ -162,8 +162,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): logger.info("metaurl: "+metaurl) try: - data = self._fetchUrl(url) - meta = self._fetchUrl(metaurl) + data = self.get_request(url) + meta = self.get_request(metaurl) if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta: if self.addurl: @@ -172,7 +172,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): metasoup = self.make_soup(meta) a = metasoup.find('a',text='Proceed') metaurl = 'https://'+self.host+a['href'] - meta = self._fetchUrl(metaurl) + meta = self.get_request(metaurl) else: raise exceptions.AdultCheckRequired(self.url) @@ -189,8 +189,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): if self.needToLoginCheck(data) or \ ( self.getConfig("always_login") and 'href="/users/logout"' not in data ): self.performLogin(url,data) - data = self._fetchUrl(url,usecache=False) - meta = self._fetchUrl(metaurl,usecache=False) + data = self.get_request(url,usecache=False) + meta = self.get_request(metaurl,usecache=False) # use BeautifulSoup HTML parser to make everything easier to find. soup = self.make_soup(data) @@ -428,7 +428,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): logger.debug("USE view_full_work") ## Assumed view_adult=true was cookied during metadata if not self.full_work_soup: - self.full_work_soup = self.make_soup(self._fetchUrl(self.url+"?view_full_work=true"+self.addurl.replace('?','&'))) + self.full_work_soup = self.make_soup(self.get_request(self.url+"?view_full_work=true"+self.addurl.replace('?','&'))) ## AO3 has had several cases now where chapter numbers ## are missing, breaking the link between ##
sections, so have to check for that
diff --git a/fanficfare/adapters/adapter_bloodshedversecom.py b/fanficfare/adapters/adapter_bloodshedversecom.py
index 6d8ae098..306fc952 100644
--- a/fanficfare/adapters/adapter_bloodshedversecom.py
+++ b/fanficfare/adapters/adapter_bloodshedversecom.py
@@ -72,7 +72,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
def extractChapterUrlsAndMetadata(self):
logger.debug("URL: "+self.url)
- soup = self.make_soup(self._fetchUrl(self.url))
+ soup = self.make_soup(self.get_request(self.url))
# Since no 404 error code we have to raise the exception ourselves.
# A title that is just 'by' indicates that there is no author name
@@ -99,7 +99,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
# Get the URL to the author's page and find the correct story entry to
# scrape the metadata
author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
- soup = self.make_soup(self._fetchUrl(author_url))
+ soup = self.make_soup(self.get_request(author_url))
# Ignore first list_box div, it only contains the author information
for list_box in soup('div', {'class': 'list_box'})[1:]:
@@ -188,7 +188,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
raise exceptions.AdultCheckRequired(self.url)
def getChapterText(self, url):
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
storytext_div = soup.find('div', {'class': 'tl'})
storytext_div = storytext_div.find('div', {'class': ''})
diff --git a/fanficfare/adapters/adapter_bloodtiesfancom.py b/fanficfare/adapters/adapter_bloodtiesfancom.py
index c378c65a..8c98ca2b 100644
--- a/fanficfare/adapters/adapter_bloodtiesfancom.py
+++ b/fanficfare/adapters/adapter_bloodtiesfancom.py
@@ -151,7 +151,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -185,7 +185,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -297,7 +297,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
series_url = 'http://'+self.host+'/fiction/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -316,7 +316,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_buffygilescom.py b/fanficfare/adapters/adapter_buffygilescom.py
index 6bf004e6..89e3bb1f 100644
--- a/fanficfare/adapters/adapter_buffygilescom.py
+++ b/fanficfare/adapters/adapter_buffygilescom.py
@@ -123,7 +123,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -133,7 +133,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
# Since the warning text can change by warning level, let's
# look for the warning pass url. ksarchive uses
@@ -157,7 +157,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -266,7 +266,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^efiction/viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -285,7 +285,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_chaossycophanthexcom.py b/fanficfare/adapters/adapter_chaossycophanthexcom.py
index a13f456a..752be20e 100644
--- a/fanficfare/adapters/adapter_chaossycophanthexcom.py
+++ b/fanficfare/adapters/adapter_chaossycophanthexcom.py
@@ -88,7 +88,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -203,7 +203,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -222,7 +222,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_chireadscom.py b/fanficfare/adapters/adapter_chireadscom.py
index 1cb92cbe..bd74f9d7 100644
--- a/fanficfare/adapters/adapter_chireadscom.py
+++ b/fanficfare/adapters/adapter_chireadscom.py
@@ -69,7 +69,7 @@ class ChireadsComSiteAdapter(BaseSiteAdapter):
def extractChapterUrlsAndMetadata(self):
logger.debug('URL: %s', self.url)
try:
- data = self._fetchUrl(self.url)
+ data = self.get_request(self.url)
except HTTPError as exception:
if exception.code == 404:
raise exceptions.StoryDoesNotExist('404 error: {}'.format(self.url))
@@ -106,7 +106,7 @@ class ChireadsComSiteAdapter(BaseSiteAdapter):
def getChapterText(self, url):
logger.debug('Getting chapter text from: %s' % url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
soup = self.make_soup(data)
content = soup.select_one('#content')
diff --git a/fanficfare/adapters/adapter_chosentwofanficcom.py b/fanficfare/adapters/adapter_chosentwofanficcom.py
index 7a4927b6..8c7434a3 100644
--- a/fanficfare/adapters/adapter_chosentwofanficcom.py
+++ b/fanficfare/adapters/adapter_chosentwofanficcom.py
@@ -94,7 +94,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -212,7 +212,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
i=1
@@ -234,7 +234,7 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_csiforensicscom.py b/fanficfare/adapters/adapter_csiforensicscom.py
index 9e90bd85..ab466f7e 100644
--- a/fanficfare/adapters/adapter_csiforensicscom.py
+++ b/fanficfare/adapters/adapter_csiforensicscom.py
@@ -87,7 +87,7 @@ class CSIForensicsComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -196,7 +196,7 @@ class CSIForensicsComAdapter(BaseSiteAdapter):
series_url = 'https://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -221,7 +221,7 @@ class CSIForensicsComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_deepinmysoulnet.py b/fanficfare/adapters/adapter_deepinmysoulnet.py
index 93202403..d06101f3 100644
--- a/fanficfare/adapters/adapter_deepinmysoulnet.py
+++ b/fanficfare/adapters/adapter_deepinmysoulnet.py
@@ -123,7 +123,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -133,7 +133,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
# Since the warning text can change by warning level, let's
# look for the warning pass url. ksarchive uses
@@ -157,7 +157,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -265,7 +265,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
series_url = 'https://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^fiction/viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -284,7 +284,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_destinysgatewaycom.py b/fanficfare/adapters/adapter_destinysgatewaycom.py
index e8e1eeed..72f744a8 100644
--- a/fanficfare/adapters/adapter_destinysgatewaycom.py
+++ b/fanficfare/adapters/adapter_destinysgatewaycom.py
@@ -88,7 +88,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -108,7 +108,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -209,7 +209,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -228,7 +228,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_dokugacom.py b/fanficfare/adapters/adapter_dokugacom.py
index e7da63f5..3bab8224 100644
--- a/fanficfare/adapters/adapter_dokugacom.py
+++ b/fanficfare/adapters/adapter_dokugacom.py
@@ -126,7 +126,7 @@ class DokugaComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -139,7 +139,7 @@ class DokugaComAdapter(BaseSiteAdapter):
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url,soup)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
soup = self.make_soup(data)
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
@@ -172,7 +172,7 @@ class DokugaComAdapter(BaseSiteAdapter):
self.add_chapter(chapter,'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value'])
- asoup = self.make_soup(self._fetchUrl(alink))
+ asoup = self.make_soup(self.get_request(alink))
if 'fanfiction' in self.section:
asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
@@ -262,7 +262,7 @@ class DokugaComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'chtext'})
diff --git a/fanficfare/adapters/adapter_dracoandginnycom.py b/fanficfare/adapters/adapter_dracoandginnycom.py
index 81485304..be4cacae 100644
--- a/fanficfare/adapters/adapter_dracoandginnycom.py
+++ b/fanficfare/adapters/adapter_dracoandginnycom.py
@@ -122,7 +122,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -132,7 +132,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
if m != None:
@@ -147,7 +147,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -261,7 +261,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
i=1
@@ -283,7 +283,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'class' : 'listbox'})
diff --git a/fanficfare/adapters/adapter_efpfanficnet.py b/fanficfare/adapters/adapter_efpfanficnet.py
index 82764764..a225c5c6 100644
--- a/fanficfare/adapters/adapter_efpfanficnet.py
+++ b/fanficfare/adapters/adapter_efpfanficnet.py
@@ -110,7 +110,7 @@ class EFPFanFicNet(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -120,7 +120,7 @@ class EFPFanFicNet(BaseSiteAdapter):
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
# if "Access denied. This story has not been validated by the adminstrators of this site." in data:
# raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
@@ -184,7 +184,7 @@ class EFPFanFicNet(BaseSiteAdapter):
# Need author page for most of the metadata.
logger.debug("fetching author page: (%s)"%authurl)
- authsoup = self.make_soup(self._fetchUrl(authurl))
+ authsoup = self.make_soup(self.get_request(authurl))
#print("authsoup:%s"%authsoup)
storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
@@ -271,7 +271,7 @@ class EFPFanFicNet(BaseSiteAdapter):
series_url = 'https://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
i=1
@@ -291,7 +291,7 @@ class EFPFanFicNet(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'class' : 'storia'})
diff --git a/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py b/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py
index 0b68b6a8..d16f09c5 100644
--- a/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py
+++ b/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py
@@ -88,7 +88,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -108,7 +108,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -219,7 +219,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
i=1
for a in storyas:
@@ -240,7 +240,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_faeriearchivecom.py b/fanficfare/adapters/adapter_faeriearchivecom.py
index f96b3704..c6b06181 100644
--- a/fanficfare/adapters/adapter_faeriearchivecom.py
+++ b/fanficfare/adapters/adapter_faeriearchivecom.py
@@ -90,7 +90,7 @@ class FaerieArchiveComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -195,7 +195,7 @@ class FaerieArchiveComAdapter(BaseSiteAdapter):
# Find Series name from series URL.
series_url = 'http://'+self.host+'/'+series['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -219,7 +219,7 @@ class FaerieArchiveComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_fanficauthorsnet.py b/fanficfare/adapters/adapter_fanficauthorsnet.py
index d61a20fe..9786d61c 100644
--- a/fanficfare/adapters/adapter_fanficauthorsnet.py
+++ b/fanficfare/adapters/adapter_fanficauthorsnet.py
@@ -197,7 +197,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
self.story.setMetadata('authorUrl','https://{0}/'.format(self.parsedUrl.netloc))
loginUrl = self.story.getMetadata('authorUrl')+'account/'
- loginsoup = self.make_soup(self._fetchUrl(loginUrl))
+ loginsoup = self.make_soup(self.get_request(loginUrl))
if True:
# if self.performLogin(loginUrl, loginsoup):
# Now go hunting for all the meta data and the chapter list.
@@ -266,7 +266,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
def getChapterText(self, url):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
story = soup.find('div',{'class':'story'})
diff --git a/fanficfare/adapters/adapter_fanficcastletvnet.py b/fanficfare/adapters/adapter_fanficcastletvnet.py
index d3ae2d1f..adb374f8 100644
--- a/fanficfare/adapters/adapter_fanficcastletvnet.py
+++ b/fanficfare/adapters/adapter_fanficcastletvnet.py
@@ -138,7 +138,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -148,7 +148,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
if m != None:
@@ -163,7 +163,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
logger.debug("URL 2nd try: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -287,7 +287,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
series_url = 'http://'+self.host+'/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -306,7 +306,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_fanfichu.py b/fanficfare/adapters/adapter_fanfichu.py
index 4b6bdc27..6bad7c93 100644
--- a/fanficfare/adapters/adapter_fanfichu.py
+++ b/fanficfare/adapters/adapter_fanfichu.py
@@ -73,7 +73,7 @@ class FanficHuAdapter(BaseSiteAdapter):
return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]).replace('https','https?') + r'\d+$'
def extractChapterUrlsAndMetadata(self):
- soup = self.make_soup(self._fetchUrl(self.url + '&i=1'))
+ soup = self.make_soup(self.get_request(self.url + '&i=1'))
if ensure_text(soup.title.string).strip(u' :') == u'írta':
raise exceptions.StoryDoesNotExist(self.url)
@@ -91,7 +91,7 @@ class FanficHuAdapter(BaseSiteAdapter):
self.add_chapter(option.string, url)
author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
- soup = self.make_soup(self._fetchUrl(author_url))
+ soup = self.make_soup(self.get_request(author_url))
story_id = self.story.getMetadata('storyId')
for table in soup('table', {'class': 'mainnav'}):
@@ -180,7 +180,7 @@ class FanficHuAdapter(BaseSiteAdapter):
raise exceptions.AdultCheckRequired(self.url)
def getChapterText(self, url):
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
story_cell = soup.find('form', action='viewstory.php').parent.parent
for div in story_cell('div'):
diff --git a/fanficfare/adapters/adapter_fanficsme.py b/fanficfare/adapters/adapter_fanficsme.py
index 4f06cbd3..10b873ea 100644
--- a/fanficfare/adapters/adapter_fanficsme.py
+++ b/fanficfare/adapters/adapter_fanficsme.py
@@ -113,7 +113,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
params['name']))
## must need a cookie or something.
- self._fetchUrl(loginUrl, usecache=False)
+ self.get_request(loginUrl, usecache=False)
d = self.post_request(loginUrl, params, usecache=False)
if self.needToLoginCheck(d):
@@ -138,7 +138,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
logger.info("url: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -170,7 +170,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
if self.story.getMetadata('rating') != 'General' and self.needToLoginCheck(data):
self.performLogin(url)
# reload after login.
- data = self._fetchUrl(url,usecache=False)
+ data = self.get_request(url,usecache=False)
soup = self.make_soup(data)
fichead = soup.find('div',class_='FicHead')
@@ -325,7 +325,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
logger.debug("USE view_full_work")
## Assumed view_adult=true was cookied during metadata
if not self.full_work_soup:
- self.full_work_soup = self.make_soup(self._fetchUrl(
+ self.full_work_soup = self.make_soup(self.get_request(
'https://' + self.getSiteDomain() + '/read.php?id='+self.story.getMetadata('storyId')))
whole_dl_soup = self.full_work_soup
@@ -334,7 +334,7 @@ class FanFicsMeAdapter(BaseSiteAdapter):
self.use_full_work_soup = False
logger.warning("c%s not found in view_full_work--ending use_view_full_work"%(index))
if chapter_div == None:
- whole_dl_soup = self.make_soup(self._fetchUrl(url))
+ whole_dl_soup = self.make_soup(self.get_request(url))
chapter_div = whole_dl_soup.find('div',{'id':'c%s'%(index)})
if None == chapter_div:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
diff --git a/fanficfare/adapters/adapter_fanfictalkcom.py b/fanficfare/adapters/adapter_fanfictalkcom.py
index e04d222e..928c890a 100644
--- a/fanficfare/adapters/adapter_fanfictalkcom.py
+++ b/fanficfare/adapters/adapter_fanfictalkcom.py
@@ -103,7 +103,7 @@ class FanfictalkComAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -197,7 +197,7 @@ class FanfictalkComAdapter(BaseSiteAdapter):
series_name = stripHTML(seriesa)
series_url = 'https://'+self.host+'/archive/'+seriesa['href']
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.find_all('a', href=re.compile(r'viewstory.php\?sid=\d+'))
# logger.debug(storyas)
j=1
@@ -224,7 +224,7 @@ class FanfictalkComAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_fanfictionjunkiesde.py b/fanficfare/adapters/adapter_fanfictionjunkiesde.py
index c51b8101..f2cb1e00 100644
--- a/fanficfare/adapters/adapter_fanfictionjunkiesde.py
+++ b/fanficfare/adapters/adapter_fanfictionjunkiesde.py
@@ -147,7 +147,7 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -157,7 +157,7 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url)
+ data = self.get_request(url)
# The actual text that is used to announce you need to be an
# adult varies from site to site. Again, print data before
@@ -256,7 +256,7 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
series_url = 'http://'+self.host+'/efiction/'+a['href']
# use BeautifulSoup HTML parser to make everything easier to find.
- seriessoup = self.make_soup(self._fetchUrl(series_url))
+ seriessoup = self.make_soup(self.get_request(series_url))
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
i=1
for a in storyas:
@@ -275,7 +275,7 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'story'})
diff --git a/fanficfare/adapters/adapter_fanfictionnet.py b/fanficfare/adapters/adapter_fanfictionnet.py
index 4c750f1e..8327a14f 100644
--- a/fanficfare/adapters/adapter_fanfictionnet.py
+++ b/fanficfare/adapters/adapter_fanfictionnet.py
@@ -72,11 +72,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
def getSiteURLPattern(self):
return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
- def _fetchUrl(self,url,extrasleep=1.0,usecache=True):
+ def get_request(self,url,extrasleep=1.0,usecache=True):
## ffnet(and, I assume, fpcom) tends to fail more if hit too
## fast. This is in additional to what ever the
## slow_down_sleep_time setting is.
- return BaseSiteAdapter._fetchUrl(self,url,
+ return BaseSiteAdapter.get_request(self,url,
extrasleep=extrasleep,
usecache=usecache)
@@ -104,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
# use BeautifulSoup HTML parser to make everything easier to find.
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
#logger.debug("\n===================\n%s\n===================\n"%data)
soup = self.make_soup(data)
except HTTPError as e:
@@ -142,7 +142,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
chapcount+1,
self.urltitle)
logger.debug('=Trying newer chapter: %s' % tryurl)
- newdata = self._fetchUrl(tryurl)
+ newdata = self.get_request(tryurl)
if "not found. Please check to see you are not using an outdated url." not in newdata \
and "This request takes too long to process, it is timed out by the server." not in newdata:
logger.debug('=======Found newer chapter: %s' % tryurl)
@@ -177,7 +177,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
self.story.addToList('category',stripHTML(categories[1]))
elif 'Crossover' in categories[0]['href']:
caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
- catsoup = self.make_soup(self._fetchUrl(caturl))
+ catsoup = self.make_soup(self.get_request(caturl))
found = False
for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
self.story.addToList('category',stripHTML(a))
@@ -309,7 +309,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
authimg_url = ""
if cover_url and self.getConfig('skip_author_cover'):
- authsoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+ authsoup = self.make_soup(self.get_request(self.story.getMetadata('authorUrl')))
try:
img = authsoup.select_one('img.lazy.cimage')
authimg_url=img['data-original']
@@ -373,7 +373,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
## AND explicitly put title URL back on chapter URL for fetch
## *only*--normalized chapter URL does NOT have urltitle
- data = self._fetchUrl(url+self.urltitle,
+ data = self.get_request(url+self.urltitle,
extrasleep=4.0)
if "Please email this error message in full to support@fanfiction.com" in data:
diff --git a/fanficfare/adapters/adapter_fanfiktionde.py b/fanficfare/adapters/adapter_fanfiktionde.py
index 3507e3ea..5ca18df4 100644
--- a/fanficfare/adapters/adapter_fanfiktionde.py
+++ b/fanficfare/adapters/adapter_fanfiktionde.py
@@ -119,7 +119,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -129,7 +129,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
if self.needToLoginCheck(data):
# need to log in for this one.
self.performLogin(url)
- data = self._fetchUrl(url,usecache=False)
+ data = self.get_request(url,usecache=False)
if "Uhr ist diese Geschichte nur nach einer" in data:
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
@@ -192,11 +192,11 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
## Get description from own URL:
## /?a=v&storyid=46ccbef30000616306614050&s=1
- descsoup = self.make_soup(self._fetchUrl("https://"+self.getSiteDomain()+"/?a=v&storyid="+self.story.getMetadata('storyId')+"&s=1"))
+ descsoup = self.make_soup(self.get_request("https://"+self.getSiteDomain()+"/?a=v&storyid="+self.story.getMetadata('storyId')+"&s=1"))
self.setDescription(url,stripHTML(descsoup))
# #find metadata on the author's page
- # asoup = self.make_soup(self._fetchUrl("https://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
+ # asoup = self.make_soup(self.get_request("https://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
# tr=asoup.findAll('tr')
# for i in range(1,len(tr)):
# a = tr[i].find('a')
@@ -217,7 +217,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
time.sleep(0.5) ## ffde has "floodlock" protection
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
div = soup.find('div', {'id' : 'storytext'})
for a in div.findAll('script'):
diff --git a/fanficfare/adapters/adapter_fastnovelnet.py b/fanficfare/adapters/adapter_fastnovelnet.py
index cd2813f3..4d608f71 100644
--- a/fanficfare/adapters/adapter_fastnovelnet.py
+++ b/fanficfare/adapters/adapter_fastnovelnet.py
@@ -80,7 +80,7 @@ class FastNovelNetAdapter(BaseSiteAdapter):
logger.debug('URL: %s', self.url)
try:
- data = self._fetchUrl(self.url)
+ data = self.get_request(self.url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist('404 error: {}'.format(self.url))
@@ -139,7 +139,7 @@ class FastNovelNetAdapter(BaseSiteAdapter):
self.add_chapter(title, 'https://' + self.host + a["href"])
def getChapterText(self, url):
- data = self._fetchUrl(url)
+ data = self.get_request(url)
soup = self.make_soup(data)
story = soup.select_one('#chapter-body')
diff --git a/fanficfare/adapters/adapter_ficbooknet.py b/fanficfare/adapters/adapter_ficbooknet.py
index 48d029bf..51b22af8 100644
--- a/fanficfare/adapters/adapter_ficbooknet.py
+++ b/fanficfare/adapters/adapter_ficbooknet.py
@@ -85,7 +85,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
url=self.url
logger.debug("URL: "+url)
try:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -176,7 +176,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
## after site change, I don't see word count anywhere.
# pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
# pr='https://'+self.host+pr['href']
- # pr = self.make_soup(self._fetchUrl(pr))
+ # pr = self.make_soup(self.get_request(pr))
# pr=pr.findAll('div', {'class' : 'part_text'})
# i=0
# for part in pr:
@@ -241,7 +241,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
- soup = self.make_soup(self._fetchUrl(url))
+ soup = self.make_soup(self.get_request(url))
chapter = soup.find('div', {'id' : 'content'})
if chapter == None: ## still needed?
diff --git a/fanficfare/adapters/adapter_fictionalleyorg.py b/fanficfare/adapters/adapter_fictionalleyorg.py
index 2c7ae4bd..9f01f881 100644
--- a/fanficfare/adapters/adapter_fictionalleyorg.py
+++ b/fanficfare/adapters/adapter_fictionalleyorg.py
@@ -70,7 +70,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
## posting on list doesn't work, but doesn't hurt, either.
data = self.post_request(url,params)
else:
- data = self._fetchUrl(url)
+ data = self.get_request(url)
return data
def extractChapterUrlsAndMetadata(self):
@@ -108,7 +108,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
logger.debug("Normalizing to URL: "+url)
## title's right there...
self.story.setMetadata('title',stripHTML(storya))
- data = self._fetchUrl(url)
+ data = self.get_request(url)
soup = self.make_soup(data)
chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
else:
@@ -138,7 +138,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
## Go scrape the rest of the metadata from the author's page.
- data = self._fetchUrl(self.story.getMetadata('authorUrl'))
+ data = self.get_request(self.story.getMetadata('authorUrl'))
soup = self.make_soup(data)
#