Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2026-05-08 21:11:59 +02:00
Code Issues Projects Releases Wiki Activity

Refactor _fetchUrl()s implicit POST to explicit post_request()s

Browse source
This commit is contained in:
Jim Miller 2021-01-23 14:17:18 -06:00
parent b948591389
commit 7c262e71fa
30 changed files with 40 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
fanficfare/adapters/adapter_bloodtiesfancom.py
View file

@ -118,7 +118,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_buffygilescom.py
View file

@ -95,7 +95,7 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_deepinmysoulnet.py
View file

@ -95,7 +95,7 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

4
fanficfare/adapters/adapter_dokugacom.py
View file

@ -104,10 +104,10 @@ class DokugaComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['username']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Your session has expired. Please log in again." in d:
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Logout" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_dracoandginnycom.py
View file

@ -94,7 +94,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_efpfanficnet.py
View file

@ -93,7 +93,7 @@ class EFPFanFicNet(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if '<a class="menu" href="newaccount.php">' in d : # register for new account link
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_fanficauthorsnet.py
View file

@ -166,7 +166,7 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
raise exceptions.FailedToLogin('You need to have your username and password set.',params['username'])
try:
data = self._fetchUrl(url+'index/', params, usecache=False)
data = self.post_request(url+'index/', params, usecache=False)
except HTTPError as e:
if e.code == 404:
raise exceptions.StoryDoesNotExist("Code: 404. {0}".format(url))

2
fanficfare/adapters/adapter_fanficcastletvnet.py
View file

@ -110,7 +110,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_fanfictionjunkiesde.py
View file

@ -119,7 +119,7 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

3
fanficfare/adapters/adapter_fanfictionnet.py
View file

@ -72,12 +72,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
def getSiteURLPattern(self):
return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
def _fetchUrl(self,url,parameters=None,extrasleep=1.0,usecache=True):
def _fetchUrl(self,url,extrasleep=1.0,usecache=True):
## ffnet(and, I assume, fpcom) tends to fail more if hit too
## fast. This is in additional to what ever the
## slow_down_sleep_time setting is.
return BaseSiteAdapter._fetchUrl(self,url,
parameters=parameters,
extrasleep=extrasleep,
usecache=usecache)

2
fanficfare/adapters/adapter_iketernalnet.py
View file

@ -94,7 +94,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_imagineeficcom.py
View file

@ -94,7 +94,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_merlinficdtwinscouk.py
View file

@ -94,7 +94,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_occlumencysycophanthexcom.py
View file

@ -97,7 +97,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Logout" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_phoenixsongnet.py
View file

@ -91,7 +91,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
loginUrl = 'https://' + self.getSiteDomain() + '/users/processlogin.php'
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['txtusername']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if 'Please login to continue.' in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_potterheadsanonymouscom.py
View file

@ -94,7 +94,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_samandjacknet.py
View file

@ -118,7 +118,7 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_sheppardweircom.py
View file

@ -118,7 +118,7 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

6
fanficfare/adapters/adapter_storiesonlinenet.py
View file

@ -139,7 +139,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
postAction,
'','',''))
# try:
data = self._fetchUrl(postUrl,params,usecache=False)
data = self.post_request(postUrl,params,usecache=False)
# logger.debug(data)
# except HTTPError as e:
# if e.code == 307:
@ -161,7 +161,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
# '','',''))
try:
data = self._fetchUrl(postUrl,params,usecache=False)
data = self.post_request(postUrl,params,usecache=False)
# logger.debug(data)
except HTTPError as e:
if e.code == 307:
@ -169,7 +169,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
try:
## need to hit redirect URL so cookies get set for
## the story site domain. I think.
data = self._fetchUrl(e.headers['Location'],params,usecache=False)
data = self.post_request(e.headers['Location'],params,usecache=False)
except HTTPError as e:
if e.code == 307:
# logger.debug(e)

2
fanficfare/adapters/adapter_tenhawkpresents.py
View file

@ -91,7 +91,7 @@ class TenhawkPresentsSiteAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_themasquenet.py
View file

@ -101,7 +101,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_thepetulantpoetesscom.py
View file

@ -93,7 +93,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "My Account Page" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_trekiverseorg.py
View file

@ -97,7 +97,7 @@ class TrekiverseOrgAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_tthfanficorg.py
View file

@ -106,7 +106,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
params['ctkn']=soup.find('input', {'name':'ctkn'})['value']
params[soup.find('input', {'id':'password'})['name']] = params['password']
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Stories Published" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

2
fanficfare/adapters/adapter_twilightednet.py
View file

@ -85,7 +85,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['penname']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Member Account" not in d : #Member Account
logger.info("Failed to login to URL %s as %s" % (loginUrl,

4
fanficfare/adapters/adapter_wwwlushstoriescom.py
View file

@ -107,13 +107,11 @@ class WWWLushStoriesComAdapter(BaseSiteAdapter): # XXX
return r"http(s)?://www\.lushstories\.com/stories/(?P<category>[^/]+)/(?P<id>.+?)\.aspx"
################################################################################################
def _fetchUrl(self,url,parameters=None,extrasleep=None,usecache=True):
def _fetchUrl(self,url,usecache=True):
## lushstories.com sets unescaped cookies with cause
## httplib.py to fail.
self.get_configuration().set_cookiejar(self.get_configuration().get_empty_cookiejar())
return BaseSiteAdapter._fetchUrl(self,url,
parameters=parameters,
extrasleep=extrasleep,
usecache=usecache)
################################################################################################
def get_page(self, page):

2
fanficfare/adapters/base_efiction_adapter.py
View file

@ -260,7 +260,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (self.getLoginUrl(), params['penname']))
d = self._fetchUrl(self.getLoginUrl(), params)
d = self.post_request(self.getLoginUrl(), params)
if self.getMessageMemberAccount() not in d : #Member Account
logger.info("Failed to login to URL <%s> as '%s'" % (self.getLoginUrl(), params['penname']))

2
fanficfare/adapters/base_xenforoforum_adapter.py
View file

@ -226,7 +226,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
params['login']))
d = self._fetchUrl(loginUrl, params)
d = self.post_request(loginUrl, params)
if "Log Out" not in d:
# logger.debug(d)

6
fanficfare/configurable.py
View file

@ -1001,9 +1001,9 @@ class Configurable(object):
def post_request(self, url,
parameters={},
usecache=True):
return self.configuration.fetcher._postUrl(url,
parameters=parameters,
usecache=usecache)
return self.configuration.fetcher.post_request(url,
parameters=parameters,
usecache=usecache)
def _fetchUrlOpened(self, url,
parameters=None,

12
fanficfare/fetcher.py
View file

@ -73,7 +73,7 @@ class Fetcher(object):
self.save_cache_file = None
self.save_cookiejar_file = None
def get_empty_cookiejar(self):
return cl.LWPCookieJar()
@ -240,11 +240,11 @@ class Fetcher(object):
time.sleep(rt)
# Assumes application/x-www-form-urlencoded. parameters, headers are dict()s
def _postUrl(self, url,
parameters={},
headers={},
extrasleep=None,
usecache=True):
def post_request(self, url,
parameters={},
headers={},
extrasleep=None,
usecache=True):
'''
When should cache be cleared or not used? logins...