Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2025-12-15 21:32:28 +01:00
Code Issues Projects Releases Wiki Activity

Abstract adapter.chapterUrls into base_adapter.

Browse source
This commit is contained in:
Jim Miller 2018-07-12 14:03:32 -05:00
parent f29f86d4f4
commit 8381dba465
115 changed files with 219 additions and 321 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
fanficfare/adapters/adapter_adastrafanficcom.py
View file

@ -103,9 +103,8 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
## <meta name='description' content='&lt;p&gt;Description&lt;/p&gt; ...' >
## Summary, strangely, is in the content attr of a <meta name='description'> tag

3
fanficfare/adapters/adapter_adultfanfictionorg.py
View file

@ -232,9 +232,8 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter):
# Find the chapters:
chapters = soup.find('div',{'class':'dropdown-content'})
for i, chapter in enumerate(chapters.findAll('a')):
self.chapterUrls.append((stripHTML(chapter),self.url+'&chapter='+str(i+1)))
self.add_chapter(chapter,self.url+'&chapter='+str(i+1))
self.story.setMetadata('numChapters', len(self.chapterUrls))
# Find authorid and URL from... author url.
a = soup.find('a', href=re.compile(r"profile.php\?no=\d+"))

4
fanficfare/adapters/adapter_andromedawebcom.py
View file

@ -192,9 +192,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl)
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

10
fanficfare/adapters/adapter_archiveofourownorg.py
View file

@ -236,11 +236,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
self.story.setMetadata('numChapters',len(chapters))
logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
if len(chapters)==1:
self.chapterUrls.append((self.story.getMetadata('title'),'https://'+self.host+chapters[0]['href']))
self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+chapters[0]['href'])
else:
for index, chapter in enumerate(chapters):
# strip just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+chapter['href']))
self.add_chapter(chapter,'https://'+self.host+chapter['href'])
# (2013-09-21)
date = stripHTML(chapter.findNext('span'))[1:-1]
chapterDate = makeDate(date,self.dateformat)
@ -389,7 +389,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
whole_dl_soup = chapter_dl_soup = None
if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and len(self.chapterUrls) > 1:
if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and self.num_chapters() > 1:
logger.debug("USE view_full_work")
## Assumed view_adult=true was cookied during metadata
if not self.full_work_soup:
@ -471,7 +471,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
## pages and after chapter-# div. Appending removes
## headnotes from whole_dl_soup, so be sure to only do it on
## the last chapter.
if 'authorfootnotes' not in exclude_notes and index+1 == len(self.chapterUrls):
if 'authorfootnotes' not in exclude_notes and index+1 == self.num_chapters():
footnotes = whole_dl_soup.find('div', {'id' : "work_endnotes"})
if footnotes != None:
footnotes = footnotes.find('blockquote')
@ -479,7 +479,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
append_tag(save_chapter,'b',"Author's Note:")
save_chapter.append(footnotes)
if 'inspiredlinks' not in exclude_notes and index+1 == len(self.chapterUrls):
if 'inspiredlinks' not in exclude_notes and index+1 == self.num_chapters():
inspiredlinks = whole_dl_soup.find('div', {'id' : "children"})
if inspiredlinks != None:
if inspiredlinks:

3
fanficfare/adapters/adapter_archiveskyehawkecom.py
View file

@ -109,9 +109,8 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
for i in range(1,len(chapter)):
ch=chapter[i]
self.chapterUrls.append((stripHTML(ch),ch['value']))
self.add_chapter(ch,ch['value'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_artemisfowlcom.py
View file

@ -192,9 +192,8 @@ class ArtemisFowlComAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/fanfiction/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

8
fanficfare/adapters/adapter_asexstoriescom.py
View file

@ -117,12 +117,9 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
description = description.encode('utf-8','ignore').strip()[0:150].decode('utf-8','ignore')
self.setDescription(url,'Excerpt from beginning of story: '+description+'...')
# Get chapter URLs
self.chapterUrls = []
### The first 'chapter' is not listed in the links, so we have to
### add it before the rest of the pages, if any
self.chapterUrls.append(('1', self.url))
self.add_chapter('1', self.url)
chapterTable = soup1.find('div',{'class':'pages'}).findAll('a')
@ -133,9 +130,8 @@ class ASexStoriesComAdapter(BaseSiteAdapter):
chapterTitle = page.string
chapterUrl = urlparse.urljoin(self.url, page['href'])
if chapterUrl.startswith(self.url): # there are other URLs in the pages block now.
self.chapterUrls.append((chapterTitle, chapterUrl))
self.add_chapter(chapterTitle, chapterUrl)
self.story.setMetadata('numChapters', len(self.chapterUrls))
rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip()
self.story.setMetadata('rating',rated)

5
fanficfare/adapters/adapter_ashwindersycophanthexcom.py
View file

@ -161,13 +161,12 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
# Find the chapters:
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
if len(chapters) == 0:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

2
fanficfare/adapters/adapter_asianfanficscom.py
View file

@ -168,7 +168,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter):
self.story.setMetadata('numChapters',len(chapters))
for index, chapter in enumerate(chapters):
if chapter.text != 'Foreword': # skip the foreword
self.chapterUrls.append((stripHTML(chapter.text),'https://' + self.getSiteDomain() + chapter['value'])) # note: AFF cuts off chapter names in list. this gets kind of fixed later on
self.add_chapter(chapter.text,'https://' + self.getSiteDomain() + chapter['value']) # note: AFF cuts off chapter names in list. this gets kind of fixed later on
# find timestamp
a = soup.find('span', text='Updated')
if a == None:

3
fanficfare/adapters/adapter_asr3slashzoneorg.py
View file

@ -140,9 +140,8 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/archive/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_bdsmlibrarycom.py
View file

@ -155,9 +155,8 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter):
for chapter in soup.findAll('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+"&chapterid=\d+$")):
value = chapter.findNext('td').findNext('td').string.replace('(added on','').replace(')','').strip()
self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
self.chapterUrls.append((stripHTML(chapter),'https://'+self.getSiteDomain()+chapter['href']))
self.add_chapter(chapter,'https://'+self.getSiteDomain()+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# Get the MetaData
# Erotia Tags

4
fanficfare/adapters/adapter_bloodshedversecom.py
View file

@ -85,11 +85,11 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
for option in soup.find('select', {'name': 'chapter'}):
title = stripHTML(option)
url = self.READ_URL_TEMPLATE % option['value']
self.chapterUrls.append((title, url))
self.add_chapter(title, url)
# Reset the storyId to be the first chapter no. Needed
# because emails contain link to later chapters instead.
query_data = urlparse.parse_qs(self.chapterUrls[0][1])
query_data = urlparse.parse_qs(self.get_chapter(0,'url'))
story_no = query_data['no'][0]
self.story.setMetadata('storyId', story_no)

3
fanficfare/adapters/adapter_bloodtiesfancom.py
View file

@ -214,9 +214,8 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_buffygilescom.py
View file

@ -190,9 +190,8 @@ class BuffyGilesComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/efiction/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_chaossycophanthexcom.py
View file

@ -126,9 +126,8 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

5
fanficfare/adapters/adapter_chosentwofanficcom.py
View file

@ -119,10 +119,9 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
#self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.chapterUrls.append((stripHTML(chapter),'http://{0}/{1}{2}'.format(self.host, chapter['href'],addURL)))
#self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.add_chapter(chapter,'http://{0}/{1}{2}'.format(self.host, chapter['href'],addURL))
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_csiforensicscom.py
View file

@ -128,9 +128,8 @@ class CSIForensicsComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_deepinmysoulnet.py
View file

@ -189,9 +189,8 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/fiction/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/fiction/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_destinysgatewaycom.py
View file

@ -138,9 +138,8 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

5
fanficfare/adapters/adapter_dokugacom.py
View file

@ -164,13 +164,12 @@ class DokugaComAdapter(BaseSiteAdapter):
# Find the chapters:
chapters = soup.find('select').findAll('option')
if len(chapters)==1:
self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1'))
self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1')
else:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
self.add_chapter(chapter,'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
asoup = self.make_soup(self._fetchUrl(alink))

3
fanficfare/adapters/adapter_dracoandginnycom.py
View file

@ -177,9 +177,8 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_dramioneorg.py
View file

@ -175,9 +175,8 @@ class DramioneOrgAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_efictionestelielde.py
View file

@ -111,9 +111,8 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

7
fanficfare/adapters/adapter_efpfanficnet.py
View file

@ -145,7 +145,7 @@ class EFPFanFicNet(BaseSiteAdapter):
if select is None:
# no selector found, so it's a one-chapter story.
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
for o in allOptions:
@ -153,13 +153,12 @@ class EFPFanFicNet(BaseSiteAdapter):
o['value'])
# just in case there's tags, like <i> in chapter titles.
title = stripHTML(o)
self.chapterUrls.append((title,url))
self.add_chapter(title,url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.story.setMetadata('language','Italian')
# normalize story URL to first chapter if later chapter URL was given:
url = self.chapterUrls[0][1].replace('&i=1','')
url = self.get_chapter(0,'url').replace('&i=1','')
logger.debug("Normalizing to URL: "+url)
self._setURL(url)
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])

3
fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py
View file

@ -142,9 +142,8 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_fanficauthorsnet.py
View file

@ -223,9 +223,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter):
else:
chapter_title = stripHTML(unicode(chapter))
chapter_Url = self.story.getMetadata('authorUrl')+chapter['href'][1:]
self.chapterUrls.append((chapter_title, chapter_Url))
self.add_chapter(chapter_title, chapter_Url)
self.story.setMetadata('numChapters', len(self.chapterUrls))
genres = ("Drama","Romance")
gotgenre = False
## Getting the Metadata that is there

3
fanficfare/adapters/adapter_fanficcastletvnet.py
View file

@ -199,9 +199,8 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

2
fanficfare/adapters/adapter_fanfichu.py
View file

@ -97,7 +97,7 @@ class FanficHuAdapter(BaseSiteAdapter):
for option in chapter_options:
url = urlparse.urljoin(self.url, option['value'])
self.chapterUrls.append((option.string, url))
self.add_chapter(option.string, url)
author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
soup = self._customized_fetch_url(author_url)

3
fanficfare/adapters/adapter_fanfictionjunkiesde.py
View file

@ -192,9 +192,8 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/efiction/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

5
fanficfare/adapters/adapter_fanfictionnet.py
View file

@ -330,7 +330,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
if select is None:
# no selector found, so it's a one-chapter story.
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
allOptions = select.findAll('option')
for o in allOptions:
@ -340,9 +340,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
# just in case there's tags, like <i> in chapter titles.
title = u"%s" % o
title = re.sub(r'<[^>]+>','',title)
self.chapterUrls.append((title,url))
self.add_chapter(title,url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
return

3
fanficfare/adapters/adapter_fanfiktionde.py
View file

@ -151,9 +151,8 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.find('select').findAll('option'):
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
self.add_chapter(chapter,'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.story.setMetadata('numWords',stripHTML(soup.find("span",title="Wörter").parent))
self.story.setMetadata('language','German')

4
fanficfare/adapters/adapter_ficbooknet.py
View file

@ -122,7 +122,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
for x in range(0,len(chapters)):
chapter=chapters[x]
churl='https://'+self.host+chapter['href']
self.chapterUrls.append((stripHTML(chapter),churl))
self.add_chapter(chapter,churl)
## First chapter doesn't always have a date, skip it.
if pubdate == None and chapter.parent.find('span'):
pubdate = translit.translit(stripHTML(chapter.parent.find('span')))
@ -131,7 +131,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
update = translit.translit(stripHTML(chapter.parent.find('span')))
# update = translit.translit(stripHTML(self.make_soup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
else:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
self.story.setMetadata('numChapters',1)
pubdate=translit.translit(stripHTML(soup.find('div',{'class':'title-area'}).find('span')))
update=pubdate

5
fanficfare/adapters/adapter_fictionalleyorg.py
View file

@ -128,14 +128,13 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
self.story.setMetadata('authorUrl',authora['href'])
if len(chapterlinklist) == 1:
self.chapterUrls.append((self.story.getMetadata('title'),chapterlinklist[0]['href']))
self.add_chapter(self.story.getMetadata('title'),chapterlinklist[0]['href'])
else:
# Find the chapters:
for chapter in chapterlinklist:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),chapter['href']))
self.add_chapter(chapter,chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
## Go scrape the rest of the metadata from the author's page.
data = self._fetchUrl(self.story.getMetadata('authorUrl'))

4
fanficfare/adapters/adapter_fictionhuntcom.py
View file

@ -130,8 +130,8 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter):
# alternative is to get the num of chaps from the last
# indiated chapter list instead.
for i in range(1,1+int(self.story.getMetadata('numChapters'))):
self.chapterUrls.append(("Chapter "+unicode(i),"http://"+self.getSiteDomain()\
+"/read/"+self.story.getMetadata('storyId')+"/%s"%i))
self.add_chapter("Chapter "+unicode(i),"http://"+self.getSiteDomain()\
+"/read/"+self.story.getMetadata('storyId')+"/%s"%i)
def getChapterText(self, url):
logger.debug('Getting chapter text from: %s' % url)

2
fanficfare/adapters/adapter_fictionmaniatv.py
View file

@ -79,7 +79,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
if key == 'Title':
self.story.setMetadata('title', value)
self.chapterUrls.append((value, self.url))
self.add_chapter(value, self.url)
elif key == 'File Name':
self.story.setMetadata('fileName', value)

3
fanficfare/adapters/adapter_fictionpadcom.py
View file

@ -176,9 +176,8 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
chtitle = "Chapter %d"%c['number']
if c['title']:
chtitle += " - %s"%c['title']
self.chapterUrls.append((chtitle,c['body_url']))
self.add_chapter(chtitle,c['body_url'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
def getChapterText(self, url):
logger.debug('Getting chapter text from: %s' % url)

11
fanficfare/adapters/adapter_ficwadcom.py
View file

@ -202,7 +202,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
storylistul = soup.find('ul',{'class':'storylist'})
if not storylistul:
# no list found, so it's a one-chapter story.
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
chapterlistlis = storylistul.findAll('li')
for chapterli in chapterlistlis:
@ -211,12 +211,9 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
raise exceptions.FailedToLogin(url,self.username)
else:
#print "chapterli.h4.a (%s)"%chapterli.h4.a
self.chapterUrls.append((chapterli.h4.a.string,
u'https://%s%s'%(self.getSiteDomain(),
chapterli.h4.a['href'])))
#print "self.chapterUrls:%s"%self.chapterUrls
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.add_chapter(chapterli.h4.a.string,
u'https://%s%s'%(self.getSiteDomain(),
chapterli.h4.a['href']))
return

3
fanficfare/adapters/adapter_fimfictionnet.py
View file

@ -147,9 +147,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
# Chapters
for chapter in soup.find('ul',{'class':'chapters'}).find_all('a',{'href':re.compile(r'.*/story/'+self.story.getMetadata('storyId')+r'/.*')}):
self.chapterUrls.append((stripHTML(chapter), 'https://'+self.host+chapter['href']))
self.add_chapter(chapter, 'https://'+self.host+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# Status
# In the case of Fimfiction, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'

2
fanficfare/adapters/adapter_fireflyfansnet.py
View file

@ -108,7 +108,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter):
# way to determine if there are other chapters to the same story, so you have
# to download them one at a time yourself. I'm also setting the status to
# complete
self.chapterUrls.append((self.story.getMetadata('title'), self.url))
self.add_chapter(self.story.getMetadata('title'), self.url)
self.story.setMetadata('numChapters', 1)
self.story.setMetadata('status', 'Completed')

2
fanficfare/adapters/adapter_fireflypopulliorg.py
View file

@ -129,7 +129,7 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter):
# This site has the entire story on one page, so we will be using the normalized URL as
# the chapterUrl and the Title as the chapter Title
self.chapterUrls.append((self.story.getMetadata('title'), url))
self.add_chapter(self.story.getMetadata('title'), url)
## i would take this out, as it is not really needed, but the calibre plugin uses it,
## so it's staying

3
fanficfare/adapters/adapter_gravitytalescom.py
View file

@ -163,9 +163,8 @@ class GravityTalesComSiteAdapter(BaseSiteAdapter):
for alink in chap_soup.find_all('a',href=re.compile(self.getSiteDomain())): # ignore anchor links
## Some stories have that same chapters in different sections
if alink['href'] not in found_chaps:
self.chapterUrls.append((stripHTML(alink),alink['href']))
self.add_chapter(alink,alink['href'])
found_chaps[alink['href']] = alink['href']
self.story.setMetadata('numChapters',len(self.chapterUrls))
if feedparser:
# Parse published and updated date from latest RSS feed entry. The RSS feed urls seems to appear due to

3
fanficfare/adapters/adapter_harrypotterfanfictioncom.py
View file

@ -110,9 +110,8 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
chapter_words+=int(tdstr)
chapter = tr.find('a')
chpt=re.sub(r'^.*?(\?chapterid=\d+).*?',r'\1',chapter['href'])
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/viewstory.php'+chpt))
self.add_chapter(chapter,'https://'+self.host+'/viewstory.php'+chpt)
#self.story.setMetadata('numWords',unicode(words))
self.story.setMetadata('numChapters',len(self.chapterUrls))
## Finding the metadata is a bit of a pain. Desc is the only thing this color.
desctable= soup.find('table',{'bgcolor':'#f0e8e8'})

5
fanficfare/adapters/adapter_hlfictionnet.py
View file

@ -109,11 +109,10 @@ class HLFictionNetAdapter(BaseSiteAdapter):
if chapters != None:
for chapter in chapters.findAll('option'):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
self.add_chapter(chapter,'https://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])
else:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))

3
fanficfare/adapters/adapter_hpfanficarchivecom.py
View file

@ -109,9 +109,8 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/stories/'+chapter['href']))
self.add_chapter(chapter,'https://'+self.host+'/stories/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_iketernalnet.py
View file

@ -189,9 +189,8 @@ class IkEternalNetAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_imagineeficcom.py
View file

@ -177,9 +177,8 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

5
fanficfare/adapters/adapter_inkbunnynet.py
View file

@ -188,10 +188,9 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter):
self.story.setMetadata('dateUpdated', makeDate(updated, self.dateformat))
self.story.setMetadata('datePublished', makeDate(updated, self.dateformat))
# This is a 1 story/page site, so we set the chapterUrls up with the story url and title
self.chapterUrls.append((self.story.getMetadata('title'), url))
# This is a 1 story/page site, so we set the chapter up with the story url and title
self.add_chapter(self.story.getMetadata('title'), url)
self.story.setMetadata('numChapters', len(self.chapterUrls))
if get_cover:
cover_img = soup.find('img', {'id':'magicbox'})

3
fanficfare/adapters/adapter_kiarepositorymujajinet.py
View file

@ -189,9 +189,8 @@ class KiaRepositoryMujajiNetAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/repository/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/repository/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_ksarchivecom.py
View file

@ -166,9 +166,8 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_lcfanficcom.py
View file

@ -167,8 +167,7 @@ class LCFanFicComSiteAdapter(BaseSiteAdapter):
## This is a 1 story/page site, so we'll keep the soup fo the getChapterText function
## the chapterUrl nd numChapters need to be set as well
self.html = data
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.add_chapter(self.story.getMetadata('title'),url)
# grab the text for an individual chapter.

3
fanficfare/adapters/adapter_lightnovelgatecom.py
View file

@ -158,7 +158,7 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter):
dt = row.find_all('span')[-1].string
cdates.append(makeDate(dt, '%b-%d-%y'))
clink = row.find('a')
self.chapterUrls.append((clink.string, clink['href']))
self.add_chapter(clink.string, clink['href'])
cdates.sort()
# dateUpdated in upper part show only date of last chapter, but if
@ -166,7 +166,6 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter):
# dates manually
self.story.setMetadata('dateUpdated', cdates[-1])
self.story.setMetadata('datePublished', cdates[0])
self.story.setMetadata('numChapters', len(self.chapterUrls))
## getting description
cdata = soup.select_one('#noidungm')

15
fanficfare/adapters/adapter_literotica.py
View file

@ -187,9 +187,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (self.url, authorurl))
if isSingleStory:
# self.chapterUrls = [(soup1.h1.string, self.url)]
# self.story.setMetadata('title', soup1.h1.string)
self.story.setMetadata('title', storyLink.text.strip('/'))
logger.debug('Title: "%s"' % storyLink.text.strip('/'))
self.story.setMetadata('description', urlTr.findAll("td")[1].text)
@ -198,7 +195,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
date = urlTr.findAll('td')[-1].text
self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
self.chapterUrls = [(storyLink.text, self.url)]
self.add_chapter(storyLink.text, self.url)
averrating = stripHTML(storyLink.parent)
## title (0.00)
averrating = averrating[averrating.rfind('(')+1:averrating.rfind(')')]
@ -218,7 +215,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
## Walk the chapters
chapterTr = seriesTr.nextSibling
self.chapterUrls = []
dates = []
descriptions = []
ratings = []
@ -242,7 +238,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
chapter = chapterLink.text[len(seriesTitle):].strip()
logger.debug('\tChapter: "%s"' % chapter)
if chapter == '':
chapter_title = 'Chapter %d' % (len(self.chapterUrls) + 1)
chapter_title = 'Chapter %d' % (self.num_chapters() + 1)
else:
separater_char = chapter[0]
logger.debug('\tseparater_char: "%s"' % separater_char)
@ -271,7 +267,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
logger.debug("Chapter Title: " + chapter_title)
logger.debug("Chapter description: " + description)
chapters.append((chapter_title, chapurl, description, pub_date))
# self.chapterUrls.append((chapter_title, chapurl))
# self.add_chapter(chapter_title, chapurl)
numrating = stripHTML(chapterLink.parent)
## title (0.00)
numrating = numrating[numrating.rfind('(')+1:numrating.rfind(')')]
@ -282,7 +278,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
chapters = sorted(chapters, key=lambda chapter: chapter[3])
for i, chapter in enumerate(chapters):
self.chapterUrls.append((chapter[0], chapter[1]))
self.add_chapter(chapter[0], chapter[1])
descriptions.append("%d. %s" % (i + 1, chapter[2]))
## Set the oldest date as publication date, the newest as update date
dates.sort()
@ -297,12 +293,11 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
self.story.setMetadata('averrating','%4.2f' % (sum(ratings) / float(len(ratings))))
# normalize on first chapter URL.
self._setURL(self.chapterUrls[0][1])
self._setURL(self.get_chapter(0,'url'))
# reset storyId to first chapter.
self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
self.story.setMetadata('numChapters', len(self.chapterUrls))
# Add the category from the breadcumb. This might duplicate a category already added.
self.story.addToList('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)

3
fanficfare/adapters/adapter_lotrgficcom.py
View file

@ -113,9 +113,8 @@ class LOTRgficComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in div.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
### Metadata is contained

3
fanficfare/adapters/adapter_lumossycophanthexcom.py
View file

@ -126,9 +126,8 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

2
fanficfare/adapters/adapter_masseffect2in.py
View file

@ -197,7 +197,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
self.story.extendList('warnings', [warning])
chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
self.chapterUrls.append((chapterTitle, url))
self.add_chapter(chapterTitle, url)
except ParsingError, error:
raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))

6
fanficfare/adapters/adapter_mcstoriescom.py
View file

@ -131,7 +131,6 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
if updatedate is not None: self.story.setMetadata('dateUpdated', updatedate)
# Get chapter URLs
self.chapterUrls = []
chapterTable = soup1.find('table', class_='index')
if chapterTable is not None:
@ -144,15 +143,14 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter):
link = chapterCell.a
chapterTitle = link.text
chapterUrl = urlparse.urljoin(self.url, link['href'])
self.chapterUrls.append((chapterTitle, chapterUrl))
self.add_chapter(chapterTitle, chapterUrl)
else:
# Single chapter
chapterDiv = soup1.find('div', class_='chapter')
chapterTitle = chapterDiv.a.text
chapterUrl = urlparse.urljoin(self.url, chapterDiv.a['href'])
self.chapterUrls = [(chapterTitle, chapterUrl)]
self.add_chapter(chapterTitle, chapterUrl)
self.story.setMetadata('numChapters', len(self.chapterUrls))
logger.debug("Story: <%s>", self.story)

3
fanficfare/adapters/adapter_mediaminerorg.py
View file

@ -147,9 +147,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
# Find the chapters - one-shot now have chapter list, too.
chap_p = soup.find('p',{'style':'margin-left:10px;'})
for (atag,aurl,name) in [ (x,x['href'],stripHTML(x)) for x in chap_p.find_all('a') ]:
self.chapterUrls.append((name,'https://'+self.host+aurl))
self.add_chapter(name,'https://'+self.host+aurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# category
# <a href="/fanfic/src.php/a/567">Ranma 1/2</a>

3
fanficfare/adapters/adapter_merlinficdtwinscouk.py
View file

@ -177,9 +177,8 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_midnightwhispers.py
View file

@ -170,9 +170,8 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_nationallibrarynet.py
View file

@ -114,10 +114,9 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
if len(chapters) > 0:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
break
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.story.setMetadata('status', 'Completed')
# <span class="label">Rated:</span> NC-17<br /> etc

3
fanficfare/adapters/adapter_ncisficcom.py
View file

@ -113,10 +113,9 @@ class NCISFicComAdapter(BaseSiteAdapter):
if len(chapters) > 0:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
break
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.story.setMetadata('status', 'Completed')
# <span class="label">Rated:</span> NC-17<br /> etc

3
fanficfare/adapters/adapter_nfacommunitycom.py
View file

@ -163,9 +163,8 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))

4
fanficfare/adapters/adapter_nocturnallightnet.py
View file

@ -88,7 +88,7 @@ class NocturnalLightNetAdapter(BaseSiteAdapter):
chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
for chapter_anchor in chapter_anchors:
url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
self.chapterUrls.append((chapter_anchor.string, url))
self.add_chapter(chapter_anchor.string, url)
author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
soup = self._customized_fetch_url(author_url)
@ -108,7 +108,7 @@ class NocturnalLightNetAdapter(BaseSiteAdapter):
# No chapter anchors found in the original story URL, so the story has
# only a single chapter.
if not chapter_anchors:
self.chapterUrls.append((title, self.url))
self.add_chapter(title, self.url)
for b_tag in listbox('b'):
key = b_tag.string.strip(':')

3
fanficfare/adapters/adapter_noveltrovecom.py
View file

@ -136,8 +136,7 @@ class NovelTroveComSiteAdapter(BaseSiteAdapter):
## This is a 1 story/page site, so we'll keep the soup for the getChapterText function
## the chapterUrl and numChapters need to be set as well
self.html = soup
self.chapterUrls.append((self.story.getMetadata('title'), url))
self.story.setMetadata('numChapters', len(self.chapterUrls))
self.add_chapter(self.story.getMetadata('title'), url)
self.story.setMetadata('status', 'Completed')
## Getting the non-standard title page entries

5
fanficfare/adapters/adapter_occlumencysycophanthexcom.py
View file

@ -161,13 +161,12 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
# Find the chapters:
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
if len(chapters) == 0:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

5
fanficfare/adapters/adapter_phoenixsongnet.py
View file

@ -142,7 +142,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
# Find the chapters:
chapters = soup.find('select')
if chapters == None:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
for b in soup.findAll('b'):
if b.text == "Updated":
date = b.nextSibling.string.split(': ')[1].split(',')
@ -152,7 +152,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
i = 0
chapters = chapters.findAll('option')
for chapter in chapters:
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+chapter['value']))
self.add_chapter(chapter,'https://'+self.host+chapter['value'])
if i == 0:
self.story.setMetadata('storyId',chapter['value'].split('/')[3])
head = self.make_soup(self._fetchUrl('https://'+self.host+chapter['value'])).findAll('b')
@ -170,7 +170,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
i = i+1
self.story.setMetadata('numChapters',len(self.chapterUrls))
asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))

3
fanficfare/adapters/adapter_ponyfictionarchivenet.py
View file

@ -143,9 +143,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_potionsandsnitches.py
View file

@ -90,9 +90,8 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/fanfiction/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
def defaultGetattr(d,k):
try:

5
fanficfare/adapters/adapter_potterficscom.py
View file

@ -235,19 +235,16 @@ class PotterFicsComAdapter(BaseSiteAdapter):
#we can go back to the table cell we found before
#get its last element and work backwards to find the last ordered list on the page
list = cell.contents[len(cell)-1].findPrevious('ol')
chapters = []
revs = 0
chnum = 0
for li in list:
chnum += 1
chTitle = unicode(chnum) + '. ' + li.a.b.string.strip()
chURL = makeAbsoluteURL(li.a['href'])
chapters.append((chTitle,chURL))
self.add_chapter(chTitle,chURL)
#Get reviews, add to total
revs += int(li.div.a.string.split()[0])
self.chapterUrls.extend(chapters)
self.story.setMetadata('numChapters', len(chapters))
self.story.setMetadata('reviews', revs)
#Now for the description... this may be tricky...

3
fanficfare/adapters/adapter_potterheadsanonymouscom.py
View file

@ -188,9 +188,8 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_pretendercentrecom.py
View file

@ -141,9 +141,8 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/missingpieces/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/missingpieces/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

5
fanficfare/adapters/adapter_qafficcom.py
View file

@ -141,11 +141,10 @@ class QafFicComAdapter(BaseSiteAdapter):
if chapters != None:
for chapter in chapters.findAll('option'):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
self.add_chapter(chapter,'https://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])
else:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))
for list in asoup.findAll('div', {'class' : re.compile('listbox')}):

3
fanficfare/adapters/adapter_quotevcom.py
View file

@ -122,9 +122,8 @@ class QuotevComAdapter(BaseSiteAdapter):
for a in soup.find('div', id='rselect')('a'):
if 'javascript' not in a['href']:
self.chapterUrls.append((a.get_text(), urlparse.urljoin(self.url, a['href'])))
self.add_chapter(a.get_text(), urlparse.urljoin(self.url, a['href']))
self.story.setMetadata('numChapters', len(self.chapterUrls))
def getChapterText(self, url):
data = self._fetchUrl(url)

3
fanficfare/adapters/adapter_royalroadl.py
View file

@ -166,9 +166,8 @@ class RoyalRoadAdapter(BaseSiteAdapter):
tds = [tr.findAll('td')[0] for tr in chapters.findAll('tr')]
for td in tds:
chapterUrl = 'https://' + self.getSiteDomain() + td.a['href']
self.chapterUrls.append((stripHTML(td.text), chapterUrl))
self.add_chapter(td.text, chapterUrl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# this is forum based so it's a bit ugly
description = soup.find('div', {'property': 'description', 'class': 'hidden-content'})

3
fanficfare/adapters/adapter_samandjacknet.py
View file

@ -222,9 +222,8 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/fanfics/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_scarvesandcoffeenet.py
View file

@ -146,9 +146,8 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_sheppardweircom.py
View file

@ -195,9 +195,8 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/fanfics/'+chapter['href']+addurl))
self.add_chapter(chapter,'https://'+self.host+'/fanfics/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

2
fanficfare/adapters/adapter_shriftweborgbfa.py
View file

@ -129,7 +129,7 @@ class BFAArchiveShriftwebOrgSiteAdapter(BaseSiteAdapter):
# This site has the entire story on one page, so we will be using the normalized URL as
# the chapterUrl and the Title as the chapter Title
self.chapterUrls.append((self.story.getMetadata('title'), url))
self.add_chapter(self.story.getMetadata('title'), url)
## i would take this out, as it is not really needed, but the calibre plugin uses it,
## so it's staying

9
fanficfare/adapters/adapter_siyecouk.py
View file

@ -118,13 +118,10 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
# Find the chapters (from soup, not authsoup):
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/siye/'+chapter['href'])
if self.chapterUrls:
self.story.setMetadata('numChapters',len(self.chapterUrls))
else:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.story.setMetadata('numChapters',1)
if self.num_chapters() < 1:
self.add_chapter(self.story.getMetadata('title'),url)
# The stuff we can get from the chapter list/one-shot page are
# in the first table with 95% width.

2
fanficfare/adapters/adapter_spikeluvercom.py
View file

@ -204,7 +204,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
title = stripHTML(chapter_anchor)
url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
self.chapterUrls.append((title, url))
self.add_chapter(title, url)
def getChapterText(self, url):
url += self.AGE_CONSENT_URL_SUFFIX

5
fanficfare/adapters/adapter_squidgeorgpeja.py
View file

@ -135,11 +135,10 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
chapterselect=soup.find('select',{'name':'chapter'})
if chapterselect:
for ch in chapterselect.findAll('option'):
self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
self.add_chapter(ch,'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value'])
else:
self.chapterUrls.append((title,url))
self.add_chapter(title,url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# utility method

5
fanficfare/adapters/adapter_storiesofardacom.py
View file

@ -104,12 +104,11 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
# Find the chapters: chapterview.asp?sid=7000&cid=30919
chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
if len(chapters)==1:
self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href'])
else:
for chapter in chapters:
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
summary = soup.find('td', {'colspan' : '3'})
summary.name='div' # change td to div. Makes Calibre

7
fanficfare/adapters/adapter_storiesonlinenet.py
View file

@ -193,11 +193,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
logger.debug("Number of chapters: {0}".format(len(chapters)))
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+chapter['href']))
self.add_chapter(chapter,'https://'+self.host+chapter['href'])
else:
self.chapterUrls.append((self.story.getMetadata('title'),'https://'+self.host+'/s/'+self.story.getMetadata('storyId')))
self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+'/s/'+self.story.getMetadata('storyId'))
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.getStoryMetadataFromAuthorPage()
@ -217,7 +216,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
# Remove all the metadata elements to leave and preamble text. This is usually
# a notice or a forward.
if len(self.chapterUrls) > 1:
if self.num_chapters() > 1:
header = soup.find('header')
header.extract()
else:

3
fanficfare/adapters/adapter_sugarquillnet.py
View file

@ -118,9 +118,8 @@ class SugarQuillNetAdapter(BaseSiteAdapter):
for chapter in chapters:
if chapter.string == 'Default':
chapter.string = 'Chapter 1'
self.chapterUrls.append((chapter.string, '{0}&chapno={1}'.format(self.url,chapter['value'])))
self.add_chapter(chapter.string, '{0}&chapno={1}'.format(self.url,chapter['value']))
self.story.setMetadata('numChapters',len(self.chapterUrls))
## This site doesn't have much metadata, so we will get what we can.
## The metadata is all on the author's page, so we have to get it to parse.

3
fanficfare/adapters/adapter_tenhawkpresentscom.py
View file

@ -147,9 +147,8 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
def defaultGetattr(d,k):
try:

64
fanficfare/adapters/adapter_test1.py
View file

@ -76,18 +76,13 @@ class TestSiteAdapter(BaseSiteAdapter):
self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr))
#print("set:%s->%s"%(key,self.story.getMetadata(key)))
self.chapterUrls = []
if self.has_config(sections,'chapter_urls'):
for l in self.get_config(sections,'chapter_urls').splitlines() :
if l:
self.chapterUrls.append( (l[1+l.index(','):],l[:l.index(',')]) )
self.add_chapter(l[1+l.index(','):],l[:l.index(',')])
else:
for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1):
self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) )
# self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
# ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
# ]
self.story.setMetadata('numChapters',len(self.chapterUrls))
self.add_chapter(chap,self.url+"&chapter=%d"%j)
return
@ -285,33 +280,34 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
self.story.setMetadata('metaB','01245')
self.story.setMetadata('metaC','The mighty metaC!')
self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
('Chapter 4',self.url+"&chapter=5"),
('Chapter 5',self.url+"&chapter=6"),
('Chapter 6',self.url+"&chapter=7"),
('Chapter 7',self.url+"&chapter=8"),
('Chapter 8',self.url+"&chapter=9"),
#('Chapter 9',self.url+"&chapter=0"),
#('Chapter 0',self.url+"&chapter=a"),
#('Chapter a',self.url+"&chapter=b"),
#('Chapter b',self.url+"&chapter=c"),
#('Chapter c',self.url+"&chapter=d"),
#('Chapter d',self.url+"&chapter=e"),
#('Chapter e',self.url+"&chapter=f"),
#('Chapter f',self.url+"&chapter=g"),
#('Chapter g',self.url+"&chapter=h"),
#('Chapter h',self.url+"&chapter=i"),
#('Chapter i',self.url+"&chapter=j"),
#('Chapter j',self.url+"&chapter=k"),
#('Chapter k',self.url+"&chapter=l"),
#('Chapter l',self.url+"&chapter=m"),
#('Chapter m',self.url+"&chapter=n"),
#('Chapter n',self.url+"&chapter=o"),
]
self.story.setMetadata('numChapters',len(self.chapterUrls))
chapters = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
('Chapter 4',self.url+"&chapter=5"),
('Chapter 5',self.url+"&chapter=6"),
('Chapter 6',self.url+"&chapter=7"),
('Chapter 7',self.url+"&chapter=8"),
('Chapter 8',self.url+"&chapter=9"),
#('Chapter 9',self.url+"&chapter=0"),
#('Chapter 0',self.url+"&chapter=a"),
#('Chapter a',self.url+"&chapter=b"),
#('Chapter b',self.url+"&chapter=c"),
#('Chapter c',self.url+"&chapter=d"),
#('Chapter d',self.url+"&chapter=e"),
#('Chapter e',self.url+"&chapter=f"),
#('Chapter f',self.url+"&chapter=g"),
#('Chapter g',self.url+"&chapter=h"),
#('Chapter h',self.url+"&chapter=i"),
#('Chapter i',self.url+"&chapter=j"),
#('Chapter j',self.url+"&chapter=k"),
#('Chapter k',self.url+"&chapter=l"),
#('Chapter l',self.url+"&chapter=m"),
#('Chapter m',self.url+"&chapter=n"),
#('Chapter n',self.url+"&chapter=o"),
]
for c in chapters:
self.add_chapter(c[0],c[1])
def getChapterText(self, url):

3
fanficfare/adapters/adapter_themasquenet.py
View file

@ -184,9 +184,8 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.section + chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host + self.section + chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

7
fanficfare/adapters/adapter_thepetulantpoetesscom.py
View file

@ -141,16 +141,15 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
# Find the chapters:
chapters=soup.find('select', {'name' : 'sid'})
if chapters == None:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['value'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# make sure that the story id is from the first chapter
self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0])
self.story.setMetadata('storyId',self.get_chapter(0,'url').split('=')[1].split('&')[0])
#locate the story on author's page
index = 1

5
fanficfare/adapters/adapter_tolkienfanfiction.py
View file

@ -146,13 +146,12 @@ class TolkienFanfictionAdapter(BaseSiteAdapter):
else:
raise e
# chapterUrls
# http://tolkienfanfiction.com/Story_Read_Chapter.php?CHid=5358
for chapterLink in soup.findAll("a", href=re.compile(r'Story_Read_Chapter\.php\?CHid=[0-9]+')):
chapterTitle = chapterLink.string
chapterUrl = 'http://' + self.host + '/' + chapterLink['href']
self.chapterUrls.append((chapterTitle, chapterUrl))
numChapters = len(self.chapterUrls)
self.add_chapter(chapterTitle, chapterUrl)
numChapters = self.num_chapters()
self.story.setMetadata('numChapters', numChapters)
logger.debug('Number of Chapters: %s' % numChapters)

3
fanficfare/adapters/adapter_tomparisdormcom.py
View file

@ -107,9 +107,8 @@ class TomParisDormComAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_trekfanfictionnet.py
View file

@ -126,13 +126,12 @@ class TrekFanFictionNetSiteAdapter(BaseSiteAdapter):
# This site has each story on one page, so there are no chapters to get. Will use original
## url since we can't get the chapter without this, I'm leaving it in.
self.chapterUrls.append((self.story.getMetadata('title'), url))
self.add_chapter(self.story.getMetadata('title'), url)
## I'm going to comment this out, because thereis always only one chapter for each story,
## so this is really not needed
## And I am uncommenting it because the rest of FFF expects
## there to always be numChapters, even if it's one. --Jimm
self.story.setMetadata('numChapters',len(self.chapterUrls))
# getting the rest of the metadata... there isn't much here, and the summary can only be
# gotten on the author's page... so we'll get it to get the information from

5
fanficfare/adapters/adapter_trekiverseorg.py
View file

@ -185,13 +185,12 @@ class TrekiverseOrgAdapter(BaseSiteAdapter):
if len(chapters)==0:
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
if len(chapters)==1:
self.chapterUrls.append((self.story.getMetadata('title'),'https://'+self.host+'/efiction/'+chapters[0]['href']))
self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+'/efiction/'+chapters[0]['href'])
else:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/efiction/'+chapter['href']))
self.add_chapter(chapter,'https://'+self.host+'/efiction/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

11
fanficfare/adapters/adapter_tthfanficorg.py
View file

@ -234,9 +234,9 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
self.story.addToList('authorUrl','https://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
self.story.addToList('author',stripHTML(autha))
# include leading number to match 1. ... 2. ...
self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
stripHTML(a),
stripHTML(autha)),'https://'+self.host+a['href']))
self.add_chapter("%d. %s by %s"%(self.num_chapters()+1,
stripHTML(a),
stripHTML(autha)),'https://'+self.host+a['href'])
except urllib2.HTTPError, e:
if e.code == 404:
@ -249,15 +249,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
if select is None:
# no selector found, so it's a one-chapter story.
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
else:
allOptions = select.findAll('option')
for o in allOptions:
url = "https://"+self.host+o['value']
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(o),url))
self.add_chapter(o,url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
verticaltable = soup.find('table', {'class':'verticaltable'})

5
fanficfare/adapters/adapter_twilightarchivescom.py
View file

@ -101,11 +101,10 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
chapters=soup.find('ol', {'class' : 'chapters'})
if chapters != None:
for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")):
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+chapter['href'])
else:
self.chapterUrls.append((self.story.getMetadata('title'),url))
self.add_chapter(self.story.getMetadata('title'),url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# rated, genre, warnings, seires

3
fanficfare/adapters/adapter_twilightednet.py
View file

@ -137,9 +137,8 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href'])
self.story.setMetadata('numChapters',len(self.chapterUrls))
def defaultGetattr(d,k):
try:

9
fanficfare/adapters/adapter_unknowableroomorg.py
View file

@ -128,14 +128,13 @@ class UnknowableRoomOrgSiteAdapter(BaseSiteAdapter):
for chapter in soup.find('select').find_all('option', value=re.compile(
'/'+self.story.getMetadata('storyId')+r'/\d+')):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value']))
self.add_chapter(chapter,'http://'+self.host+chapter['value'])
## One chapter stories do not have a listing for the chapters, so we have to check to make
## sure, and if there aren't any chapterUrls, we set it to the Url entered.
if len(self.chapterUrls) == 0:
self.chapterUrls.append((self.story.getMetadata('title'), url))
## sure, and if there aren't any chapters, we set it to the Url entered.
if self.num_chapters() == 0:
self.add_chapter(self.story.getMetadata('title'), url)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# Most of the metadata can be gotten from the story page, but it can all be gotten from the
# author's fic page, so we are going to get it from there. Unless there is no author page,

2
fanficfare/adapters/adapter_voracity2eficcom.py
View file

@ -225,7 +225,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
chapter_anchor = b_tag.a
title = chapter_anchor.string
url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
self.chapterUrls.append((title, url))
self.add_chapter(title, url)
def getChapterText(self, url):
url += self.AGE_CONSENT_URL_SUFFIX

3
fanficfare/adapters/adapter_walkingtheplankorg.py
View file

@ -122,9 +122,8 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
# Find the chapters:
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
self.add_chapter(chapter,'http://'+self.host+'/archive/'+chapter['href']+addurl)
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.

3
fanficfare/adapters/adapter_wattpadcom.py
View file

@ -116,8 +116,7 @@ class WattpadComAdapter(BaseSiteAdapter):
self.story.setMetadata('dateUpdated', makeDate(storyInfo['modifyDate'].rstrip('Z'), "%Y-%m-%dT%H:%M:%S"))
self.story.setMetadata('datePublished', makeDate(storyInfo['createDate'].rstrip('Z'), "%Y-%m-%dT%H:%M:%S"))
self.chapterUrls = [(part['title'], part['url']) for part in storyInfo['parts']]
self.story.setMetadata('numChapters', len(self.chapterUrls))
[self.add_chapter(part['title'], part['url']) for part in storyInfo['parts']]
self.setCoverImage(storyInfo['url'], storyInfo['cover'].replace('-256-','-512-'))
self.story.setMetadata('language', storyInfo['language']['name'])

3
fanficfare/adapters/adapter_webnovelcom.py
View file

@ -185,9 +185,8 @@ class WWWWebNovelComAdapter(BaseSiteAdapter):
chap_title = 'Chapter ' + unicode(chap['index']) + ' - ' + chap['name']
chap_Url = url.rstrip('/') + '/' + chap['id']
self.chapterUrls.append((chap_title, chap_Url))
self.add_chapter(chap_title, chap_Url)
self.story.setMetadata('numChapters', len(self.chapterUrls))
if get_cover:
cover_meta = soup.find('div', {'class': 'g_col_4'}).find('img')

Some files were not shown because too many files have changed in this diff Show more