diff --git a/fanficfare/adapters/adapter_adastrafanficcom.py b/fanficfare/adapters/adapter_adastrafanficcom.py index e1705fa1..e679cda5 100644 --- a/fanficfare/adapters/adapter_adastrafanficcom.py +++ b/fanficfare/adapters/adapter_adastrafanficcom.py @@ -103,9 +103,8 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ## ## Summary, strangely, is in the content attr of a tag diff --git a/fanficfare/adapters/adapter_adultfanfictionorg.py b/fanficfare/adapters/adapter_adultfanfictionorg.py index 8f7b43b0..aabd1b20 100644 --- a/fanficfare/adapters/adapter_adultfanfictionorg.py +++ b/fanficfare/adapters/adapter_adultfanfictionorg.py @@ -232,9 +232,8 @@ class AdultFanFictionOrgAdapter(BaseSiteAdapter): # Find the chapters: chapters = soup.find('div',{'class':'dropdown-content'}) for i, chapter in enumerate(chapters.findAll('a')): - self.chapterUrls.append((stripHTML(chapter),self.url+'&chapter='+str(i+1))) + self.add_chapter(chapter,self.url+'&chapter='+str(i+1)) - self.story.setMetadata('numChapters', len(self.chapterUrls)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"profile.php\?no=\d+")) diff --git a/fanficfare/adapters/adapter_andromedawebcom.py b/fanficfare/adapters/adapter_andromedawebcom.py index 9f02b2a5..46d793d3 100644 --- a/fanficfare/adapters/adapter_andromedawebcom.py +++ b/fanficfare/adapters/adapter_andromedawebcom.py @@ -192,9 +192,7 @@ class AndromedaWebComAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl)) - - self.story.setMetadata('numChapters',len(self.chapterUrls)) + self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_archiveofourownorg.py b/fanficfare/adapters/adapter_archiveofourownorg.py index 0ab40304..fcde1b0d 100644 --- a/fanficfare/adapters/adapter_archiveofourownorg.py +++ b/fanficfare/adapters/adapter_archiveofourownorg.py @@ -236,11 +236,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): self.story.setMetadata('numChapters',len(chapters)) logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters')) if len(chapters)==1: - self.chapterUrls.append((self.story.getMetadata('title'),'https://'+self.host+chapters[0]['href'])) + self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+chapters[0]['href']) else: for index, chapter in enumerate(chapters): # strip just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+chapter['href'])) + self.add_chapter(chapter,'https://'+self.host+chapter['href']) # (2013-09-21) date = stripHTML(chapter.findNext('span'))[1:-1] chapterDate = makeDate(date,self.dateformat) @@ -389,7 +389,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): whole_dl_soup = chapter_dl_soup = None - if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and len(self.chapterUrls) > 1: + if self.use_full_work_soup and self.getConfig("use_view_full_work",True) and self.num_chapters() > 1: logger.debug("USE view_full_work") ## Assumed view_adult=true was cookied during metadata if not self.full_work_soup: @@ -471,7 +471,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): ## pages and after chapter-# div. Appending removes ## headnotes from whole_dl_soup, so be sure to only do it on ## the last chapter. - if 'authorfootnotes' not in exclude_notes and index+1 == len(self.chapterUrls): + if 'authorfootnotes' not in exclude_notes and index+1 == self.num_chapters(): footnotes = whole_dl_soup.find('div', {'id' : "work_endnotes"}) if footnotes != None: footnotes = footnotes.find('blockquote') @@ -479,7 +479,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): append_tag(save_chapter,'b',"Author's Note:") save_chapter.append(footnotes) - if 'inspiredlinks' not in exclude_notes and index+1 == len(self.chapterUrls): + if 'inspiredlinks' not in exclude_notes and index+1 == self.num_chapters(): inspiredlinks = whole_dl_soup.find('div', {'id' : "children"}) if inspiredlinks != None: if inspiredlinks: diff --git a/fanficfare/adapters/adapter_archiveskyehawkecom.py b/fanficfare/adapters/adapter_archiveskyehawkecom.py index d4d1732c..ea66db2f 100644 --- a/fanficfare/adapters/adapter_archiveskyehawkecom.py +++ b/fanficfare/adapters/adapter_archiveskyehawkecom.py @@ -109,9 +109,8 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter): for i in range(1,len(chapter)): ch=chapter[i] - self.chapterUrls.append((stripHTML(ch),ch['value'])) + self.add_chapter(ch,ch['value']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_artemisfowlcom.py b/fanficfare/adapters/adapter_artemisfowlcom.py index 653a2ed5..c3c2da67 100644 --- a/fanficfare/adapters/adapter_artemisfowlcom.py +++ b/fanficfare/adapters/adapter_artemisfowlcom.py @@ -192,9 +192,8 @@ class ArtemisFowlComAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/fanfiction/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_asexstoriescom.py b/fanficfare/adapters/adapter_asexstoriescom.py index ab978bc8..4c9a1789 100644 --- a/fanficfare/adapters/adapter_asexstoriescom.py +++ b/fanficfare/adapters/adapter_asexstoriescom.py @@ -117,12 +117,9 @@ class ASexStoriesComAdapter(BaseSiteAdapter): description = description.encode('utf-8','ignore').strip()[0:150].decode('utf-8','ignore') self.setDescription(url,'Excerpt from beginning of story: '+description+'...') - # Get chapter URLs - self.chapterUrls = [] - ### The first 'chapter' is not listed in the links, so we have to ### add it before the rest of the pages, if any - self.chapterUrls.append(('1', self.url)) + self.add_chapter('1', self.url) chapterTable = soup1.find('div',{'class':'pages'}).findAll('a') @@ -133,9 +130,8 @@ class ASexStoriesComAdapter(BaseSiteAdapter): chapterTitle = page.string chapterUrl = urlparse.urljoin(self.url, page['href']) if chapterUrl.startswith(self.url): # there are other URLs in the pages block now. - self.chapterUrls.append((chapterTitle, chapterUrl)) + self.add_chapter(chapterTitle, chapterUrl) - self.story.setMetadata('numChapters', len(self.chapterUrls)) rated = soup1.find('div',{'class':'story-info'}).findAll('div',{'story-info-bl5'})[0].find('img')['title'].replace('- Rate','').strip() self.story.setMetadata('rating',rated) diff --git a/fanficfare/adapters/adapter_ashwindersycophanthexcom.py b/fanficfare/adapters/adapter_ashwindersycophanthexcom.py index 3634cf7b..986977ad 100644 --- a/fanficfare/adapters/adapter_ashwindersycophanthexcom.py +++ b/fanficfare/adapters/adapter_ashwindersycophanthexcom.py @@ -161,13 +161,12 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter): # Find the chapters: chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')) if len(chapters) == 0: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: for chapter in chapters: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_asianfanficscom.py b/fanficfare/adapters/adapter_asianfanficscom.py index 3b4bf539..3fc39a87 100644 --- a/fanficfare/adapters/adapter_asianfanficscom.py +++ b/fanficfare/adapters/adapter_asianfanficscom.py @@ -168,7 +168,7 @@ class AsianFanFicsComAdapter(BaseSiteAdapter): self.story.setMetadata('numChapters',len(chapters)) for index, chapter in enumerate(chapters): if chapter.text != 'Foreword': # skip the foreword - self.chapterUrls.append((stripHTML(chapter.text),'https://' + self.getSiteDomain() + chapter['value'])) # note: AFF cuts off chapter names in list. this gets kind of fixed later on + self.add_chapter(chapter.text,'https://' + self.getSiteDomain() + chapter['value']) # note: AFF cuts off chapter names in list. this gets kind of fixed later on # find timestamp a = soup.find('span', text='Updated') if a == None: diff --git a/fanficfare/adapters/adapter_asr3slashzoneorg.py b/fanficfare/adapters/adapter_asr3slashzoneorg.py index 8b0e9c46..0faeeff6 100644 --- a/fanficfare/adapters/adapter_asr3slashzoneorg.py +++ b/fanficfare/adapters/adapter_asr3slashzoneorg.py @@ -140,9 +140,8 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/archive/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_bdsmlibrarycom.py b/fanficfare/adapters/adapter_bdsmlibrarycom.py index 020a7327..ede7e94f 100644 --- a/fanficfare/adapters/adapter_bdsmlibrarycom.py +++ b/fanficfare/adapters/adapter_bdsmlibrarycom.py @@ -155,9 +155,8 @@ class BDSMLibraryComSiteAdapter(BaseSiteAdapter): for chapter in soup.findAll('a', href=re.compile(r'/stories/chapter.php\?storyid='+self.story.getMetadata('storyId')+"&chapterid=\d+$")): value = chapter.findNext('td').findNext('td').string.replace('(added on','').replace(')','').strip() self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat)) - self.chapterUrls.append((stripHTML(chapter),'https://'+self.getSiteDomain()+chapter['href'])) + self.add_chapter(chapter,'https://'+self.getSiteDomain()+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # Get the MetaData # Erotia Tags diff --git a/fanficfare/adapters/adapter_bloodshedversecom.py b/fanficfare/adapters/adapter_bloodshedversecom.py index 37e5959f..7c561274 100644 --- a/fanficfare/adapters/adapter_bloodshedversecom.py +++ b/fanficfare/adapters/adapter_bloodshedversecom.py @@ -85,11 +85,11 @@ class BloodshedverseComAdapter(BaseSiteAdapter): for option in soup.find('select', {'name': 'chapter'}): title = stripHTML(option) url = self.READ_URL_TEMPLATE % option['value'] - self.chapterUrls.append((title, url)) + self.add_chapter(title, url) # Reset the storyId to be the first chapter no. Needed # because emails contain link to later chapters instead. - query_data = urlparse.parse_qs(self.chapterUrls[0][1]) + query_data = urlparse.parse_qs(self.get_chapter(0,'url')) story_no = query_data['no'][0] self.story.setMetadata('storyId', story_no) diff --git a/fanficfare/adapters/adapter_bloodtiesfancom.py b/fanficfare/adapters/adapter_bloodtiesfancom.py index bec8e7af..31484159 100644 --- a/fanficfare/adapters/adapter_bloodtiesfancom.py +++ b/fanficfare/adapters/adapter_bloodtiesfancom.py @@ -214,9 +214,8 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/fiction/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_buffygilescom.py b/fanficfare/adapters/adapter_buffygilescom.py index 9970e7a9..60727150 100644 --- a/fanficfare/adapters/adapter_buffygilescom.py +++ b/fanficfare/adapters/adapter_buffygilescom.py @@ -190,9 +190,8 @@ class BuffyGilesComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/efiction/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_chaossycophanthexcom.py b/fanficfare/adapters/adapter_chaossycophanthexcom.py index cdc110bc..c77e23b6 100644 --- a/fanficfare/adapters/adapter_chaossycophanthexcom.py +++ b/fanficfare/adapters/adapter_chaossycophanthexcom.py @@ -126,9 +126,8 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_chosentwofanficcom.py b/fanficfare/adapters/adapter_chosentwofanficcom.py index e1a3f9a7..d292b0f4 100644 --- a/fanficfare/adapters/adapter_chosentwofanficcom.py +++ b/fanficfare/adapters/adapter_chosentwofanficcom.py @@ -119,10 +119,9 @@ class ChosenTwoFanFicArchiveAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - #self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) - self.chapterUrls.append((stripHTML(chapter),'http://{0}/{1}{2}'.format(self.host, chapter['href'],addURL))) + #self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) + self.add_chapter(chapter,'http://{0}/{1}{2}'.format(self.host, chapter['href'],addURL)) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_csiforensicscom.py b/fanficfare/adapters/adapter_csiforensicscom.py index 046f8905..cba47bca 100644 --- a/fanficfare/adapters/adapter_csiforensicscom.py +++ b/fanficfare/adapters/adapter_csiforensicscom.py @@ -128,9 +128,8 @@ class CSIForensicsComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_deepinmysoulnet.py b/fanficfare/adapters/adapter_deepinmysoulnet.py index c4106202..a25a6e25 100644 --- a/fanficfare/adapters/adapter_deepinmysoulnet.py +++ b/fanficfare/adapters/adapter_deepinmysoulnet.py @@ -189,9 +189,8 @@ class DeepInMySoulNetAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/fiction/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/fiction/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_destinysgatewaycom.py b/fanficfare/adapters/adapter_destinysgatewaycom.py index 7f4011d2..28bde864 100644 --- a/fanficfare/adapters/adapter_destinysgatewaycom.py +++ b/fanficfare/adapters/adapter_destinysgatewaycom.py @@ -138,9 +138,8 @@ class DestinysGatewayComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_dokugacom.py b/fanficfare/adapters/adapter_dokugacom.py index 22f74348..80913b88 100644 --- a/fanficfare/adapters/adapter_dokugacom.py +++ b/fanficfare/adapters/adapter_dokugacom.py @@ -164,13 +164,12 @@ class DokugaComAdapter(BaseSiteAdapter): # Find the chapters: chapters = soup.find('select').findAll('option') if len(chapters)==1: - self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1')) + self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1') else: for chapter in chapters: # just in case there's tags, like in chapter titles. /fanfiction/story/7406/1 - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value'])) + self.add_chapter(chapter,'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) asoup = self.make_soup(self._fetchUrl(alink)) diff --git a/fanficfare/adapters/adapter_dracoandginnycom.py b/fanficfare/adapters/adapter_dracoandginnycom.py index 44ba06c3..436b09ab 100644 --- a/fanficfare/adapters/adapter_dracoandginnycom.py +++ b/fanficfare/adapters/adapter_dracoandginnycom.py @@ -177,9 +177,8 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_dramioneorg.py b/fanficfare/adapters/adapter_dramioneorg.py index 5f1b42c6..856f928b 100644 --- a/fanficfare/adapters/adapter_dramioneorg.py +++ b/fanficfare/adapters/adapter_dramioneorg.py @@ -175,9 +175,8 @@ class DramioneOrgAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_efictionestelielde.py b/fanficfare/adapters/adapter_efictionestelielde.py index 719f84fc..233eda25 100644 --- a/fanficfare/adapters/adapter_efictionestelielde.py +++ b/fanficfare/adapters/adapter_efictionestelielde.py @@ -111,9 +111,8 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_efpfanficnet.py b/fanficfare/adapters/adapter_efpfanficnet.py index a93eb69f..52cd53b3 100644 --- a/fanficfare/adapters/adapter_efpfanficnet.py +++ b/fanficfare/adapters/adapter_efpfanficnet.py @@ -145,7 +145,7 @@ class EFPFanFicNet(BaseSiteAdapter): if select is None: # no selector found, so it's a one-chapter story. - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')}) for o in allOptions: @@ -153,13 +153,12 @@ class EFPFanFicNet(BaseSiteAdapter): o['value']) # just in case there's tags, like in chapter titles. title = stripHTML(o) - self.chapterUrls.append((title,url)) + self.add_chapter(title,url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) self.story.setMetadata('language','Italian') # normalize story URL to first chapter if later chapter URL was given: - url = self.chapterUrls[0][1].replace('&i=1','') + url = self.get_chapter(0,'url').replace('&i=1','') logger.debug("Normalizing to URL: "+url) self._setURL(url) self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) diff --git a/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py b/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py index d59dfa57..fd5789ae 100644 --- a/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py +++ b/fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py @@ -142,9 +142,8 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_fanficauthorsnet.py b/fanficfare/adapters/adapter_fanficauthorsnet.py index 86914580..a18b925f 100644 --- a/fanficfare/adapters/adapter_fanficauthorsnet.py +++ b/fanficfare/adapters/adapter_fanficauthorsnet.py @@ -223,9 +223,8 @@ class FanficAuthorsNetAdapter(BaseSiteAdapter): else: chapter_title = stripHTML(unicode(chapter)) chapter_Url = self.story.getMetadata('authorUrl')+chapter['href'][1:] - self.chapterUrls.append((chapter_title, chapter_Url)) + self.add_chapter(chapter_title, chapter_Url) - self.story.setMetadata('numChapters', len(self.chapterUrls)) genres = ("Drama","Romance") gotgenre = False ## Getting the Metadata that is there diff --git a/fanficfare/adapters/adapter_fanficcastletvnet.py b/fanficfare/adapters/adapter_fanficcastletvnet.py index bf2947e9..ee186272 100644 --- a/fanficfare/adapters/adapter_fanficcastletvnet.py +++ b/fanficfare/adapters/adapter_fanficcastletvnet.py @@ -199,9 +199,8 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_fanfichu.py b/fanficfare/adapters/adapter_fanfichu.py index f6925e51..7685a6d4 100644 --- a/fanficfare/adapters/adapter_fanfichu.py +++ b/fanficfare/adapters/adapter_fanfichu.py @@ -97,7 +97,7 @@ class FanficHuAdapter(BaseSiteAdapter): for option in chapter_options: url = urlparse.urljoin(self.url, option['value']) - self.chapterUrls.append((option.string, url)) + self.add_chapter(option.string, url) author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href']) soup = self._customized_fetch_url(author_url) diff --git a/fanficfare/adapters/adapter_fanfictionjunkiesde.py b/fanficfare/adapters/adapter_fanfictionjunkiesde.py index 57dff4f5..c2418b47 100644 --- a/fanficfare/adapters/adapter_fanfictionjunkiesde.py +++ b/fanficfare/adapters/adapter_fanfictionjunkiesde.py @@ -192,9 +192,8 @@ class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/efiction/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_fanfictionnet.py b/fanficfare/adapters/adapter_fanfictionnet.py index ae28de26..9140ff6f 100644 --- a/fanficfare/adapters/adapter_fanfictionnet.py +++ b/fanficfare/adapters/adapter_fanfictionnet.py @@ -330,7 +330,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter): if select is None: # no selector found, so it's a one-chapter story. - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: allOptions = select.findAll('option') for o in allOptions: @@ -340,9 +340,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter): # just in case there's tags, like in chapter titles. title = u"%s" % o title = re.sub(r'<[^>]+>','',title) - self.chapterUrls.append((title,url)) + self.add_chapter(title,url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) return diff --git a/fanficfare/adapters/adapter_fanfiktionde.py b/fanficfare/adapters/adapter_fanfiktionde.py index 6ba98fca..70516081 100644 --- a/fanficfare/adapters/adapter_fanfiktionde.py +++ b/fanficfare/adapters/adapter_fanfiktionde.py @@ -151,9 +151,8 @@ class FanFiktionDeAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.find('select').findAll('option'): - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value'])) + self.add_chapter(chapter,'https://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) self.story.setMetadata('numWords',stripHTML(soup.find("span",title="Wörter").parent)) self.story.setMetadata('language','German') diff --git a/fanficfare/adapters/adapter_ficbooknet.py b/fanficfare/adapters/adapter_ficbooknet.py index 72cc1f14..14304ab7 100644 --- a/fanficfare/adapters/adapter_ficbooknet.py +++ b/fanficfare/adapters/adapter_ficbooknet.py @@ -122,7 +122,7 @@ class FicBookNetAdapter(BaseSiteAdapter): for x in range(0,len(chapters)): chapter=chapters[x] churl='https://'+self.host+chapter['href'] - self.chapterUrls.append((stripHTML(chapter),churl)) + self.add_chapter(chapter,churl) ## First chapter doesn't always have a date, skip it. if pubdate == None and chapter.parent.find('span'): pubdate = translit.translit(stripHTML(chapter.parent.find('span'))) @@ -131,7 +131,7 @@ class FicBookNetAdapter(BaseSiteAdapter): update = translit.translit(stripHTML(chapter.parent.find('span'))) # update = translit.translit(stripHTML(self.make_soup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span'))) else: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) self.story.setMetadata('numChapters',1) pubdate=translit.translit(stripHTML(soup.find('div',{'class':'title-area'}).find('span'))) update=pubdate diff --git a/fanficfare/adapters/adapter_fictionalleyorg.py b/fanficfare/adapters/adapter_fictionalleyorg.py index 63fd870b..bdbf323d 100644 --- a/fanficfare/adapters/adapter_fictionalleyorg.py +++ b/fanficfare/adapters/adapter_fictionalleyorg.py @@ -128,14 +128,13 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter): self.story.setMetadata('authorUrl',authora['href']) if len(chapterlinklist) == 1: - self.chapterUrls.append((self.story.getMetadata('title'),chapterlinklist[0]['href'])) + self.add_chapter(self.story.getMetadata('title'),chapterlinklist[0]['href']) else: # Find the chapters: for chapter in chapterlinklist: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),chapter['href'])) + self.add_chapter(chapter,chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ## Go scrape the rest of the metadata from the author's page. data = self._fetchUrl(self.story.getMetadata('authorUrl')) diff --git a/fanficfare/adapters/adapter_fictionhuntcom.py b/fanficfare/adapters/adapter_fictionhuntcom.py index 3c47db12..e3ae1bef 100644 --- a/fanficfare/adapters/adapter_fictionhuntcom.py +++ b/fanficfare/adapters/adapter_fictionhuntcom.py @@ -130,8 +130,8 @@ class FictionHuntComSiteAdapter(BaseSiteAdapter): # alternative is to get the num of chaps from the last # indiated chapter list instead. for i in range(1,1+int(self.story.getMetadata('numChapters'))): - self.chapterUrls.append(("Chapter "+unicode(i),"http://"+self.getSiteDomain()\ - +"/read/"+self.story.getMetadata('storyId')+"/%s"%i)) + self.add_chapter("Chapter "+unicode(i),"http://"+self.getSiteDomain()\ + +"/read/"+self.story.getMetadata('storyId')+"/%s"%i) def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) diff --git a/fanficfare/adapters/adapter_fictionmaniatv.py b/fanficfare/adapters/adapter_fictionmaniatv.py index 928e6fbe..d6de271c 100644 --- a/fanficfare/adapters/adapter_fictionmaniatv.py +++ b/fanficfare/adapters/adapter_fictionmaniatv.py @@ -79,7 +79,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter): if key == 'Title': self.story.setMetadata('title', value) - self.chapterUrls.append((value, self.url)) + self.add_chapter(value, self.url) elif key == 'File Name': self.story.setMetadata('fileName', value) diff --git a/fanficfare/adapters/adapter_fictionpadcom.py b/fanficfare/adapters/adapter_fictionpadcom.py index f675bb27..2881bfe6 100644 --- a/fanficfare/adapters/adapter_fictionpadcom.py +++ b/fanficfare/adapters/adapter_fictionpadcom.py @@ -176,9 +176,8 @@ class FictionPadSiteAdapter(BaseSiteAdapter): chtitle = "Chapter %d"%c['number'] if c['title']: chtitle += " - %s"%c['title'] - self.chapterUrls.append((chtitle,c['body_url'])) + self.add_chapter(chtitle,c['body_url']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) diff --git a/fanficfare/adapters/adapter_ficwadcom.py b/fanficfare/adapters/adapter_ficwadcom.py index f774c7f2..2f2fc834 100644 --- a/fanficfare/adapters/adapter_ficwadcom.py +++ b/fanficfare/adapters/adapter_ficwadcom.py @@ -202,7 +202,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter): storylistul = soup.find('ul',{'class':'storylist'}) if not storylistul: # no list found, so it's a one-chapter story. - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: chapterlistlis = storylistul.findAll('li') for chapterli in chapterlistlis: @@ -211,12 +211,9 @@ class FicwadComSiteAdapter(BaseSiteAdapter): raise exceptions.FailedToLogin(url,self.username) else: #print "chapterli.h4.a (%s)"%chapterli.h4.a - self.chapterUrls.append((chapterli.h4.a.string, - u'https://%s%s'%(self.getSiteDomain(), - chapterli.h4.a['href']))) - #print "self.chapterUrls:%s"%self.chapterUrls - self.story.setMetadata('numChapters',len(self.chapterUrls)) - + self.add_chapter(chapterli.h4.a.string, + u'https://%s%s'%(self.getSiteDomain(), + chapterli.h4.a['href'])) return diff --git a/fanficfare/adapters/adapter_fimfictionnet.py b/fanficfare/adapters/adapter_fimfictionnet.py index fe37413e..42f7233b 100644 --- a/fanficfare/adapters/adapter_fimfictionnet.py +++ b/fanficfare/adapters/adapter_fimfictionnet.py @@ -147,9 +147,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): # Chapters for chapter in soup.find('ul',{'class':'chapters'}).find_all('a',{'href':re.compile(r'.*/story/'+self.story.getMetadata('storyId')+r'/.*')}): - self.chapterUrls.append((stripHTML(chapter), 'https://'+self.host+chapter['href'])) + self.add_chapter(chapter, 'https://'+self.host+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # Status # In the case of Fimfiction, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled' diff --git a/fanficfare/adapters/adapter_fireflyfansnet.py b/fanficfare/adapters/adapter_fireflyfansnet.py index a7982312..005884f2 100644 --- a/fanficfare/adapters/adapter_fireflyfansnet.py +++ b/fanficfare/adapters/adapter_fireflyfansnet.py @@ -108,7 +108,7 @@ class FireFlyFansNetSiteAdapter(BaseSiteAdapter): # way to determine if there are other chapters to the same story, so you have # to download them one at a time yourself. I'm also setting the status to # complete - self.chapterUrls.append((self.story.getMetadata('title'), self.url)) + self.add_chapter(self.story.getMetadata('title'), self.url) self.story.setMetadata('numChapters', 1) self.story.setMetadata('status', 'Completed') diff --git a/fanficfare/adapters/adapter_fireflypopulliorg.py b/fanficfare/adapters/adapter_fireflypopulliorg.py index feaca090..028862c9 100644 --- a/fanficfare/adapters/adapter_fireflypopulliorg.py +++ b/fanficfare/adapters/adapter_fireflypopulliorg.py @@ -129,7 +129,7 @@ class FireflyPopulliOrgSiteAdapter(BaseSiteAdapter): # This site has the entire story on one page, so we will be using the normalized URL as # the chapterUrl and the Title as the chapter Title - self.chapterUrls.append((self.story.getMetadata('title'), url)) + self.add_chapter(self.story.getMetadata('title'), url) ## i would take this out, as it is not really needed, but the calibre plugin uses it, ## so it's staying diff --git a/fanficfare/adapters/adapter_gravitytalescom.py b/fanficfare/adapters/adapter_gravitytalescom.py index 2c210128..97183427 100644 --- a/fanficfare/adapters/adapter_gravitytalescom.py +++ b/fanficfare/adapters/adapter_gravitytalescom.py @@ -163,9 +163,8 @@ class GravityTalesComSiteAdapter(BaseSiteAdapter): for alink in chap_soup.find_all('a',href=re.compile(self.getSiteDomain())): # ignore anchor links ## Some stories have that same chapters in different sections if alink['href'] not in found_chaps: - self.chapterUrls.append((stripHTML(alink),alink['href'])) + self.add_chapter(alink,alink['href']) found_chaps[alink['href']] = alink['href'] - self.story.setMetadata('numChapters',len(self.chapterUrls)) if feedparser: # Parse published and updated date from latest RSS feed entry. The RSS feed urls seems to appear due to diff --git a/fanficfare/adapters/adapter_harrypotterfanfictioncom.py b/fanficfare/adapters/adapter_harrypotterfanfictioncom.py index 5baac2a8..9bb09dca 100644 --- a/fanficfare/adapters/adapter_harrypotterfanfictioncom.py +++ b/fanficfare/adapters/adapter_harrypotterfanfictioncom.py @@ -110,9 +110,8 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter): chapter_words+=int(tdstr) chapter = tr.find('a') chpt=re.sub(r'^.*?(\?chapterid=\d+).*?',r'\1',chapter['href']) - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/viewstory.php'+chpt)) + self.add_chapter(chapter,'https://'+self.host+'/viewstory.php'+chpt) #self.story.setMetadata('numWords',unicode(words)) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ## Finding the metadata is a bit of a pain. Desc is the only thing this color. desctable= soup.find('table',{'bgcolor':'#f0e8e8'}) diff --git a/fanficfare/adapters/adapter_hlfictionnet.py b/fanficfare/adapters/adapter_hlfictionnet.py index c242af9a..92686186 100644 --- a/fanficfare/adapters/adapter_hlfictionnet.py +++ b/fanficfare/adapters/adapter_hlfictionnet.py @@ -109,11 +109,10 @@ class HLFictionNetAdapter(BaseSiteAdapter): if chapters != None: for chapter in chapters.findAll('option'): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])) + self.add_chapter(chapter,'https://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']) else: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl'))) diff --git a/fanficfare/adapters/adapter_hpfanficarchivecom.py b/fanficfare/adapters/adapter_hpfanficarchivecom.py index 121ee13d..1cebac01 100644 --- a/fanficfare/adapters/adapter_hpfanficarchivecom.py +++ b/fanficfare/adapters/adapter_hpfanficarchivecom.py @@ -109,9 +109,8 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/stories/'+chapter['href'])) + self.add_chapter(chapter,'https://'+self.host+'/stories/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_iketernalnet.py b/fanficfare/adapters/adapter_iketernalnet.py index 1ab7b1a3..a0e5bd13 100644 --- a/fanficfare/adapters/adapter_iketernalnet.py +++ b/fanficfare/adapters/adapter_iketernalnet.py @@ -189,9 +189,8 @@ class IkEternalNetAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_imagineeficcom.py b/fanficfare/adapters/adapter_imagineeficcom.py index f6945ead..32ecf81e 100644 --- a/fanficfare/adapters/adapter_imagineeficcom.py +++ b/fanficfare/adapters/adapter_imagineeficcom.py @@ -177,9 +177,8 @@ class ImagineEFicComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_inkbunnynet.py b/fanficfare/adapters/adapter_inkbunnynet.py index 3e0cea68..d8a15eff 100644 --- a/fanficfare/adapters/adapter_inkbunnynet.py +++ b/fanficfare/adapters/adapter_inkbunnynet.py @@ -188,10 +188,9 @@ class InkBunnyNetSiteAdapter(BaseSiteAdapter): self.story.setMetadata('dateUpdated', makeDate(updated, self.dateformat)) self.story.setMetadata('datePublished', makeDate(updated, self.dateformat)) - # This is a 1 story/page site, so we set the chapterUrls up with the story url and title - self.chapterUrls.append((self.story.getMetadata('title'), url)) + # This is a 1 story/page site, so we set the chapter up with the story url and title + self.add_chapter(self.story.getMetadata('title'), url) - self.story.setMetadata('numChapters', len(self.chapterUrls)) if get_cover: cover_img = soup.find('img', {'id':'magicbox'}) diff --git a/fanficfare/adapters/adapter_kiarepositorymujajinet.py b/fanficfare/adapters/adapter_kiarepositorymujajinet.py index 19f1f6eb..df6f06b9 100644 --- a/fanficfare/adapters/adapter_kiarepositorymujajinet.py +++ b/fanficfare/adapters/adapter_kiarepositorymujajinet.py @@ -189,9 +189,8 @@ class KiaRepositoryMujajiNetAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/repository/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/repository/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_ksarchivecom.py b/fanficfare/adapters/adapter_ksarchivecom.py index 2cb16b88..ad7dbe80 100644 --- a/fanficfare/adapters/adapter_ksarchivecom.py +++ b/fanficfare/adapters/adapter_ksarchivecom.py @@ -166,9 +166,8 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_lcfanficcom.py b/fanficfare/adapters/adapter_lcfanficcom.py index f847da1d..14155e5a 100644 --- a/fanficfare/adapters/adapter_lcfanficcom.py +++ b/fanficfare/adapters/adapter_lcfanficcom.py @@ -167,8 +167,7 @@ class LCFanFicComSiteAdapter(BaseSiteAdapter): ## This is a 1 story/page site, so we'll keep the soup fo the getChapterText function ## the chapterUrl nd numChapters need to be set as well self.html = data - self.chapterUrls.append((self.story.getMetadata('title'),url)) - self.story.setMetadata('numChapters',len(self.chapterUrls)) + self.add_chapter(self.story.getMetadata('title'),url) # grab the text for an individual chapter. diff --git a/fanficfare/adapters/adapter_lightnovelgatecom.py b/fanficfare/adapters/adapter_lightnovelgatecom.py index a75579cc..fe79e103 100644 --- a/fanficfare/adapters/adapter_lightnovelgatecom.py +++ b/fanficfare/adapters/adapter_lightnovelgatecom.py @@ -158,7 +158,7 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter): dt = row.find_all('span')[-1].string cdates.append(makeDate(dt, '%b-%d-%y')) clink = row.find('a') - self.chapterUrls.append((clink.string, clink['href'])) + self.add_chapter(clink.string, clink['href']) cdates.sort() # dateUpdated in upper part show only date of last chapter, but if @@ -166,7 +166,6 @@ class LightNovelGateSiteAdapter(BaseSiteAdapter): # dates manually self.story.setMetadata('dateUpdated', cdates[-1]) self.story.setMetadata('datePublished', cdates[0]) - self.story.setMetadata('numChapters', len(self.chapterUrls)) ## getting description cdata = soup.select_one('#noidungm') diff --git a/fanficfare/adapters/adapter_literotica.py b/fanficfare/adapters/adapter_literotica.py index c9f51590..638dbf51 100644 --- a/fanficfare/adapters/adapter_literotica.py +++ b/fanficfare/adapters/adapter_literotica.py @@ -187,9 +187,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (self.url, authorurl)) if isSingleStory: -# self.chapterUrls = [(soup1.h1.string, self.url)] -# self.story.setMetadata('title', soup1.h1.string) - self.story.setMetadata('title', storyLink.text.strip('/')) logger.debug('Title: "%s"' % storyLink.text.strip('/')) self.story.setMetadata('description', urlTr.findAll("td")[1].text) @@ -198,7 +195,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): date = urlTr.findAll('td')[-1].text self.story.setMetadata('datePublished', makeDate(date, self.dateformat)) self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat)) - self.chapterUrls = [(storyLink.text, self.url)] + self.add_chapter(storyLink.text, self.url) averrating = stripHTML(storyLink.parent) ## title (0.00) averrating = averrating[averrating.rfind('(')+1:averrating.rfind(')')] @@ -218,7 +215,6 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): ## Walk the chapters chapterTr = seriesTr.nextSibling - self.chapterUrls = [] dates = [] descriptions = [] ratings = [] @@ -242,7 +238,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): chapter = chapterLink.text[len(seriesTitle):].strip() logger.debug('\tChapter: "%s"' % chapter) if chapter == '': - chapter_title = 'Chapter %d' % (len(self.chapterUrls) + 1) + chapter_title = 'Chapter %d' % (self.num_chapters() + 1) else: separater_char = chapter[0] logger.debug('\tseparater_char: "%s"' % separater_char) @@ -271,7 +267,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): logger.debug("Chapter Title: " + chapter_title) logger.debug("Chapter description: " + description) chapters.append((chapter_title, chapurl, description, pub_date)) -# self.chapterUrls.append((chapter_title, chapurl)) +# self.add_chapter(chapter_title, chapurl) numrating = stripHTML(chapterLink.parent) ## title (0.00) numrating = numrating[numrating.rfind('(')+1:numrating.rfind(')')] @@ -282,7 +278,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): chapters = sorted(chapters, key=lambda chapter: chapter[3]) for i, chapter in enumerate(chapters): - self.chapterUrls.append((chapter[0], chapter[1])) + self.add_chapter(chapter[0], chapter[1]) descriptions.append("%d. %s" % (i + 1, chapter[2])) ## Set the oldest date as publication date, the newest as update date dates.sort() @@ -297,12 +293,11 @@ class LiteroticaSiteAdapter(BaseSiteAdapter): self.story.setMetadata('averrating','%4.2f' % (sum(ratings) / float(len(ratings)))) # normalize on first chapter URL. - self._setURL(self.chapterUrls[0][1]) + self._setURL(self.get_chapter(0,'url')) # reset storyId to first chapter. self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) - self.story.setMetadata('numChapters', len(self.chapterUrls)) # Add the category from the breadcumb. This might duplicate a category already added. self.story.addToList('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string) diff --git a/fanficfare/adapters/adapter_lotrgficcom.py b/fanficfare/adapters/adapter_lotrgficcom.py index 6d72828e..6fb6392d 100644 --- a/fanficfare/adapters/adapter_lotrgficcom.py +++ b/fanficfare/adapters/adapter_lotrgficcom.py @@ -113,9 +113,8 @@ class LOTRgficComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in div.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ### Metadata is contained diff --git a/fanficfare/adapters/adapter_lumossycophanthexcom.py b/fanficfare/adapters/adapter_lumossycophanthexcom.py index 11d4cfbb..f87410be 100644 --- a/fanficfare/adapters/adapter_lumossycophanthexcom.py +++ b/fanficfare/adapters/adapter_lumossycophanthexcom.py @@ -126,9 +126,8 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_masseffect2in.py b/fanficfare/adapters/adapter_masseffect2in.py index 0380e052..2fb26965 100644 --- a/fanficfare/adapters/adapter_masseffect2in.py +++ b/fanficfare/adapters/adapter_masseffect2in.py @@ -197,7 +197,7 @@ class MassEffect2InAdapter(BaseSiteAdapter): self.story.extendList('warnings', [warning]) chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:]) - self.chapterUrls.append((chapterTitle, url)) + self.add_chapter(chapterTitle, url) except ParsingError, error: raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error)) diff --git a/fanficfare/adapters/adapter_mcstoriescom.py b/fanficfare/adapters/adapter_mcstoriescom.py index c90bdabc..93572bf2 100644 --- a/fanficfare/adapters/adapter_mcstoriescom.py +++ b/fanficfare/adapters/adapter_mcstoriescom.py @@ -131,7 +131,6 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter): if updatedate is not None: self.story.setMetadata('dateUpdated', updatedate) # Get chapter URLs - self.chapterUrls = [] chapterTable = soup1.find('table', class_='index') if chapterTable is not None: @@ -144,15 +143,14 @@ class MCStoriesComSiteAdapter(BaseSiteAdapter): link = chapterCell.a chapterTitle = link.text chapterUrl = urlparse.urljoin(self.url, link['href']) - self.chapterUrls.append((chapterTitle, chapterUrl)) + self.add_chapter(chapterTitle, chapterUrl) else: # Single chapter chapterDiv = soup1.find('div', class_='chapter') chapterTitle = chapterDiv.a.text chapterUrl = urlparse.urljoin(self.url, chapterDiv.a['href']) - self.chapterUrls = [(chapterTitle, chapterUrl)] + self.add_chapter(chapterTitle, chapterUrl) - self.story.setMetadata('numChapters', len(self.chapterUrls)) logger.debug("Story: <%s>", self.story) diff --git a/fanficfare/adapters/adapter_mediaminerorg.py b/fanficfare/adapters/adapter_mediaminerorg.py index 28c43134..2bfb4bdc 100644 --- a/fanficfare/adapters/adapter_mediaminerorg.py +++ b/fanficfare/adapters/adapter_mediaminerorg.py @@ -147,9 +147,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter): # Find the chapters - one-shot now have chapter list, too. chap_p = soup.find('p',{'style':'margin-left:10px;'}) for (atag,aurl,name) in [ (x,x['href'],stripHTML(x)) for x in chap_p.find_all('a') ]: - self.chapterUrls.append((name,'https://'+self.host+aurl)) + self.add_chapter(name,'https://'+self.host+aurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # category # Ranma 1/2 diff --git a/fanficfare/adapters/adapter_merlinficdtwinscouk.py b/fanficfare/adapters/adapter_merlinficdtwinscouk.py index d946a871..b34cdc40 100644 --- a/fanficfare/adapters/adapter_merlinficdtwinscouk.py +++ b/fanficfare/adapters/adapter_merlinficdtwinscouk.py @@ -177,9 +177,8 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_midnightwhispers.py b/fanficfare/adapters/adapter_midnightwhispers.py index b059dcf5..fa889f8a 100644 --- a/fanficfare/adapters/adapter_midnightwhispers.py +++ b/fanficfare/adapters/adapter_midnightwhispers.py @@ -170,9 +170,8 @@ class MidnightwhispersAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_nationallibrarynet.py b/fanficfare/adapters/adapter_nationallibrarynet.py index 0cb72948..bdea9c85 100644 --- a/fanficfare/adapters/adapter_nationallibrarynet.py +++ b/fanficfare/adapters/adapter_nationallibrarynet.py @@ -114,10 +114,9 @@ class NationalLibraryNetAdapter(BaseSiteAdapter): if len(chapters) > 0: for chapter in chapters: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) break - self.story.setMetadata('numChapters',len(self.chapterUrls)) self.story.setMetadata('status', 'Completed') # Rated: NC-17
etc diff --git a/fanficfare/adapters/adapter_ncisficcom.py b/fanficfare/adapters/adapter_ncisficcom.py index b6c5854c..79ceccce 100644 --- a/fanficfare/adapters/adapter_ncisficcom.py +++ b/fanficfare/adapters/adapter_ncisficcom.py @@ -113,10 +113,9 @@ class NCISFicComAdapter(BaseSiteAdapter): if len(chapters) > 0: for chapter in chapters: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) break - self.story.setMetadata('numChapters',len(self.chapterUrls)) self.story.setMetadata('status', 'Completed') # Rated: NC-17
etc diff --git a/fanficfare/adapters/adapter_nfacommunitycom.py b/fanficfare/adapters/adapter_nfacommunitycom.py index 391f143a..754948a6 100644 --- a/fanficfare/adapters/adapter_nfacommunitycom.py +++ b/fanficfare/adapters/adapter_nfacommunitycom.py @@ -163,9 +163,8 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) diff --git a/fanficfare/adapters/adapter_nocturnallightnet.py b/fanficfare/adapters/adapter_nocturnallightnet.py index b6c67713..db86d4cd 100644 --- a/fanficfare/adapters/adapter_nocturnallightnet.py +++ b/fanficfare/adapters/adapter_nocturnallightnet.py @@ -88,7 +88,7 @@ class NocturnalLightNetAdapter(BaseSiteAdapter): chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/')) for chapter_anchor in chapter_anchors: url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) - self.chapterUrls.append((chapter_anchor.string, url)) + self.add_chapter(chapter_anchor.string, url) author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href']) soup = self._customized_fetch_url(author_url) @@ -108,7 +108,7 @@ class NocturnalLightNetAdapter(BaseSiteAdapter): # No chapter anchors found in the original story URL, so the story has # only a single chapter. if not chapter_anchors: - self.chapterUrls.append((title, self.url)) + self.add_chapter(title, self.url) for b_tag in listbox('b'): key = b_tag.string.strip(':') diff --git a/fanficfare/adapters/adapter_noveltrovecom.py b/fanficfare/adapters/adapter_noveltrovecom.py index 21396df2..24ddf5c8 100644 --- a/fanficfare/adapters/adapter_noveltrovecom.py +++ b/fanficfare/adapters/adapter_noveltrovecom.py @@ -136,8 +136,7 @@ class NovelTroveComSiteAdapter(BaseSiteAdapter): ## This is a 1 story/page site, so we'll keep the soup for the getChapterText function ## the chapterUrl and numChapters need to be set as well self.html = soup - self.chapterUrls.append((self.story.getMetadata('title'), url)) - self.story.setMetadata('numChapters', len(self.chapterUrls)) + self.add_chapter(self.story.getMetadata('title'), url) self.story.setMetadata('status', 'Completed') ## Getting the non-standard title page entries diff --git a/fanficfare/adapters/adapter_occlumencysycophanthexcom.py b/fanficfare/adapters/adapter_occlumencysycophanthexcom.py index 4fe3ad81..ce49c8d5 100644 --- a/fanficfare/adapters/adapter_occlumencysycophanthexcom.py +++ b/fanficfare/adapters/adapter_occlumencysycophanthexcom.py @@ -161,13 +161,12 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter): # Find the chapters: chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')) if len(chapters) == 0: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: for chapter in chapters: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_phoenixsongnet.py b/fanficfare/adapters/adapter_phoenixsongnet.py index e6a9fb57..6994cc38 100644 --- a/fanficfare/adapters/adapter_phoenixsongnet.py +++ b/fanficfare/adapters/adapter_phoenixsongnet.py @@ -142,7 +142,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter): # Find the chapters: chapters = soup.find('select') if chapters == None: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) for b in soup.findAll('b'): if b.text == "Updated": date = b.nextSibling.string.split(': ')[1].split(',') @@ -152,7 +152,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter): i = 0 chapters = chapters.findAll('option') for chapter in chapters: - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+chapter['value'])) + self.add_chapter(chapter,'https://'+self.host+chapter['value']) if i == 0: self.story.setMetadata('storyId',chapter['value'].split('/')[3]) head = self.make_soup(self._fetchUrl('https://'+self.host+chapter['value'])).findAll('b') @@ -170,7 +170,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter): i = i+1 - self.story.setMetadata('numChapters',len(self.chapterUrls)) asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl'))) diff --git a/fanficfare/adapters/adapter_ponyfictionarchivenet.py b/fanficfare/adapters/adapter_ponyfictionarchivenet.py index bfdc7ab6..54775eff 100644 --- a/fanficfare/adapters/adapter_ponyfictionarchivenet.py +++ b/fanficfare/adapters/adapter_ponyfictionarchivenet.py @@ -143,9 +143,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_potionsandsnitches.py b/fanficfare/adapters/adapter_potionsandsnitches.py index e09b342a..d25212a6 100644 --- a/fanficfare/adapters/adapter_potionsandsnitches.py +++ b/fanficfare/adapters/adapter_potionsandsnitches.py @@ -90,9 +90,8 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/fanfiction/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) def defaultGetattr(d,k): try: diff --git a/fanficfare/adapters/adapter_potterficscom.py b/fanficfare/adapters/adapter_potterficscom.py index 70a62970..e3f789e4 100644 --- a/fanficfare/adapters/adapter_potterficscom.py +++ b/fanficfare/adapters/adapter_potterficscom.py @@ -235,19 +235,16 @@ class PotterFicsComAdapter(BaseSiteAdapter): #we can go back to the table cell we found before #get its last element and work backwards to find the last ordered list on the page list = cell.contents[len(cell)-1].findPrevious('ol') - chapters = [] revs = 0 chnum = 0 for li in list: chnum += 1 chTitle = unicode(chnum) + '. ' + li.a.b.string.strip() chURL = makeAbsoluteURL(li.a['href']) - chapters.append((chTitle,chURL)) + self.add_chapter(chTitle,chURL) #Get reviews, add to total revs += int(li.div.a.string.split()[0]) - self.chapterUrls.extend(chapters) - self.story.setMetadata('numChapters', len(chapters)) self.story.setMetadata('reviews', revs) #Now for the description... this may be tricky... diff --git a/fanficfare/adapters/adapter_potterheadsanonymouscom.py b/fanficfare/adapters/adapter_potterheadsanonymouscom.py index 4b278f49..4184e19d 100644 --- a/fanficfare/adapters/adapter_potterheadsanonymouscom.py +++ b/fanficfare/adapters/adapter_potterheadsanonymouscom.py @@ -188,9 +188,8 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_pretendercentrecom.py b/fanficfare/adapters/adapter_pretendercentrecom.py index 7c44c087..0d644c34 100644 --- a/fanficfare/adapters/adapter_pretendercentrecom.py +++ b/fanficfare/adapters/adapter_pretendercentrecom.py @@ -141,9 +141,8 @@ class PretenderCenterComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/missingpieces/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/missingpieces/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_qafficcom.py b/fanficfare/adapters/adapter_qafficcom.py index 0609da7c..9d5d252f 100644 --- a/fanficfare/adapters/adapter_qafficcom.py +++ b/fanficfare/adapters/adapter_qafficcom.py @@ -141,11 +141,10 @@ class QafFicComAdapter(BaseSiteAdapter): if chapters != None: for chapter in chapters.findAll('option'): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])) + self.add_chapter(chapter,'https://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']) else: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl'))) for list in asoup.findAll('div', {'class' : re.compile('listbox')}): diff --git a/fanficfare/adapters/adapter_quotevcom.py b/fanficfare/adapters/adapter_quotevcom.py index 5175521a..282ba283 100644 --- a/fanficfare/adapters/adapter_quotevcom.py +++ b/fanficfare/adapters/adapter_quotevcom.py @@ -122,9 +122,8 @@ class QuotevComAdapter(BaseSiteAdapter): for a in soup.find('div', id='rselect')('a'): if 'javascript' not in a['href']: - self.chapterUrls.append((a.get_text(), urlparse.urljoin(self.url, a['href']))) + self.add_chapter(a.get_text(), urlparse.urljoin(self.url, a['href'])) - self.story.setMetadata('numChapters', len(self.chapterUrls)) def getChapterText(self, url): data = self._fetchUrl(url) diff --git a/fanficfare/adapters/adapter_royalroadl.py b/fanficfare/adapters/adapter_royalroadl.py index cbb2e610..f6ebccf8 100644 --- a/fanficfare/adapters/adapter_royalroadl.py +++ b/fanficfare/adapters/adapter_royalroadl.py @@ -166,9 +166,8 @@ class RoyalRoadAdapter(BaseSiteAdapter): tds = [tr.findAll('td')[0] for tr in chapters.findAll('tr')] for td in tds: chapterUrl = 'https://' + self.getSiteDomain() + td.a['href'] - self.chapterUrls.append((stripHTML(td.text), chapterUrl)) + self.add_chapter(td.text, chapterUrl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # this is forum based so it's a bit ugly description = soup.find('div', {'property': 'description', 'class': 'hidden-content'}) diff --git a/fanficfare/adapters/adapter_samandjacknet.py b/fanficfare/adapters/adapter_samandjacknet.py index 2c4355b3..8304bafb 100644 --- a/fanficfare/adapters/adapter_samandjacknet.py +++ b/fanficfare/adapters/adapter_samandjacknet.py @@ -222,9 +222,8 @@ class SamAndJackNetAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/fanfics/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_scarvesandcoffeenet.py b/fanficfare/adapters/adapter_scarvesandcoffeenet.py index c8f4a024..b2b1d9c1 100644 --- a/fanficfare/adapters/adapter_scarvesandcoffeenet.py +++ b/fanficfare/adapters/adapter_scarvesandcoffeenet.py @@ -146,9 +146,8 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_sheppardweircom.py b/fanficfare/adapters/adapter_sheppardweircom.py index 7821cea8..6b1fa76a 100644 --- a/fanficfare/adapters/adapter_sheppardweircom.py +++ b/fanficfare/adapters/adapter_sheppardweircom.py @@ -195,9 +195,8 @@ class SheppardWeirComAdapter(BaseSiteAdapter): # XXX # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/fanfics/'+chapter['href']+addurl)) + self.add_chapter(chapter,'https://'+self.host+'/fanfics/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_shriftweborgbfa.py b/fanficfare/adapters/adapter_shriftweborgbfa.py index c8a448f4..b1fbaa4f 100644 --- a/fanficfare/adapters/adapter_shriftweborgbfa.py +++ b/fanficfare/adapters/adapter_shriftweborgbfa.py @@ -129,7 +129,7 @@ class BFAArchiveShriftwebOrgSiteAdapter(BaseSiteAdapter): # This site has the entire story on one page, so we will be using the normalized URL as # the chapterUrl and the Title as the chapter Title - self.chapterUrls.append((self.story.getMetadata('title'), url)) + self.add_chapter(self.story.getMetadata('title'), url) ## i would take this out, as it is not really needed, but the calibre plugin uses it, ## so it's staying diff --git a/fanficfare/adapters/adapter_siyecouk.py b/fanficfare/adapters/adapter_siyecouk.py index 899bc8ba..15628ff9 100644 --- a/fanficfare/adapters/adapter_siyecouk.py +++ b/fanficfare/adapters/adapter_siyecouk.py @@ -118,13 +118,10 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX # Find the chapters (from soup, not authsoup): for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/siye/'+chapter['href']) - if self.chapterUrls: - self.story.setMetadata('numChapters',len(self.chapterUrls)) - else: - self.chapterUrls.append((self.story.getMetadata('title'),url)) - self.story.setMetadata('numChapters',1) + if self.num_chapters() < 1: + self.add_chapter(self.story.getMetadata('title'),url) # The stuff we can get from the chapter list/one-shot page are # in the first table with 95% width. diff --git a/fanficfare/adapters/adapter_spikeluvercom.py b/fanficfare/adapters/adapter_spikeluvercom.py index b02e60cf..adf8d2eb 100644 --- a/fanficfare/adapters/adapter_spikeluvercom.py +++ b/fanficfare/adapters/adapter_spikeluvercom.py @@ -204,7 +204,7 @@ class SpikeluverComAdapter(BaseSiteAdapter): title = stripHTML(chapter_anchor) url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) - self.chapterUrls.append((title, url)) + self.add_chapter(title, url) def getChapterText(self, url): url += self.AGE_CONSENT_URL_SUFFIX diff --git a/fanficfare/adapters/adapter_squidgeorgpeja.py b/fanficfare/adapters/adapter_squidgeorgpeja.py index dc532122..c49acaf2 100644 --- a/fanficfare/adapters/adapter_squidgeorgpeja.py +++ b/fanficfare/adapters/adapter_squidgeorgpeja.py @@ -135,11 +135,10 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter): chapterselect=soup.find('select',{'name':'chapter'}) if chapterselect: for ch in chapterselect.findAll('option'): - self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value'])) + self.add_chapter(ch,'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']) else: - self.chapterUrls.append((title,url)) + self.add_chapter(title,url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # utility method diff --git a/fanficfare/adapters/adapter_storiesofardacom.py b/fanficfare/adapters/adapter_storiesofardacom.py index d1189495..e31def73 100644 --- a/fanficfare/adapters/adapter_storiesofardacom.py +++ b/fanficfare/adapters/adapter_storiesofardacom.py @@ -104,12 +104,11 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter): # Find the chapters: chapterview.asp?sid=7000&cid=30919 chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$")) if len(chapters)==1: - self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href'])) + self.add_chapter(self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']) else: for chapter in chapters: - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) summary = soup.find('td', {'colspan' : '3'}) summary.name='div' # change td to div. Makes Calibre diff --git a/fanficfare/adapters/adapter_storiesonlinenet.py b/fanficfare/adapters/adapter_storiesonlinenet.py index 7c94b558..2f224769 100644 --- a/fanficfare/adapters/adapter_storiesonlinenet.py +++ b/fanficfare/adapters/adapter_storiesonlinenet.py @@ -193,11 +193,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): logger.debug("Number of chapters: {0}".format(len(chapters))) for chapter in chapters: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+chapter['href'])) + self.add_chapter(chapter,'https://'+self.host+chapter['href']) else: - self.chapterUrls.append((self.story.getMetadata('title'),'https://'+self.host+'/s/'+self.story.getMetadata('storyId'))) + self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+'/s/'+self.story.getMetadata('storyId')) - self.story.setMetadata('numChapters',len(self.chapterUrls)) self.getStoryMetadataFromAuthorPage() @@ -217,7 +216,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): # Remove all the metadata elements to leave and preamble text. This is usually # a notice or a forward. - if len(self.chapterUrls) > 1: + if self.num_chapters() > 1: header = soup.find('header') header.extract() else: diff --git a/fanficfare/adapters/adapter_sugarquillnet.py b/fanficfare/adapters/adapter_sugarquillnet.py index e5237b29..65c43093 100644 --- a/fanficfare/adapters/adapter_sugarquillnet.py +++ b/fanficfare/adapters/adapter_sugarquillnet.py @@ -118,9 +118,8 @@ class SugarQuillNetAdapter(BaseSiteAdapter): for chapter in chapters: if chapter.string == 'Default': chapter.string = 'Chapter 1' - self.chapterUrls.append((chapter.string, '{0}&chapno={1}'.format(self.url,chapter['value']))) + self.add_chapter(chapter.string, '{0}&chapno={1}'.format(self.url,chapter['value'])) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ## This site doesn't have much metadata, so we will get what we can. ## The metadata is all on the author's page, so we have to get it to parse. diff --git a/fanficfare/adapters/adapter_tenhawkpresentscom.py b/fanficfare/adapters/adapter_tenhawkpresentscom.py index 057efb51..89bc3d81 100644 --- a/fanficfare/adapters/adapter_tenhawkpresentscom.py +++ b/fanficfare/adapters/adapter_tenhawkpresentscom.py @@ -147,9 +147,8 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) def defaultGetattr(d,k): try: diff --git a/fanficfare/adapters/adapter_test1.py b/fanficfare/adapters/adapter_test1.py index 2515b4e0..940dd114 100644 --- a/fanficfare/adapters/adapter_test1.py +++ b/fanficfare/adapters/adapter_test1.py @@ -76,18 +76,13 @@ class TestSiteAdapter(BaseSiteAdapter): self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr)) #print("set:%s->%s"%(key,self.story.getMetadata(key))) - self.chapterUrls = [] if self.has_config(sections,'chapter_urls'): for l in self.get_config(sections,'chapter_urls').splitlines() : if l: - self.chapterUrls.append( (l[1+l.index(','):],l[:l.index(',')]) ) + self.add_chapter(l[1+l.index(','):],l[:l.index(',')]) else: for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1): - self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) ) - # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"), - # ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"), - # ] - self.story.setMetadata('numChapters',len(self.chapterUrls)) + self.add_chapter(chap,self.url+"&chapter=%d"%j) return @@ -285,33 +280,34 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" self.story.setMetadata('metaB','01245') self.story.setMetadata('metaC','The mighty metaC!') - self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"), - ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"), - ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"), - ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"), - ('Chapter 4',self.url+"&chapter=5"), - ('Chapter 5',self.url+"&chapter=6"), - ('Chapter 6',self.url+"&chapter=7"), - ('Chapter 7',self.url+"&chapter=8"), - ('Chapter 8',self.url+"&chapter=9"), - #('Chapter 9',self.url+"&chapter=0"), - #('Chapter 0',self.url+"&chapter=a"), - #('Chapter a',self.url+"&chapter=b"), - #('Chapter b',self.url+"&chapter=c"), - #('Chapter c',self.url+"&chapter=d"), - #('Chapter d',self.url+"&chapter=e"), - #('Chapter e',self.url+"&chapter=f"), - #('Chapter f',self.url+"&chapter=g"), - #('Chapter g',self.url+"&chapter=h"), - #('Chapter h',self.url+"&chapter=i"), - #('Chapter i',self.url+"&chapter=j"), - #('Chapter j',self.url+"&chapter=k"), - #('Chapter k',self.url+"&chapter=l"), - #('Chapter l',self.url+"&chapter=m"), - #('Chapter m',self.url+"&chapter=n"), - #('Chapter n',self.url+"&chapter=o"), - ] - self.story.setMetadata('numChapters',len(self.chapterUrls)) + chapters = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"), + ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"), + ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"), + ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"), + ('Chapter 4',self.url+"&chapter=5"), + ('Chapter 5',self.url+"&chapter=6"), + ('Chapter 6',self.url+"&chapter=7"), + ('Chapter 7',self.url+"&chapter=8"), + ('Chapter 8',self.url+"&chapter=9"), + #('Chapter 9',self.url+"&chapter=0"), + #('Chapter 0',self.url+"&chapter=a"), + #('Chapter a',self.url+"&chapter=b"), + #('Chapter b',self.url+"&chapter=c"), + #('Chapter c',self.url+"&chapter=d"), + #('Chapter d',self.url+"&chapter=e"), + #('Chapter e',self.url+"&chapter=f"), + #('Chapter f',self.url+"&chapter=g"), + #('Chapter g',self.url+"&chapter=h"), + #('Chapter h',self.url+"&chapter=i"), + #('Chapter i',self.url+"&chapter=j"), + #('Chapter j',self.url+"&chapter=k"), + #('Chapter k',self.url+"&chapter=l"), + #('Chapter l',self.url+"&chapter=m"), + #('Chapter m',self.url+"&chapter=n"), + #('Chapter n',self.url+"&chapter=o"), + ] + for c in chapters: + self.add_chapter(c[0],c[1]) def getChapterText(self, url): diff --git a/fanficfare/adapters/adapter_themasquenet.py b/fanficfare/adapters/adapter_themasquenet.py index 732eddf3..60f7482b 100644 --- a/fanficfare/adapters/adapter_themasquenet.py +++ b/fanficfare/adapters/adapter_themasquenet.py @@ -184,9 +184,8 @@ class TheMasqueNetAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.section + chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host + self.section + chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_thepetulantpoetesscom.py b/fanficfare/adapters/adapter_thepetulantpoetesscom.py index 9b6a39d9..a1a70f5e 100644 --- a/fanficfare/adapters/adapter_thepetulantpoetesscom.py +++ b/fanficfare/adapters/adapter_thepetulantpoetesscom.py @@ -141,16 +141,15 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter): # Find the chapters: chapters=soup.find('select', {'name' : 'sid'}) if chapters == None: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['value']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # make sure that the story id is from the first chapter - self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0]) + self.story.setMetadata('storyId',self.get_chapter(0,'url').split('=')[1].split('&')[0]) #locate the story on author's page index = 1 diff --git a/fanficfare/adapters/adapter_tolkienfanfiction.py b/fanficfare/adapters/adapter_tolkienfanfiction.py index e462549a..c53d65fd 100644 --- a/fanficfare/adapters/adapter_tolkienfanfiction.py +++ b/fanficfare/adapters/adapter_tolkienfanfiction.py @@ -146,13 +146,12 @@ class TolkienFanfictionAdapter(BaseSiteAdapter): else: raise e - # chapterUrls # http://tolkienfanfiction.com/Story_Read_Chapter.php?CHid=5358 for chapterLink in soup.findAll("a", href=re.compile(r'Story_Read_Chapter\.php\?CHid=[0-9]+')): chapterTitle = chapterLink.string chapterUrl = 'http://' + self.host + '/' + chapterLink['href'] - self.chapterUrls.append((chapterTitle, chapterUrl)) - numChapters = len(self.chapterUrls) + self.add_chapter(chapterTitle, chapterUrl) + numChapters = self.num_chapters() self.story.setMetadata('numChapters', numChapters) logger.debug('Number of Chapters: %s' % numChapters) diff --git a/fanficfare/adapters/adapter_tomparisdormcom.py b/fanficfare/adapters/adapter_tomparisdormcom.py index 8563272b..76c67881 100644 --- a/fanficfare/adapters/adapter_tomparisdormcom.py +++ b/fanficfare/adapters/adapter_tomparisdormcom.py @@ -107,9 +107,8 @@ class TomParisDormComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_trekfanfictionnet.py b/fanficfare/adapters/adapter_trekfanfictionnet.py index fe3d4ce7..11603c5a 100644 --- a/fanficfare/adapters/adapter_trekfanfictionnet.py +++ b/fanficfare/adapters/adapter_trekfanfictionnet.py @@ -126,13 +126,12 @@ class TrekFanFictionNetSiteAdapter(BaseSiteAdapter): # This site has each story on one page, so there are no chapters to get. Will use original ## url since we can't get the chapter without this, I'm leaving it in. - self.chapterUrls.append((self.story.getMetadata('title'), url)) + self.add_chapter(self.story.getMetadata('title'), url) ## I'm going to comment this out, because thereis always only one chapter for each story, ## so this is really not needed ## And I am uncommenting it because the rest of FFF expects ## there to always be numChapters, even if it's one. --Jimm - self.story.setMetadata('numChapters',len(self.chapterUrls)) # getting the rest of the metadata... there isn't much here, and the summary can only be # gotten on the author's page... so we'll get it to get the information from diff --git a/fanficfare/adapters/adapter_trekiverseorg.py b/fanficfare/adapters/adapter_trekiverseorg.py index 80bd915c..e32a8499 100644 --- a/fanficfare/adapters/adapter_trekiverseorg.py +++ b/fanficfare/adapters/adapter_trekiverseorg.py @@ -185,13 +185,12 @@ class TrekiverseOrgAdapter(BaseSiteAdapter): if len(chapters)==0: raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.") if len(chapters)==1: - self.chapterUrls.append((self.story.getMetadata('title'),'https://'+self.host+'/efiction/'+chapters[0]['href'])) + self.add_chapter(self.story.getMetadata('title'),'https://'+self.host+'/efiction/'+chapters[0]['href']) else: for chapter in chapters: # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/efiction/'+chapter['href'])) + self.add_chapter(chapter,'https://'+self.host+'/efiction/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_tthfanficorg.py b/fanficfare/adapters/adapter_tthfanficorg.py index f81bd818..206a349e 100644 --- a/fanficfare/adapters/adapter_tthfanficorg.py +++ b/fanficfare/adapters/adapter_tthfanficorg.py @@ -234,9 +234,9 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): self.story.addToList('authorUrl','https://'+self.host+autha['href'].replace("/Author-","/AuthorStories-")) self.story.addToList('author',stripHTML(autha)) # include leading number to match 1. ... 2. ... - self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1, - stripHTML(a), - stripHTML(autha)),'https://'+self.host+a['href'])) + self.add_chapter("%d. %s by %s"%(self.num_chapters()+1, + stripHTML(a), + stripHTML(autha)),'https://'+self.host+a['href']) except urllib2.HTTPError, e: if e.code == 404: @@ -249,15 +249,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): if select is None: # no selector found, so it's a one-chapter story. - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: allOptions = select.findAll('option') for o in allOptions: url = "https://"+self.host+o['value'] # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(o),url)) + self.add_chapter(o,url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) verticaltable = soup.find('table', {'class':'verticaltable'}) diff --git a/fanficfare/adapters/adapter_twilightarchivescom.py b/fanficfare/adapters/adapter_twilightarchivescom.py index e60ccab2..7cabe5c7 100644 --- a/fanficfare/adapters/adapter_twilightarchivescom.py +++ b/fanficfare/adapters/adapter_twilightarchivescom.py @@ -101,11 +101,10 @@ class TwilightArchivesComAdapter(BaseSiteAdapter): chapters=soup.find('ol', {'class' : 'chapters'}) if chapters != None: for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")): - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+chapter['href']) else: - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # rated, genre, warnings, seires diff --git a/fanficfare/adapters/adapter_twilightednet.py b/fanficfare/adapters/adapter_twilightednet.py index 280d2a76..44f689a9 100644 --- a/fanficfare/adapters/adapter_twilightednet.py +++ b/fanficfare/adapters/adapter_twilightednet.py @@ -137,9 +137,8 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) def defaultGetattr(d,k): try: diff --git a/fanficfare/adapters/adapter_unknowableroomorg.py b/fanficfare/adapters/adapter_unknowableroomorg.py index b6f1a595..eb033a46 100644 --- a/fanficfare/adapters/adapter_unknowableroomorg.py +++ b/fanficfare/adapters/adapter_unknowableroomorg.py @@ -128,14 +128,13 @@ class UnknowableRoomOrgSiteAdapter(BaseSiteAdapter): for chapter in soup.find('select').find_all('option', value=re.compile( '/'+self.story.getMetadata('storyId')+r'/\d+')): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value'])) + self.add_chapter(chapter,'http://'+self.host+chapter['value']) ## One chapter stories do not have a listing for the chapters, so we have to check to make - ## sure, and if there aren't any chapterUrls, we set it to the Url entered. - if len(self.chapterUrls) == 0: - self.chapterUrls.append((self.story.getMetadata('title'), url)) + ## sure, and if there aren't any chapters, we set it to the Url entered. + if self.num_chapters() == 0: + self.add_chapter(self.story.getMetadata('title'), url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # Most of the metadata can be gotten from the story page, but it can all be gotten from the # author's fic page, so we are going to get it from there. Unless there is no author page, diff --git a/fanficfare/adapters/adapter_voracity2eficcom.py b/fanficfare/adapters/adapter_voracity2eficcom.py index 06e13335..3ee014ed 100644 --- a/fanficfare/adapters/adapter_voracity2eficcom.py +++ b/fanficfare/adapters/adapter_voracity2eficcom.py @@ -225,7 +225,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter): chapter_anchor = b_tag.a title = chapter_anchor.string url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) - self.chapterUrls.append((title, url)) + self.add_chapter(title, url) def getChapterText(self, url): url += self.AGE_CONSENT_URL_SUFFIX diff --git a/fanficfare/adapters/adapter_walkingtheplankorg.py b/fanficfare/adapters/adapter_walkingtheplankorg.py index 56d6efa0..037c87ac 100644 --- a/fanficfare/adapters/adapter_walkingtheplankorg.py +++ b/fanficfare/adapters/adapter_walkingtheplankorg.py @@ -122,9 +122,8 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl)) + self.add_chapter(chapter,'http://'+self.host+'/archive/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_wattpadcom.py b/fanficfare/adapters/adapter_wattpadcom.py index ab61eef8..5e23a9ac 100644 --- a/fanficfare/adapters/adapter_wattpadcom.py +++ b/fanficfare/adapters/adapter_wattpadcom.py @@ -116,8 +116,7 @@ class WattpadComAdapter(BaseSiteAdapter): self.story.setMetadata('dateUpdated', makeDate(storyInfo['modifyDate'].rstrip('Z'), "%Y-%m-%dT%H:%M:%S")) self.story.setMetadata('datePublished', makeDate(storyInfo['createDate'].rstrip('Z'), "%Y-%m-%dT%H:%M:%S")) - self.chapterUrls = [(part['title'], part['url']) for part in storyInfo['parts']] - self.story.setMetadata('numChapters', len(self.chapterUrls)) + [self.add_chapter(part['title'], part['url']) for part in storyInfo['parts']] self.setCoverImage(storyInfo['url'], storyInfo['cover'].replace('-256-','-512-')) self.story.setMetadata('language', storyInfo['language']['name']) diff --git a/fanficfare/adapters/adapter_webnovelcom.py b/fanficfare/adapters/adapter_webnovelcom.py index a6be5fd5..e17a0ea9 100644 --- a/fanficfare/adapters/adapter_webnovelcom.py +++ b/fanficfare/adapters/adapter_webnovelcom.py @@ -185,9 +185,8 @@ class WWWWebNovelComAdapter(BaseSiteAdapter): chap_title = 'Chapter ' + unicode(chap['index']) + ' - ' + chap['name'] chap_Url = url.rstrip('/') + '/' + chap['id'] - self.chapterUrls.append((chap_title, chap_Url)) + self.add_chapter(chap_title, chap_Url) - self.story.setMetadata('numChapters', len(self.chapterUrls)) if get_cover: cover_meta = soup.find('div', {'class': 'g_col_4'}).find('img') diff --git a/fanficfare/adapters/adapter_whoficcom.py b/fanficfare/adapters/adapter_whoficcom.py index 06c27a7d..af8a608e 100644 --- a/fanficfare/adapters/adapter_whoficcom.py +++ b/fanficfare/adapters/adapter_whoficcom.py @@ -83,7 +83,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter): if select is None: # no selector found, so it's a one-chapter story. - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) else: allOptions = select.findAll('option') for o in allOptions: @@ -91,9 +91,8 @@ class WhoficComSiteAdapter(BaseSiteAdapter): # just in case there's tags, like in chapter titles. title = "%s" % o title = re.sub(r'<[^>]+>','',title) - self.chapterUrls.append((title,url)) + self.add_chapter(title,url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ## Whofic.com puts none of the other meta data in the chapters ## or even the story chapter index page. Need to scrape the diff --git a/fanficfare/adapters/adapter_wolverineandroguecom.py b/fanficfare/adapters/adapter_wolverineandroguecom.py index e569ae90..4db5801c 100644 --- a/fanficfare/adapters/adapter_wolverineandroguecom.py +++ b/fanficfare/adapters/adapter_wolverineandroguecom.py @@ -110,9 +110,8 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'https://'+self.host+'/wrfa/'+chapter['href'])) + self.add_chapter(chapter,'https://'+self.host+'/wrfa/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_wraithbaitcom.py b/fanficfare/adapters/adapter_wraithbaitcom.py index 486bc154..47e659a5 100644 --- a/fanficfare/adapters/adapter_wraithbaitcom.py +++ b/fanficfare/adapters/adapter_wraithbaitcom.py @@ -133,9 +133,8 @@ class WraithBaitComAdapter(BaseSiteAdapter): else: add = "" # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter)+add,'https://'+self.host+'/'+chapter['href']+addurl)) + self.add_chapter(stripHTML(chapter)+add,'https://'+self.host+'/'+chapter['href']+addurl) - self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data # formating, so it's a little ugly. diff --git a/fanficfare/adapters/adapter_writingwhimsicalwanderingsnet.py b/fanficfare/adapters/adapter_writingwhimsicalwanderingsnet.py index fb5f2457..203bb0ad 100644 --- a/fanficfare/adapters/adapter_writingwhimsicalwanderingsnet.py +++ b/fanficfare/adapters/adapter_writingwhimsicalwanderingsnet.py @@ -111,9 +111,8 @@ class WritingWhimsicalwanderingsNetAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.add_chapter(chapter,'http://'+self.host+'/'+chapter['href']) - self.story.setMetadata('numChapters',len(self.chapterUrls)) ## This site's metadata is not very well formatted... so we have to cludge a bit.. ## The only ones I see that are, are Relationships and Warnings... diff --git a/fanficfare/adapters/adapter_wuxiaworldco.py b/fanficfare/adapters/adapter_wuxiaworldco.py index b335a1da..b1d91572 100644 --- a/fanficfare/adapters/adapter_wuxiaworldco.py +++ b/fanficfare/adapters/adapter_wuxiaworldco.py @@ -92,9 +92,8 @@ class WuxiaWorldCoSiteAdapter(BaseSiteAdapter): for a in soup.select('#list a'): url = urlparse.urljoin(self.url, a['href']) title = stripHTML(a) - self.chapterUrls.append((title, url)) + self.add_chapter(title, url) - self.story.setMetadata('numChapters', len(self.chapterUrls)) def getChapterText(self, url): logger.debug('Getting chapter text from: %s', url) diff --git a/fanficfare/adapters/adapter_wuxiaworldcom.py b/fanficfare/adapters/adapter_wuxiaworldcom.py index 61060640..ce159e48 100644 --- a/fanficfare/adapters/adapter_wuxiaworldcom.py +++ b/fanficfare/adapters/adapter_wuxiaworldcom.py @@ -112,11 +112,10 @@ class WuxiaWorldComSiteAdapter(BaseSiteAdapter): for a in soup.select('#accordion .chapter-item > a'): title = stripHTML(a) url = urlparse.urljoin(self.url, a['href']) - self.chapterUrls.append((title, url)) + self.add_chapter(title, url) - self.story.setMetadata('numChapters', len(self.chapterUrls)) - last_chapter_data = self._fetchUrl(self.chapterUrls[-1][1]) + last_chapter_data = self._fetchUrl(self.get_chapter(-1,'url')) last_chapter_soup = self.make_soup(last_chapter_data) last_chapter_ld = self._parse_linked_data(last_chapter_soup) self.story.setMetadata('dateUpdated', self._parse_date(last_chapter_ld['datePublished'])) diff --git a/fanficfare/adapters/adapter_wwwaneroticstorycom.py b/fanficfare/adapters/adapter_wwwaneroticstorycom.py index cccd15e4..19f5453a 100644 --- a/fanficfare/adapters/adapter_wwwaneroticstorycom.py +++ b/fanficfare/adapters/adapter_wwwaneroticstorycom.py @@ -122,17 +122,12 @@ class WWWAnEroticStoryComAdapter(BaseSiteAdapter): description = description.encode('utf-8','ignore').strip()[0:350].decode('utf-8','ignore')+'...' self.setDescription(url,'Excerpt from beginning of story: '+description+'...') - # Get chapter URLs - self.chapterUrls = [] - ### This is a 1 page/ 1 story site, so the only chapterurl is the current story - self.chapterUrls.append(('1', self.url)) + self.add_chapter('1', self.url) # Setting the status to complete self.story.setMetadata('status', 'Completed') - self.story.setMetadata('numChapters', len(self.chapterUrls)) - ## Getting the date Posted and setting the Published and Updated metadata datePosted = soup1.find('span', {'class':'data divided'}).text.replace('on', '').strip() self.story.setMetadata('datePublished', makeDate(datePosted, self.dateformat)) diff --git a/fanficfare/adapters/adapter_wwwarea52hkhnet.py b/fanficfare/adapters/adapter_wwwarea52hkhnet.py index 82260411..efbc22e9 100644 --- a/fanficfare/adapters/adapter_wwwarea52hkhnet.py +++ b/fanficfare/adapters/adapter_wwwarea52hkhnet.py @@ -122,9 +122,8 @@ class WWWArea52HKHNetAdapter(BaseSiteAdapter): # There is only one 'chapter' for each story, so we go with the self.url # and the title of the story for the heading - self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.add_chapter(self.story.getMetadata('title'),url) - self.story.setMetadata('numChapters',len(self.chapterUrls)) storya = None authsoup = None diff --git a/fanficfare/adapters/adapter_wwwlushstoriescom.py b/fanficfare/adapters/adapter_wwwlushstoriescom.py index a4024fc8..e14da56f 100644 --- a/fanficfare/adapters/adapter_wwwlushstoriescom.py +++ b/fanficfare/adapters/adapter_wwwlushstoriescom.py @@ -164,8 +164,7 @@ class WWWLushStoriesComAdapter(BaseSiteAdapter): # XXX # Find the chapters: # The stories on this site are all on one page, so we use the original URL - self.chapterUrls.append((self.story.getMetadata('title'),self.url)) - self.story.setMetadata('numChapters',len(self.chapterUrls)) + self.add_chapter(self.story.getMetadata('title'),self.url) self.story.setMetadata('status', 'Completed') #Need to get the metadata from the author's story page diff --git a/fanficfare/adapters/adapter_wwwnovelallcom.py b/fanficfare/adapters/adapter_wwwnovelallcom.py index aaa9f9e1..542adebb 100644 --- a/fanficfare/adapters/adapter_wwwnovelallcom.py +++ b/fanficfare/adapters/adapter_wwwnovelallcom.py @@ -203,13 +203,12 @@ class WWWNovelAllComAdapter(BaseSiteAdapter): # a = li.find('a') ctitle = re.sub(r"^%s(.+)$" % re.escape(title), r"\1", a['title'], 0, re.UNICODE | re.IGNORECASE).strip() - self.chapterUrls.append((ctitle, a['href'])) + self.add_chapter(ctitle, a['href']) cdates.sort() self.story.setMetadata('datePublished', cdates[0]) self.story.setMetadata('dateUpdated', cdates[-1]) - self.story.setMetadata('numChapters', len(self.chapterUrls)) def getChapterText(self, url): data = self._fetchUrl(url) diff --git a/fanficfare/adapters/adapter_wwwutopiastoriescom.py b/fanficfare/adapters/adapter_wwwutopiastoriescom.py index ff14f2ca..19db47f5 100644 --- a/fanficfare/adapters/adapter_wwwutopiastoriescom.py +++ b/fanficfare/adapters/adapter_wwwutopiastoriescom.py @@ -135,10 +135,9 @@ class WWWUtopiastoriesComAdapter(BaseSiteAdapter): self.story.setMetadata('title',stripHTML(a)) # Find the chapters: - ## This site is a 1 story/page site, so I'm setting the chapterUrls to the entered url and + ## This site is a 1 story/page site, so I'm setting the chapter to the entered url and # the status to complete - self.chapterUrls.append(('',url)) - self.story.setMetadata('numChapters',1) + self.add_chapter('',url) self.story.setMetadata('status', 'Completed') diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 9b4f2fbb..599a7566 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -142,6 +142,25 @@ class BaseSiteAdapter(Configurable): self.chapterLast=int(last)-1 self.story.set_chapters_range(first,last) + def add_chapter(self,title,url): + self.chapterUrls.append((stripHTML(title),url)) + self.story.setMetadata('numChapters', self.num_chapters()) + + def num_chapters(self): + return len(self.chapterUrls) + + def get_chapter(self,i,attr): + if attr == 'url': + return self.chapterUrls[i][1] + return None + + def get_chapters(self): + return copy.copy(self.chapterUrls) + + def del_chapter(self,i): + del self.chapterUrls[i] + self.story.setMetadata('numChapters', self.num_chapters()) + # Does the download the first time it's called. def getStory(self): if not self.storyDone: @@ -327,7 +346,7 @@ class BaseSiteAdapter(Configurable): return self.extractChapterUrlsAndMetadata() def extractChapterUrlsAndMetadata(self): - "Needs to be overriden in each adapter class. Populates self.story metadata and self.chapterUrls" + "Needs to be overriden in each adapter class. Populates self.story metadata" pass def getChapterTextNum(self, url, index): diff --git a/fanficfare/adapters/base_efiction_adapter.py b/fanficfare/adapters/base_efiction_adapter.py index 075d2ad0..69df5ce8 100644 --- a/fanficfare/adapters/base_efiction_adapter.py +++ b/fanficfare/adapters/base_efiction_adapter.py @@ -452,7 +452,7 @@ class BaseEfictionAdapter(BaseSiteAdapter): chapterLink = b.findNext("a") chapterLink['href'] = "%s&chapter=%s" % (self.url, chapterId) if chapterLink.string !='Back to index': - self.chapterUrls.append((chapterLink.string, chapterLink['href'])) + self.add_chapter(chapterLink.string, chapterLink['href']) ## Store reference to soup for getChapterText self.html = soup diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 6fb32525..6ac94c8f 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -324,7 +324,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): self.reader = topsoup.find('a',href=re.compile(r'\.'+self.story.getMetadata('storyId')+r"/reader$")) is not None if self.getConfig('always_include_first_post'): - self.chapterUrls.append((first_post_title,useurl)) + self.add_chapter(first_post_title,useurl) use_threadmark_chaps = True @@ -353,7 +353,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): if tm.get('words',None): words = words + tm['words'] - self.chapterUrls.append((prepend+tm['title'],tm['url'])) + self.add_chapter(prepend+tm['title'],tm['url']) if words and self.getConfig('use_threadmark_wordcounts',True): self.story.setMetadata('numWords',words) souptag = souptag.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above) @@ -401,16 +401,16 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): # otherwise, use first post links--include first post since # that's often also the first chapter. - if not self.chapterUrls: - self.chapterUrls.append((first_post_title,useurl)) + if self.num_chapters < 1: + self.add_chapter(first_post_title,useurl) for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: (is_chapter_url,url) = self._is_normalize_chapterurl(url) if is_chapter_url and name != u"\u2191": # skip quote links as indicated by up arrow character. - self.chapterUrls.append((name,url)) - if url == useurl and first_post_title == self.chapterUrls[0][0] \ + self.add_chapter(name,url) + if url == useurl and first_post_title == self.get_chapter(0,'url') \ and not self.getConfig('always_include_first_post',False): # remove "First Post" if included in list. - del self.chapterUrls[0] + self.del_chapter(0) # Didn't use threadmarks, so take created/updated dates # from the 'first' posting created and updated. @@ -423,8 +423,6 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): if date: self.story.setMetadata('dateUpdated', date) - self.story.setMetadata('numChapters',len(self.chapterUrls)) - def make_date(self,parenttag): # forums use a BS thing where dates # can appear different if recent. datestr=None diff --git a/fanficfare/cli.py b/fanficfare/cli.py index 19ad1b6c..e9738c20 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -435,7 +435,7 @@ def do_download(arg, if options.metaonly: metadata = adapter.getStoryMetadataOnly().getAllMetadata() metadata['zchapters'] = [] - for i, x in enumerate(adapter.chapterUrls): + for i, x in enumerate(adapter.get_chapters()): metadata['zchapters'].append((i+1,x[0],x[1])) if not options.metaonly and adapter.getConfig('pre_process_cmd'):