diff --git a/fanficfare/adapters/adapter_silmarillionwritersguildorg.py b/fanficfare/adapters/adapter_silmarillionwritersguildorg.py index 990757a8..eb778fec 100644 --- a/fanficfare/adapters/adapter_silmarillionwritersguildorg.py +++ b/fanficfare/adapters/adapter_silmarillionwritersguildorg.py @@ -68,6 +68,13 @@ class SilmarillionWritersGuildOrgAdapter(BaseSiteAdapter): def getSiteURLPattern(self): return r"https?://"+re.escape(self.getSiteDomain()+"/archive/home/viewstory.php?sid=")+r"\d+$" + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + ## Getting the chapter list and the meta data def extractChapterUrlsAndMetadata(self): @@ -119,33 +126,40 @@ class SilmarillionWritersGuildOrgAdapter(BaseSiteAdapter): #logger.debug("Series Url: "+seriesUrl) # Get Series page and convert to soup - seriesPageSoup = self.make_soup(self._fetchUrl(seriesUrl)) + seriesPageSoup = self.make_soup(self._fetchUrl(seriesUrl+"&offset=0")) + ## &offset=0 is the same as the first page, by adding + ## that, the page cache will save us from fetching it + ## twice in the loop below. + # Find Series page sub-pages seriesPageUrlList = [] + seriesStoryList = [] for i in seriesPageSoup.findAll('a', href=re.compile("viewseries.php\?seriesid=\d+&offset=\d+$")): - # Don't include url from next button, is another http request and parse + could cause more bugs! - if i.string != '[Next]': - seriesPageUrlList.append(i) + # Don't include url from next button, is another http request and parse + could cause more bugs! + if i.string != '[Next]': + seriesPageUrlList.append(i) #get urls from all subpages and append to list - seriesStoryList = [] + i=1 for seriesPagePageUrl in seriesPageUrlList: seriesPagePageSoup = self.make_soup(self._fetchUrl('https://'+self.host+'/archive/home/'+seriesPagePageUrl['href'])) - seriesPagePageStoryList = seriesPagePageSoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) - - for seriesPagePageStoryUrl in seriesPagePageStoryList: - seriesStoryList.append(seriesPagePageStoryUrl) - - # Find series index for story - i=1 - for seriesStoriesUrl in seriesStoryList: - if seriesStoriesUrl['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(seriesName, i) - #logger.debug("Series Name: "+ seriesName) - #logger.debug("Series Index: "+i) - break - i+=1 + storyHeaders = seriesPagePageSoup.findAll('h5') + ## can't just search for story URLs, some story + ## descs also contain story URLs. Looks like only + ## story titles are