From f0eaae888315f92d389467c775b81c9f0e27b16d Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 1 Dec 2014 12:31:40 -0600 Subject: [PATCH] Merge default updates into bs4 branch. --- fanficdownloader/adapters/__init__.py | 3 +- .../adapters/adapter_csiforensicscom.py | 293 ++++++++++++++---- ...ansorg.py => adapter_fanficcastletvnet.py} | 33 +- .../adapters/adapter_fanfictionjunkiesde.py | 291 +++++++++++++++++ 4 files changed, 542 insertions(+), 78 deletions(-) rename fanficdownloader/adapters/{adapter_castlefansorg.py => adapter_fanficcastletvnet.py} (89%) create mode 100644 fanficdownloader/adapters/adapter_fanfictionjunkiesde.py diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index 0fe37961..2414504a 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -29,7 +29,7 @@ from ..configurable import Configuration import adapter_test1 import adapter_fanfictionnet -import adapter_castlefansorg +import adapter_fanficcastletvnet import adapter_fictionalleyorg import adapter_fictionpresscom import adapter_ficwadcom @@ -139,6 +139,7 @@ import adapter_samandjacknet import adapter_csiforensicscom import adapter_lotrfanfictioncom import adapter_fhsarchivecom +import adapter_fanfictionjunkiesde ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficdownloader/adapters/adapter_csiforensicscom.py b/fanficdownloader/adapters/adapter_csiforensicscom.py index 4ae84349..b1154d78 100644 --- a/fanficdownloader/adapters/adapter_csiforensicscom.py +++ b/fanficdownloader/adapters/adapter_csiforensicscom.py @@ -1,56 +1,237 @@ -# -*- coding: utf-8 -*- - -# Copyright 2014 Fanficdownloader team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Software: eFiction -import re -from base_efiction_adapter import BaseEfictionAdapter - -class CSIForensicsComAdapter(BaseEfictionAdapter): - - @staticmethod - def getSiteDomain(): - return 'csi-forensics.com' - - @classmethod - def getPathToArchive(self): - return '' - - @classmethod - def getSiteAbbrev(self): - return 'csiforensics' - - @classmethod - def getDateFormat(self): - return "%d %b %Y" - - def handleMetadataPair(self, key, value): - if key == 'Warnings': - for val in re.split("\s*,\s*", value): - if value == 'None': - return - else: - self.story.addToList('warnings', val) - - elif 'Categories' in key: - for val in re.split("\s*>\s*", value): - self.story.addToList('category', val) - else: - super(CSIForensicsComAdapter, self).handleMetadataPair(key, value) - -def getClass(): - return CSIForensicsComAdapter - +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + + +def getClass(): + return CSIForensicsComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class CSIForensicsComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','csiforensics') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'csi-forensics.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=5&skin=elegantcsi" + else: + addurl="&skin=elegantcsi" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "This story is rated NC-17, and therefore is not suitable for minors. If you are below the age required to view such material in your locality, please return from whence you came." in data: # XXX + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + + pt = soup.find('div', {'id' : 'pagetitle'}) + a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',a.string) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Rating + rate = stripHTML(soup.find('div',{'id':'pagetitle'})) + rate = rate[rate.rindex('[')+1:rate.rindex(']')] + self.story.setMetadata('rating', rate) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + smalldiv = soup.find('div', {'class' : 'small'}) + + + chars = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + metatext = stripHTML(smalldiv) + + if 'Completed: Yes' in metatext: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + word=soup.find(text=re.compile("Word count:")).split(':') + self.story.setMetadata('numWords', word[1]) + + cats = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + warnings = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=class(&)type_id=2(&)classid=\d+')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + date=soup.find('div',{'class' : 'bottom'}) + pd=date.find(text=re.compile("Published:")).string.split(': ') + self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat)) + + # Rated: NC-17
etc + labels = soup.findAll('span',{'class':'label'}) + pub=0 + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Genres' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + smalldiv.extract() + + # Summary + summary = soup.find('div', {'class' : 'content'}) + self.setDescription(url,summary) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) + \ No newline at end of file diff --git a/fanficdownloader/adapters/adapter_castlefansorg.py b/fanficdownloader/adapters/adapter_fanficcastletvnet.py similarity index 89% rename from fanficdownloader/adapters/adapter_castlefansorg.py rename to fanficdownloader/adapters/adapter_fanficcastletvnet.py index 47a6dedb..3bc2040e 100644 --- a/fanficdownloader/adapters/adapter_castlefansorg.py +++ b/fanficdownloader/adapters/adapter_fanficcastletvnet.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2011 Fanficdownloader team +# Copyright 2014 Fanficdownloader team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,15 +28,6 @@ from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, makeDate -# By virtue of being recent and requiring both is_adult and user/pass, -# adapter_fanficcastletvnet.py is the best choice for learning to -# write adapters--especially for sites that use the eFiction system. -# Most sites that have ".../viewstory.php?sid=123" in the story URL -# are eFiction. - -# For non-eFiction sites, it can be considerably more complex, but -# this is still a good starting point. - # In general an 'adapter' needs to do these five things: # - 'Register' correctly with the downloader @@ -53,11 +44,11 @@ from base_adapter import BaseSiteAdapter, makeDate # updated to reflect the class below it. That, plus getSiteDomain() # take care of 'Registering'. def getClass(): - return CastleFansOrgAdapter # XXX + return FanficCastleTVNetAdapter # XXX # Class name has to be unique. Our convention is camel case the # sitename with Adapter at the end. www is skipped. -class CastleFansOrgAdapter(BaseSiteAdapter): # XXX +class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX def __init__(self, config, url): BaseSiteAdapter.__init__(self, config, url) @@ -77,10 +68,10 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX # normalized story URL. # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. - self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId')) + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','cslf') # XXX + self.story.setMetadata('siteabbrev','csltv') # XXX # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior @@ -89,14 +80,14 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): # The site domain. Does have www here, if it uses it. - return 'castlefans.org' # XXX + return 'fanfic.castletv.net' # XXX @classmethod def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/fanfic/viewstory.php?sid=1234" + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" def getSiteURLPattern(self): - return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$" + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" ## Login seems to be reasonably standard across eFiction sites. def needToLoginCheck(self, data): @@ -119,7 +110,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX params['cookiecheck'] = '1' params['submit'] = 'Submit' - loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login' + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, params['penname'])) @@ -186,13 +177,13 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX # Find authorid and URL from... author url. a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) self.story.setMetadata('authorId',a['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/fanfic/'+a['href']) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) self.story.setMetadata('author',a.string) # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfic/'+chapter['href']+addurl)) + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) self.story.setMetadata('numChapters',len(self.chapterUrls)) @@ -277,7 +268,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX # Find Series name from series URL. a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) series_name = a.string - series_url = 'http://'+self.host+'/fanfic/'+a['href'] + series_url = 'http://'+self.host+'/'+a['href'] # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) diff --git a/fanficdownloader/adapters/adapter_fanfictionjunkiesde.py b/fanficdownloader/adapters/adapter_fanfictionjunkiesde.py new file mode 100644 index 00000000..616e1439 --- /dev/null +++ b/fanficdownloader/adapters/adapter_fanfictionjunkiesde.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# By virtue of being recent and requiring both is_adult and user/pass, +# adapter_fanficcastletvnet.py is the best choice for learning to +# write adapters--especially for sites that use the eFiction system. +# Most sites that have ".../viewstory.php?sid=123" in the story URL +# are eFiction. + +# For non-eFiction sites, it can be considerably more complex, but +# this is still a good starting point. + +# In general an 'adapter' needs to do these five things: + +# - 'Register' correctly with the downloader +# - Site Login (if needed) +# - 'Are you adult?' check (if needed--some do one, some the other, some both) +# - Grab the chapter list +# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page) +# - Grab the chapter texts + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return FanfictionJunkiesDeAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/efiction/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ffjde') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'fanfiction-junkies.de' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/efiction/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=1" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "For adults only " in data: # XXX + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + pagetitle = soup.find('h4') + ## Title + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',a.string) + + # Find authorid and URL from... author url. + a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/efiction/'+a['href']) + self.story.setMetadata('author',a.string) + + # Reviews + reviewdata = soup.find('div', {'id' : 'sort'}) + a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one. + self.story.setMetadata('reviews',stripHTML(a)) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + + # Rated: NC-17
etc + list = soup.find('div', {'class':'listbox'}) + + + labels = list.findAll('b') + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Zusammenfassung' in label: + self.setDescription(url,value) + + if 'Eingestuft' in label: + self.story.setMetadata('rating', value) + + if 'Wörter' in label: + self.story.setMetadata('numWords', value) + + if 'Kategorie' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Charaktere' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Abgeschlossen' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Veröffentlicht' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Aktualisiert' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/efiction/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div)