From 6d2f6577292e9ff738b884d1b6935662e77a3e74 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 10 May 2011 13:26:34 -0500 Subject: [PATCH] Support for thewriterscoffeeshop.com, allow both login/pass and Are Adult in one site. --- .../adapter_harrypotterfanfictioncom.py | 4 +- .../adapters/adapter_potionsandsnitchesnet.py | 4 +- fanficdownloader/adapters/adapter_test1.py | 8 +- .../adapter_thewriterscoffeeshopcom.py | 201 ++++++++++++++++++ .../adapters/adapter_twilightednet.py | 14 +- .../adapters/adapter_twiwritenet.py | 2 +- index.html | 9 +- login.html | 4 +- main.py | 9 +- 9 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py index 2efd2720..4f3de5e9 100644 --- a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py @@ -45,9 +45,9 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter): return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$" def needToLoginCheck(self, data): - if 'Registered Users Only.' in data \ + if 'Registered Users Only' in data \ or 'There is no such account on our website' in data \ - or "That password doesn't match the one in our database." in data: + or "That password doesn't match the one in our database" in data: return True else: return False diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py index 197f2336..7fb7e637 100644 --- a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py +++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py @@ -44,9 +44,9 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter): return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$" def needToLoginCheck(self, data): - if 'Registered Users Only.' in data \ + if 'Registered Users Only' in data \ or 'There is no such account on our website' in data \ - or "That password doesn't match the one in our database." in data: + or "That password doesn't match the one in our database" in data: return True else: return False diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index d2cb6457..c6d925eb 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -43,8 +43,11 @@ class TestSiteAdapter(BaseSiteAdapter): if self.story.getMetadata('storyId') == '668' and self.username != "Me" : raise exceptions.FailedToLogin(self.url,self.username) - - self.story.setMetadata(u'title',"Test Story Title "+self.crazystring) + + if self.story.getMetadata('storyId') == '664': + self.story.setMetadata(u'title',"Test Story Title "+self.crazystring) + else: + self.story.setMetadata(u'title',"Test Story Title") self.story.setMetadata('storyUrl',self.url) self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done @@ -92,6 +95,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"

Prologue

This is a fake adapter for testing purposes. Different storyId's will give different errors:

+

http://test1.com?sid=664 - Crazy string title

http://test1.com?sid=665 - raises AdultCheckRequired

http://test1.com?sid=666 - raises StoryDoesNotExist

http://test1.com?sid=667 - raises FailedToDownload on chapter 1

diff --git a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py new file mode 100644 index 00000000..18cc9d1b --- /dev/null +++ b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- + +import time +import datetime +import logging +import re +import urllib +import urllib2 + +import fanficdownloader.BeautifulSoup as bs +from fanficdownloader.htmlcleanup import stripHTML +import fanficdownloader.exceptions as exceptions + +from base_adapter import BaseSiteAdapter, utf8FromSoup + +class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','twcs') + self.decode = "ISO-8859-1" + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId')) + + + @staticmethod + def getSiteDomain(): + return 'www.thewriterscoffeeshop.com' + + @classmethod + def getAcceptDomains(cls): + return [cls.getSiteDomain()] + + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/library/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/library/viewstory.php?sid=")+r"\d+$" + + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/library/user.php?action=login' + logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logging.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + addurl = "&ageconsent=ok&warning=3" + else: + addurl="" + + url = self.url+'&index=1'+addurl + logging.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Age Consent Required" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',a.string) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/library/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/library/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class')) + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', datetime.datetime.fromtimestamp(time.mktime(time.strptime(stripHTML(value), "%B %d, %Y")))) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', datetime.datetime.fromtimestamp(time.mktime(time.strptime(stripHTML(value), "%B %d, %Y")))) + + + def getChapterText(self, url): + + logging.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + span = soup.find('div', {'id' : 'story'}) + + if None == span: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return utf8FromSoup(span) + +def getClass(): + return TheWritersCoffeeShopComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py index 6a654e46..ec1b5140 100644 --- a/fanficdownloader/adapters/adapter_twilightednet.py +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -20,7 +20,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): self.story.setMetadata('siteabbrev','tw') self.decode = "utf8" self.story.addToList("category","Twilight") - self.username = "NoneGiven" # if left empty, twilighted.net doesn't return any message at all. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. self.password = "" # get storyId from url--url validation guarantees query is only sid=1234 @@ -46,9 +46,9 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): return re.escape("http://")+r"(www\.)?"+re.escape("twilighted.net/viewstory.php?sid=")+r"\d+$" def needToLoginCheck(self, data): - if 'Registered Users Only.' in data \ + if 'Registered Users Only' in data \ or 'There is no such account on our website' in data \ - or "That password doesn't match the one in our database." in data: + or "That password doesn't match the one in our database" in data: return True else: return False @@ -120,14 +120,6 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): self.story.setMetadata('numChapters',len(self.chapterUrls)) - ## - ## Summary, strangely, is in the content attr of a tag - ## which is escaped HTML. Unfortunately, we can't use it because they don't - ## escape (') chars in the desc, breakin the tag. - #meta_desc = soup.find('meta',{'name':'description'}) - #metasoup = bs.BeautifulStoneSoup(meta_desc['content']) - #self.story.setMetadata('description',stripHTML(metasoup)) - def defaultGetattr(d,k): try: return d[k] diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py index 18b83b02..c9174fda 100644 --- a/fanficdownloader/adapters/adapter_twiwritenet.py +++ b/fanficdownloader/adapters/adapter_twiwritenet.py @@ -48,7 +48,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter): def needToLoginCheck(self, data): if 'Registered Users Only' in data \ or 'There is no such account on our website' in data \ - or "That password doesn't match the one in our database." in data: + or "That password doesn't match the one in our database" in data: return True else: return False diff --git a/index.html b/index.html index e7574a52..0376cfa8 100644 --- a/index.html +++ b/index.html @@ -56,10 +56,10 @@ This version is a new re-org/re-write of the code.

- So far, the only sites supported are: fanfiction.net, fictionalley.com, ficwad.com, twilighted.net and whofic.com. + fictionalley.org and mediaminer.org aren't support yet.

- Login/Password is asked for when required now. + Login/Password is asked for when required now, as is 'Are you an Adult?' where required.

Mobi support (for Kindle) is only via EPub conversion in this version. @@ -164,6 +164,11 @@ Use the URL of the story's chapter list, such as
http://www.whofic.com/viewstory.php?sid=16334. +

thewriterscoffeeshop.com
+
+ Use the URL of the story's chapter list, such as +
http://www.thewriterscoffeeshop.com/library/viewstory.php?sid=2110. +
diff --git a/login.html b/login.html index 2d1240f8..a36d9193 100644 --- a/login.html +++ b/login.html @@ -49,7 +49,7 @@
- {% if login %} + {% if is_login %}

Login and Password

@@ -69,6 +69,8 @@ {% else %} + +
Are you an Adult?
diff --git a/main.py b/main.py index f34a8256..c93c9afb 100644 --- a/main.py +++ b/main.py @@ -316,14 +316,19 @@ class FanfictionDownloader(UserConfigServer): download.failure = str(e) download.put() logging.debug('Need to Login, display log in page') - login= ( isinstance(e, exceptions.FailedToLogin) ) + is_login= ( isinstance(e, exceptions.FailedToLogin) ) template_values = dict(nickname = user.nickname(), url = url, format = format, site = adapter.getSiteDomain(), fic = download, - login=login, + is_login=is_login, ) + # thewriterscoffeeshop.com can do adult check *and* user required. + if isinstance(e,exceptions.AdultCheckRequired): + template_values['login']=login + template_values['password']=password + path = os.path.join(os.path.dirname(__file__), 'login.html') self.response.out.write(template.render(path, template_values)) return