mirror of
https://github.com/JimmXinu/FanFicFare.git
synced 2026-05-04 18:40:51 +02:00
231 lines
9.4 KiB
Python
231 lines
9.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2011 Fanficdownloader team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
import time
|
|
import logging
|
|
import re
|
|
import urllib
|
|
import urllib2
|
|
|
|
from .. import BeautifulSoup as bs
|
|
from ..htmlcleanup import stripHTML
|
|
from .. import exceptions as exceptions
|
|
|
|
from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
|
|
|
|
class TwiwriteNetSiteAdapter(BaseSiteAdapter):
|
|
|
|
def __init__(self, config, url):
|
|
BaseSiteAdapter.__init__(self, config, url)
|
|
self.story.setMetadata('siteabbrev','twrt')
|
|
self.decode = ["Windows-1252",
|
|
"utf8"] # 1252 is a superset of iso-8859-1.
|
|
# Most sites that claim to be
|
|
# iso-8859-1 (and some that claim to be
|
|
# utf8) are really windows-1252.
|
|
self.story.addToList("category","Twilight")
|
|
self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
|
|
self.password = ""
|
|
|
|
# get storyId from url--url validation guarantees query is only sid=1234
|
|
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
|
logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
|
|
|
# normalized story URL.
|
|
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
|
|
|
|
|
@staticmethod
|
|
def getSiteDomain():
|
|
return 'www.twiwrite.net'
|
|
|
|
@classmethod
|
|
def getAcceptDomains(cls):
|
|
return ['www.twiwrite.net','twiwrite.net']
|
|
|
|
def getSiteExampleURLs(self):
|
|
return "http://www.twiwrite.net/viewstory.php?sid=1234 http://twiwrite.net/viewstory.php?sid=5678"
|
|
|
|
def getSiteURLPattern(self):
|
|
return re.escape("http://")+r"(www\.)?"+re.escape("twiwrite.net/viewstory.php?sid=")+r"\d+$"
|
|
|
|
def needToLoginCheck(self, data):
|
|
if 'Registered Users Only' in data \
|
|
or 'There is no such account on our website' in data \
|
|
or "That password doesn't match the one in our database" in data:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def performLogin(self, url):
|
|
params = {}
|
|
|
|
if self.password:
|
|
params['penname'] = self.username
|
|
params['password'] = self.password
|
|
else:
|
|
params['penname'] = self.getConfig("username")
|
|
params['password'] = self.getConfig("password")
|
|
params['cookiecheck'] = '1'
|
|
params['submit'] = 'Submit'
|
|
|
|
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
|
logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
|
|
params['penname']))
|
|
|
|
d = self._fetchUrl(loginUrl, params)
|
|
|
|
if "Member Account" not in d : #Member Account
|
|
logging.info("Failed to login to URL %s as %s" % (loginUrl,
|
|
params['penname']))
|
|
raise exceptions.FailedToLogin(url,params['penname'])
|
|
return False
|
|
else:
|
|
return True
|
|
|
|
def extractChapterUrlsAndMetadata(self):
|
|
|
|
url = self.url+'&index=1'
|
|
logging.debug("URL: "+url)
|
|
|
|
try:
|
|
data = self._fetchUrl(url)
|
|
except urllib2.HTTPError, e:
|
|
if e.code == 404:
|
|
raise exceptions.StoryDoesNotExist(self.url)
|
|
else:
|
|
raise e
|
|
|
|
if self.needToLoginCheck(data):
|
|
# need to log in for this one.
|
|
self.performLogin(url)
|
|
data = self._fetchUrl(url)
|
|
|
|
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
|
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
|
|
|
# use BeautifulSoup HTML parser to make everything easier to find.
|
|
data = data[data.index("<body"):] # desperate--strip before <body
|
|
# in calibre plugin only, soup wasn't parsing the html properly.
|
|
soup = bs.BeautifulSoup(data)
|
|
|
|
## Title
|
|
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
|
self.story.setMetadata('title',a.string)
|
|
|
|
# Find authorid and URL from... author url.
|
|
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
|
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
|
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
|
self.story.setMetadata('author',a.string)
|
|
|
|
# Find the chapters:
|
|
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
|
|
# just in case there's tags, like <i> in chapter titles.
|
|
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
|
|
|
|
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
|
|
|
## <meta name='description' content='<p>Description</p> ...' >
|
|
## Summary, strangely, is in the content attr of a <meta name='description'> tag
|
|
## which is escaped HTML. Unfortunately, we can't use it because they don't
|
|
## escape (') chars in the desc, breakin the tag.
|
|
#meta_desc = soup.find('meta',{'name':'description'})
|
|
#metasoup = bs.BeautifulStoneSoup(meta_desc['content'])
|
|
#self.story.setMetadata('description',stripHTML(metasoup))
|
|
|
|
def defaultGetattr(d,k):
|
|
try:
|
|
return d[k]
|
|
except:
|
|
return ""
|
|
|
|
# <span class="label">Rated:</span> NC-17<br /> etc
|
|
labels = soup.findAll('span',{'class':'label'})
|
|
for labelspan in labels:
|
|
value = labelspan.nextSibling
|
|
label = labelspan.string
|
|
|
|
if 'Summary' in label:
|
|
## Everything until the next span class='label'
|
|
svalue = ""
|
|
while not defaultGetattr(value,'class') == 'label':
|
|
svalue += str(value)
|
|
value = value.nextSibling
|
|
self.story.setMetadata('description',stripHTML(svalue))
|
|
|
|
if 'Rated' in label:
|
|
self.story.setMetadata('rating', value)
|
|
|
|
if 'Word count' in label:
|
|
self.story.setMetadata('numWords', value)
|
|
|
|
if 'Categories' in label:
|
|
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
|
catstext = [cat.string for cat in cats]
|
|
for cat in catstext:
|
|
self.story.addToList('category',cat.string)
|
|
|
|
if 'Genre' in label:
|
|
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
|
|
genrestext = [genre.string for genre in genres]
|
|
self.genre = ', '.join(genrestext)
|
|
for genre in genrestext:
|
|
self.story.addToList('genre',genre.string)
|
|
|
|
if 'Warnings' in label:
|
|
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=8'))
|
|
warningstext = [warning.string for warning in warnings]
|
|
self.warning = ', '.join(warningstext)
|
|
for warning in warningstext:
|
|
self.story.addToList('warning',warning.string)
|
|
|
|
if 'Completed' in label:
|
|
if 'Yes' in value:
|
|
self.story.setMetadata('status', 'Completed')
|
|
else:
|
|
self.story.setMetadata('status', 'In-Progress')
|
|
|
|
if 'Published' in label:
|
|
self.story.setMetadata('datePublished', makeDate(value.strip(), "%B %d, %Y"))
|
|
|
|
if 'Updated' in label:
|
|
# there's a stray [ at the end.
|
|
value = value[0:-1]
|
|
self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
|
|
|
|
|
|
def getChapterText(self, url):
|
|
|
|
logging.debug('Getting chapter text from: %s' % url)
|
|
|
|
data = self._fetchUrl(url)
|
|
data = data[data.index("<body"):] # desperate--strip before <body
|
|
# in calibre plugin only, soup wasn't parsing the html properly.
|
|
soup = bs.BeautifulStoneSoup(data,
|
|
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
|
|
|
|
span = soup.find('div', {'id' : 'story'})
|
|
|
|
if None == span:
|
|
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
|
|
|
return utf8FromSoup(span)
|
|
|
|
def getClass():
|
|
return TwiwriteNetSiteAdapter
|
|
|