diff --git a/defaults.ini b/defaults.ini
index 8c26d9d7..30d4bc0c 100644
--- a/defaults.ini
+++ b/defaults.ini
@@ -58,6 +58,9 @@ authorId_label:Author ID
## Primarily to put specific values in dc:subject tags for epub. Will
## show up in Calibre as tags. Also carried into mobi when converted.
extratags_label:Extra Tags
+## The version of fanficdownloader
+##
+version_label:FFD Version
## items to include in the title page
## Empty entries will *not* appear, even if in the list.
diff --git a/newdownload.py b/downloader.py
similarity index 100%
rename from newdownload.py
rename to downloader.py
diff --git a/fanficdownloader/epubmerge.py b/epubmerge.py
similarity index 100%
rename from fanficdownloader/epubmerge.py
rename to epubmerge.py
diff --git a/fanficdownloader/adapter.py b/fanficdownloader/adapter.py
deleted file mode 100644
index 48e8facd..00000000
--- a/fanficdownloader/adapter.py
+++ /dev/null
@@ -1,231 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import logging
-import datetime
-from output import makeAcceptableFilename
-
-try:
- from google.appengine.api.urlfetch import fetch as googlefetch
- appEngineGlob = True
-except:
- appEngineGlob = False
-
-class LoginRequiredException(Exception):
- def __init__(self, url):
- self.url = url
-
- def __str__(self):
- return repr(self.url + ' requires user to be logged in')
-
-class StoryArchivedAlready(Exception):
- pass
-
-class StoryDoesNotExist(Exception):
- pass
-
-class FailedToDownload(Exception):
- pass
-
-class InvalidStoryURL(Exception):
- pass
-
-class FanfictionSiteAdapter:
- appEngine = appEngineGlob
- login = ''
- password = ''
- url = ''
- host = ''
- path = ''
- uuid = ''
- storyName = ''
- storyId = ''
- authorName = ''
- authorId = ''
- authorURL = ''
- outputStorySep = '-Ukn_'
- outputName = ''
- outputFileName = ''
- storyDescription = ''
- storyCharacters = []
- storySeries = ''
- storyPublished = datetime.date(1970, 01, 31)
- storyCreated = datetime.datetime.now()
- storyUpdated = datetime.date(1970, 01, 31)
- languageId = 'en-UK'
- language = 'English'
- subjects = []
- publisher = ''
- numChapters = '0'
- numWords = '0'
- genre = ''
- category = ''
- storyStatus = 'In-Progress'
- storyRating = ''
- storyUserRating = '0'
- def __init__(self, url):
- # basic plain url parsing...
- self.url = url
- parsedUrl = up.urlparse(url)
- self.host = parsedUrl.netloc
- self.path = parsedUrl.path
-
- def hasAppEngine(self):
- return self.appEngine
-
- def fetchUrl(self, url):
- if not self.appEngine:
- return self.opener.open(url).read().decode('utf-8')
- else:
- return googlefetch(url,deadline=10).content
-
- def requiresLogin(self, url = None):
- return False
-
- def performLogin(self, url = None):
- return True
-
- def extractIndividualUrls(self):
- pass
-
- def getText(self, url):
- pass
-
- def setLogin(self, login):
- self.login = login
-
- def setPassword(self, password):
- self.password = password
-
- def getHost(self):
- logging.debug('self.host=%s' % self.host)
- return self.host
-
- def getUUID(self):
- self.uuid = 'urn:uuid:' + self.host + '-u.' + self.authorId + '-s.' + self.storyId
- logging.debug('self.uuid=%s' % self.uuid)
- return self.uuid
-
- def getOutputName(self):
- self.outputName = makeAcceptableFilename(self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId)
- logging.debug('self.outputName=%s' % self.outputName)
- return self.outputName
-
- def getOutputFileName(self, booksDirectory, bookExt):
- self.getOutputName() # make sure self.outputName is populated
- self.outputFileName = booksDirectory + "/" + self.outputName + bookExt
- logging.debug('self.outputFileName=%s' % self.outputFileName)
- return self.outputFileName
-
- def getAuthorURL(self):
- logging.debug('self.authorURL=%s' % self.authorURL)
- return self.authorURL
-
- def getAuthorId(self):
- logging.debug('self.authorId=%s' % self.authorId)
- return self.authorId
-
- def getAuthorName(self):
- logging.debug('self.authorName=%s' % self.authorName)
- return self.authorName
-
- def getStoryURL(self):
- logging.debug('self.url=%s' % self.url)
- return self.url
-
- def getStoryId(self):
- logging.debug('self.storyId=%s' % self.storyId)
- return self.storyId
-
- def getStoryName(self):
- logging.debug('self.storyName=%s' % self.storyName)
- return self.storyName
-
- def getStoryDescription(self):
- ## with out stripping \n's, appengine treats additional lines from this debug
- ## output as error messages.
- #logging.debug('self.storyDescription=%s' % self.storyDescription.replace("\n"," ").replace('\r',''))
- return self.storyDescription
-
- def getStoryCreated(self):
- self.storyCreated = datetime.datetime.now()
- logging.debug('self.storyCreated=%s' % self.storyCreated)
- return self.storyCreated
-
- def addCharacter(self, character):
- chara = character.upper()
- for c in self.storyCharacters:
- if c.upper() == chara:
- return False
- self.storyCharacters.append(character)
- return True
-
- def getStoryCharacters(self):
- logging.debug('self.storyCharacters=%s' % self.storyCharacters)
- return self.storyCharacters
-
- def getStoryPublished(self):
- logging.debug('self.storyPublished=%s' % self.storyPublished)
- return self.storyPublished
-
- def getStoryUpdated(self):
- logging.debug('self.storyUpdated=%s' % self.storyUpdated)
- return self.storyUpdated
-
- def getStorySeries(self):
- logging.debug('self.storySeries=%s' % self.storySeries)
- return self.storySeries
-
- def getLanguage(self):
- logging.debug('self.language=%s' % self.language)
- return self.language
-
- def getLanguageId(self):
- logging.debug('self.languageId=%s' % self.languageId)
- return self.languageId
-
- def addSubject(self, subject):
- subj = subject.upper()
- for s in self.subjects:
- if s.upper() == subj:
- return False
- self.subjects.append(subject)
- return True
-
- def getSubjects(self):
- logging.debug('self.subjects=%s' % self.subjects)
- return self.subjects
-
- def getPublisher(self):
- logging.debug('self.publisher=%s' % self.publisher)
- return self.publisher
-
- def getNumChapters(self):
- logging.debug('self.numChapters=%s' % self.numChapters)
- return self.numChapters
-
- def getNumWords(self):
- logging.debug('self.numWords=%s' % self.numWords)
- return self.numWords
-
- def getCategory(self):
- logging.debug('self.category=%s' % self.category)
- return self.category
-
- def getGenre(self):
- logging.debug('self.genre=%s' % self.genre)
- return self.genre
-
- def getStoryStatus(self):
- logging.debug('self.storyStatus=%s' % self.storyStatus)
- return self.storyStatus
-
- def getStoryRating(self):
- logging.debug('self.storyRating=%s' % self.storyRating)
- return self.storyRating
-
- def getStoryUserRating(self):
- logging.debug('self.storyUserRating=%s' % self.storyUserRating)
- return self.storyUserRating
-
- def getPrintableUrl(self, url):
- return url
diff --git a/fanficdownloader/adapters/adapter_adastrafanficcom.py b/fanficdownloader/adapters/adapter_adastrafanficcom.py
index 5bdc1c9b..11f07739 100644
--- a/fanficdownloader/adapters/adapter_adastrafanficcom.py
+++ b/fanficdownloader/adapters/adapter_adastrafanficcom.py
@@ -32,7 +32,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
def __init__(self, config, url):
BaseSiteAdapter.__init__(self, config, url)
self.story.setMetadata('siteabbrev','aaff')
- self.decode = "ISO-8859-1"
+ self.decode = "Windows-1252" # 1252 is a superset of
+ # iso-8859-1. Most sites that
+ # claim to be iso-8859-1 (and
+ # some that claim to be utf8) are
+ # really windows-1252.
self.story.addToList("category","Star Trek")
self.is_adult=False
diff --git a/fanficdownloader/adapters/adapter_fanfictionnet.py b/fanficdownloader/adapters/adapter_fanfictionnet.py
index 39afe38b..8cd08650 100644
--- a/fanficdownloader/adapters/adapter_fanfictionnet.py
+++ b/fanficdownloader/adapters/adapter_fanfictionnet.py
@@ -63,13 +63,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
# use BeautifulSoup HTML parser to make everything easier to find.
try:
- soup = bs.BeautifulSoup(self._fetchUrl(url))
+ data = self._fetchUrl(url)
+ soup = bs.BeautifulSoup(data)
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
raise e
+ if "Unable to locate story with id of " in data:
+ raise exceptions.StoryDoesNotExist(self.url)
+
# Find authorid and URL from... author url.
a = soup.find('a', href=re.compile(r"^/u/\d+"))
self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -186,12 +190,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
- span = soup.find('div', {'id' : 'storytext'})
+ div = soup.find('div', {'id' : 'storytext'})
- if None == span:
+ if None == div:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
- return utf8FromSoup(span)
+ return utf8FromSoup(div)
def getClass():
return FanFictionNetSiteAdapter
diff --git a/fanficdownloader/adapters/adapter_fictionalleyorg.py b/fanficdownloader/adapters/adapter_fictionalleyorg.py
index 594de50e..aa175312 100644
--- a/fanficdownloader/adapters/adapter_fictionalleyorg.py
+++ b/fanficdownloader/adapters/adapter_fictionalleyorg.py
@@ -32,7 +32,11 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
def __init__(self, config, url):
BaseSiteAdapter.__init__(self, config, url)
self.story.setMetadata('siteabbrev','fa')
- self.decode = "ISO-8859-1" ## fa *lies*. It claims to be UTF8 in the headers, but it isn't.
+ self.decode = "Windows-1252" # 1252 is a superset of
+ # iso-8859-1. Most sites that
+ # claim to be iso-8859-1 (and
+ # some that claim to be utf8) are
+ # really windows-1252.
self.story.addToList("category","Harry Potter")
self.is_adult=False
@@ -127,7 +131,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
self.story.setMetadata('author',authora.string)
self.story.setMetadata('authorUrl',authora['href'])
- print chapterlinklist
if len(chapterlinklist) == 1:
self.chapterUrls.append((self.story.getMetadata('title'),chapterlinklist[0]['href']))
else:
diff --git a/fanficdownloader/adapters/adapter_fictionpresscom.py b/fanficdownloader/adapters/adapter_fictionpresscom.py
index bc9c2905..79022960 100644
--- a/fanficdownloader/adapters/adapter_fictionpresscom.py
+++ b/fanficdownloader/adapters/adapter_fictionpresscom.py
@@ -51,34 +51,88 @@ class FictionPressComSiteAdapter(BaseSiteAdapter):
def extractChapterUrlsAndMetadata(self):
- # fetch the chapter. From that we will get metadata and chapter list
- # You'd think this would be very similar to ffnet. But you'd be wrong.
+ # fetch the chapter. From that we will get almost all the
+ # metadata and chapter list
url = self.url
logging.debug("URL: "+url)
- logging.debug('Getting metadata from: %s' % url)
# use BeautifulSoup HTML parser to make everything easier to find.
try:
- soup = bs.BeautifulSoup(self._fetchUrl(url))
+ data = self._fetchUrl(url)
+ soup = bs.BeautifulSoup(data)
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
raise e
+ if "Unable to locate story with id of " in data:
+ raise exceptions.StoryDoesNotExist(self.url)
+
# Find authorid and URL from... author url.
a = soup.find('a', href=re.compile(r"^/u/\d+"))
self.story.setMetadata('authorId',a['href'].split('/')[2])
self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
self.story.setMetadata('author',a.string)
- # Starr and Temple Detective Agency, a Sci-Fi fanfic - FictionPress.com
- title = soup.find('title')
- m = re.match(r"^(.*?), a (.*?) fanfic - FictionPress.com",title.string)
- title,category = m.groups()
- self.story.setMetadata('title', title)
- self.story.addToList('category',category)
+
+ # start by finding a script towards the bottom that has a
+ # bunch of useful stuff in it.
+
+ # var storyid = 6577076;
+ # var chapter = 1;
+ # var chapters = 17;
+ # var words = 42787;
+ # var userid = 2645830;
+ # var title = 'The+Invitation';
+ # var title_t = 'The Invitation';
+ # var summary = 'Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter\'s wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story.';
+ # var categoryid = 224;
+ # var cat_title = 'Harry Potter';
+ # var datep = '12-21-10';
+ # var dateu = '04-06-11';
+ # var author = 'U n F a b u l o u s M e';
+
+ for script in soup.findAll('script', src=None):
+ if not script:
+ continue
+ if not script.string:
+ continue
+ if 'var storyid' in script.string:
+ for line in script.string.split('\n'):
+ m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;$",line)
+ if m == None : continue
+ var,value = m.groups()
+ # remove javascript escaping from values.
+ value = re.sub(r'\\(.)',r'\1',value)
+ #print var,value
+ if 'words' in var:
+ self.story.setMetadata('numWords', value)
+ if 'title_t' in var:
+ self.story.setMetadata('title', value)
+ if 'summary' in var:
+ self.story.setMetadata('description', value)
+ if 'datep' in var:
+ self.story.setMetadata('datePublished',makeDate(value, '%m-%d-%y'))
+ if 'dateu' in var:
+ self.story.setMetadata('dateUpdated',makeDate(value, '%m-%d-%y'))
+ if 'cat_title' in var:
+ if "Crossover" in value:
+ value = re.sub(r' Crossover$','',value)
+ for c in value.split(' and '):
+ self.story.addToList('category',c)
+ # Screws up when the category itself
+ # contains ' and '. But that's rare
+ # and the only alternative is to find
+ # the 'Crossover' category URL and
+ # parse that page to search for
+ # with href /crossovers/(name)/(num)/
+ # Harry Potter
+ # Naruto
+ else:
+ self.story.addToList('category',value)
+ break # for script in soup.findAll('script', src=None):
# Find the chapter selector
select = soup.find('select', { 'name' : 'chapter' } )
@@ -99,58 +153,31 @@ class FictionPressComSiteAdapter(BaseSiteAdapter):
self.story.setMetadata('numChapters',len(self.chapterUrls))
- ## Pull some additional data from html.
+ ## Pull some additional data from html. Find Rating and look around it.
- # Find Rating and look around it.
- a = soup.find('a', href=re.compile(r'^http://www.fictionratings.com'))
- # "Fiction Rated: K+"
- self.story.setMetadata('rating',a.string.split()[-1])
+ a = soup.find('a', href='http://www.fictionratings.com/')
+ self.story.setMetadata('rating',a.string)
# after Rating, the same bit of text containing id:123456 contains
- # Complete--if completed, and Published/Updated dates.
- # - Published: 02-07-11 - Updated: 02-07-11 - Complete - id:2889508
- dataline = a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId')))
- if dataline:
- if 'Complete' in dataline:
- self.story.setMetadata('status', 'Completed')
- else:
- self.story.setMetadata('status', 'In-Progress')
+ # Complete--if completed.
+ if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
+ self.story.setMetadata('status', 'Completed')
+ else:
+ self.story.setMetadata('status', 'In-Progress')
- m = re.match(r".*?Published: ([0-9-]+) - Updated: ([0-9-]+).*?",dataline)
- if m:
- published,updated = m.groups()
- self.story.setMetadata('datePublished',makeDate(published, '%m-%d-%y'))
- self.story.setMetadata('dateUpdated',makeDate(updated, '%m-%d-%y'))
-
- # category, genres, then desc.
- #
- # Parse genre(s) and description from .*?), (?P.*?), (?P.*?)$",
+ # Parse genre(s) from
+ #
+ #
+ # genre is after first -, but before first 'fanfiction'.
+ m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?) (?:- (?P.*?)) (?:crossover )?fanfiction",
soup.find('meta',{'name':'description'})['content'])
if m != None:
- self.story.setMetadata('description', m.group('desc'))
genres=m.group('genres')
# Hurt/Comfort is one genre.
genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
for g in genres.split('/'):
self.story.addToList('genre',g)
-
- # Number of words only on author page.
- # status, category, etc could also be parsed from here, but this way the one
- # off-page hit is isolated.
- logging.debug('Getting more metadata from: %s' % self.story.getMetadata('authorUrl'))
- soup = bs.BeautifulStoneSoup(self._fetchUrl(self.story.getMetadata('authorUrl')),
- selfClosingTags=('br')) # normalize tags to
- # Find the link for this story.
- a = soup.find('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId')+'/'))
- # Find the 'data line' after it.
- # Complete - Sci-Fi - Fiction Rated: T - English - Suspense/Hurt/Comfort - Chapters: 1 - Words: 2,762 - Reviews: 2 - Updated: 2-7-11 - Published: 2-7-11
- dataline = a.findNext(text=re.compile(r'Words: '))
- if dataline:
- m = re.match(r".*?Words: ([0-9,]+).*?",dataline)
- if m:
- words = m.group(1)
- self.story.setMetadata('numWords',words)
return
@@ -165,8 +192,6 @@ class FictionPressComSiteAdapter(BaseSiteAdapter):
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
div = soup.find('div', {'id' : 'storytext'})
- ## fp puts a padding style on the div that we don't want.
- del div['style']
if None == div:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py
index a24c63f4..d1fafc18 100644
--- a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py
+++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py
@@ -87,7 +87,6 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
## Title
a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
- #print "title a:%s"%a
self.story.setMetadata('title',a.string)
## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
@@ -183,7 +182,6 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
if None == div:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
- print div
return utf8FromSoup(div)
def getClass():
diff --git a/fanficdownloader/adapters/adapter_mediaminerorg.py b/fanficdownloader/adapters/adapter_mediaminerorg.py
index 273bdf78..9b7efb51 100644
--- a/fanficdownloader/adapters/adapter_mediaminerorg.py
+++ b/fanficdownloader/adapters/adapter_mediaminerorg.py
@@ -32,7 +32,11 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
def __init__(self, config, url):
BaseSiteAdapter.__init__(self, config, url)
self.story.setMetadata('siteabbrev','mm')
- self.decode = "ISO-8859-1"
+ self.decode = "Windows-1252" # 1252 is a superset of
+ # iso-8859-1. Most sites that
+ # claim to be iso-8859-1 (and
+ # some that claim to be utf8) are
+ # really windows-1252.
# get storyId from url--url validation guarantees query correct
m = re.match(self.getSiteURLPattern(),url)
@@ -80,27 +84,36 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
rating = soup.find("font",{"class":"smtxt"}).string[1:-1]
self.story.setMetadata('rating',rating)
- ## Title - Good grief. Title varies by chaptered, 1chapter and 'type=one shot'.
- ##
Atmosphere: Chapter 1 [ P - Pre-Teen ]
- ##
Hearts of Ice [ P - Pre-Teen ]
- ##
Suzaku no Princess [ P - Pre-Teen ]
- ##
The Kraut, The Bartender, and The Drunkard: Chapter 1 [ P - Pre-Teen ]
- ##
Betrayal and Justice: A Cold Heart ( Chapter 1 ) [ A - All Readers ]
- title = soup.find('td',{'class':'ffh'})
- for font in title.findAll('font'):
- font.extract() # removes 'font' tags from inside the td.
- if title.has_key('colspan') or 'src.php/t/ONE_SHOT' in data:
- titlet = title.text
- else:
- titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X', but only when no colspan and not one-shot
- self.story.setMetadata('title',titlet)
-
# Find authorid and URL from... author url.
a = soup.find('a', href=re.compile(r"/fanfic/src.php/u/\d+"))
self.story.setMetadata('authorId',a['href'].split('/')[-1])
self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
self.story.setMetadata('author',a.string)
+ ## Title - Good grief. Title varies by chaptered, 1chapter and 'type=one shot'--and even 'one-shot's can have titled chapter.
+ ## But, if colspan=2, there's no chapter title.
+ ##
Atmosphere: Chapter 1 [ P - Pre-Teen ]
+ ##
Hearts of Ice [ P - Pre-Teen ]
+ ##
Suzaku no Princess [ P - Pre-Teen ]
+ ##
The Kraut, The Bartender, and The Drunkard: Chapter 1 [ P - Pre-Teen ]
+ ##
Betrayal and Justice: A Cold Heart ( Chapter 1 ) [ A - All Readers ]
+ ##
Question and Answer: Question and Answer ( One-Shot ) [ A - All Readers ]