You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?
"%l confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui) - def _find_existing_book_id(self,db,book,matchurl=True): - mi = MetaInformation(book["title"],[book["author"]]) # author is a list. - identicalbooks = db.find_identical_books(mi) - if matchurl: # only *really* identical if URL matches, too. - for ib in identicalbooks: - if self._get_story_url(db,ib) == book['url']: - return ib - if identicalbooks: - return identicalbooks.pop() - return None + # def _find_existing_book_id(self,db,book,matchurl=True): + # mi = MetaInformation(book["title"],book["author"]) # author is a list. + # identicalbooks = db.find_identical_books(mi) + # if matchurl: # only *really* identical if URL matches, too. + # for ib in identicalbooks: + # if self._get_story_url(db,ib) == book['url']: + # return ib + # if identicalbooks: + # return identicalbooks.pop() + # return None def _make_mi_from_book(self,book): - mi = MetaInformation(book['title'],[book['author']]) # author is a list. + mi = MetaInformation(book['title'],book['author']) # author is a list. mi.set_identifiers({'url':book['url']}) mi.publisher = book['publisher'] mi.tags = book['tags'] @@ -1052,8 +1057,7 @@ make_firstimage_cover:true book['good'] = True book['calibre_id'] = None book['title'] = 'Unknown' - book['author'] = 'Unknown' - book['author_sort'] = 'Unknown' + book['author_sort'] = book['author'] = ['Unknown'] # list book['begin'] = None book['end'] = None @@ -1069,8 +1073,7 @@ make_firstimage_cover:true book['good'] = good book['calibre_id'] = idval book['title'] = 'Unknown' - book['author'] = 'Unknown' - book['author_sort'] = 'Unknown' + book['author_sort'] = book['author'] = ['Unknown'] # list book['begin'] = None book['end'] = None @@ -1086,7 +1089,7 @@ make_firstimage_cover:true book['good'] = True book['calibre_id'] = mi.id book['title'] = mi.title - book['author'] = authors_to_string(mi.authors) + book['author'] = mi.authors book['author_sort'] = mi.author_sort book['comment'] = '' book['url'] = "" diff --git a/fanficdownloader/adapters/adapter_archiveofourownorg.py b/fanficdownloader/adapters/adapter_archiveofourownorg.py index 8de8009b..1e04d2bf 100644 --- a/fanficdownloader/adapters/adapter_archiveofourownorg.py +++ b/fanficdownloader/adapters/adapter_archiveofourownorg.py @@ -155,14 +155,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): self.story.setMetadata('title',a.string) # Find authorid and URL from... author url. - a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+")) - if a == None: # ao3 allows for author 'Anonymous' with no author link. + alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+")) + if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link. self.story.setMetadata('author','Anonymous') self.story.setMetadata('authorUrl',self.url) else: - self.story.setMetadata('authorId',a['href'].split('/')[2]) - self.story.setMetadata('authorUrl','http://'+self.host+a['href']) - self.story.setMetadata('author',a.text) + for a in alist: + self.story.addToList('authorId',a['href'].split('/')[2]) + self.story.addToList('authorUrl','http://'+self.host+a['href']) + self.story.addToList('author',a.text) # Find the chapters: chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$")) diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index ee4d8763..30673902 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -103,8 +103,19 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" self.story.setMetadata('rating','Tweenie') - self.story.setMetadata('authorId','98765') - self.story.setMetadata('authorUrl','http://author/url') + + if self.story.getMetadata('storyId') == '673': + self.story.addToList('author','Author From List') + self.story.addToList('author','Author From List 2') + + self.story.addToList('authorId','98765') + self.story.addToList('authorId','98765-2') + + self.story.addToList('authorUrl','http://author/url') + self.story.addToList('authorUrl','http://author/url-2') + else: + self.story.setMetadata('authorId','98765') + self.story.setMetadata('authorUrl','http://author/url') self.story.addToList('warnings','Swearing') self.story.addToList('warnings','Violence') @@ -175,6 +186,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter
http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only
http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only
+http://test1.com?sid=673 - Succeeds, multiple authors
Odd sid's will be In-Progress, evens complete. sid<10 will be assigned one of four languages and included in a series.
''' diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py index 2864d92c..8c3047e6 100644 --- a/fanficdownloader/adapters/adapter_tthfanficorg.py +++ b/fanficdownloader/adapters/adapter_tthfanficorg.py @@ -151,12 +151,33 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1]) self.story.setMetadata('authorUrl','http://'+self.host+a['href']) self.story.setMetadata('author',stripHTML(a)) + authorurl = 'http://'+self.host+a['href'] + ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId')) + if ainfo != None: # indicates multiple authors/contributors. + try: + # going to pull part of the meta data from author list page. + infourl = 'http://'+self.host+ainfo['href'] + logging.debug("**StoryInfo** URL: "+infourl) + infodata = self._fetchUrl(infourl) + infosoup = bs.BeautifulSoup(infodata) + + for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")): + self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1]) + self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-")) + self.story.addToList('author',stripHTML(a)) + + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + try: - # going to pull part of the meta data from author list page. - logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl')) - authordata = self._fetchUrl(self.story.getMetadata('authorUrl')) - descurl=self.story.getMetadata('authorUrl') + # going to pull part of the meta data from *primary* author list page. + logging.debug("**AUTHOR** URL: "+authorurl) + authordata = self._fetchUrl(authorurl) + descurl=authorurl authorsoup = bs.BeautifulSoup(authordata) # author can have several pages, scan until we find it. while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ): @@ -209,7 +230,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): if not BtVSNonX: BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover - print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX)) + #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX)) if BtVS: self.story.addToList('category','Buffy: The Vampire Slayer') diff --git a/fanficdownloader/adapters/adapter_wraithbaitcom.py b/fanficdownloader/adapters/adapter_wraithbaitcom.py index dcd2b285..c9b7d9ef 100644 --- a/fanficdownloader/adapters/adapter_wraithbaitcom.py +++ b/fanficdownloader/adapters/adapter_wraithbaitcom.py @@ -119,10 +119,11 @@ class WraithBaitComAdapter(BaseSiteAdapter): self.story.setMetadata('title',a.string) # Find authorid and URL from... author url. - a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) - self.story.setMetadata('authorId',a['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) - self.story.setMetadata('author',a.string) + alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")) + for a in alist: + self.story.addToList('authorId',a['href'].split('=')[1]) + self.story.addToList('authorUrl','http://'+self.host+'/'+a['href']) + self.story.addToList('author',a.string) rating=pt.text.split('[')[1].split(']')[0] self.story.setMetadata('rating', rating) diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index deb51005..3be4a955 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -223,9 +223,9 @@ class Story: def getMetadata(self, key, removeallentities=False, doreplacements=True): value = None - if self.getLists().has_key(key): + if self.isList(key): value = ', '.join(self.getList(key)) - if self.metadata.has_key(key): + elif self.metadata.has_key(key): value = self.metadata[key] if value: if key == "numWords": @@ -247,11 +247,33 @@ class Story: All single value *and* list value metadata as strings. ''' allmetadata = {} + + # special handling for authors/authorUrls + authlinkhtml="%s" + if 'author' in self.listables.keys(): # more than one author, assume multiple authorUrl too. + htmllist=[] + for i, v in enumerate(self.listables['author']): + aurl = self.listables['authorUrl'][i] + auth = v + # make sure doreplacements & removeallentities are honored. + if doreplacements: + aurl=self.doReplacments(aurl) + auth=self.doReplacments(auth) + if removeallentities: + aurl=removeAllEntities(aurl) + auth=removeAllEntities(auth) + + htmllist.append(authlinkhtml%(aurl,auth)) + self.setMetadata('authorHTML',', '.join(htmllist)) + else: + self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements), + self.getMetadata('author', removeallentities, doreplacements))) + for k in self.metadata.keys(): allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements) for l in self.listables.keys(): allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements) - + return allmetadata # just for less clutter in adapters. @@ -263,23 +285,36 @@ class Story: if value==None: return value = conditionalRemoveEntities(value) - if not self.listables.has_key(listname): + if not self.isList(listname): self.listables[listname]=[] # prevent duplicates. if not value in self.listables[listname]: self.listables[listname].append(value) - def getList(self,listname): - if not self.listables.has_key(listname): - return [] - return filter( lambda x : x!=None and x!='' , - map(self.doReplacments,self.listables[listname]) ) + def getList(self,listname, removeallentities=False, doreplacements=True): + retlist = [] + if not self.isList(listname): + retlist = [self.getMetadata(listname,removeallentities=removeallentities)] + else: + retlist = self.listables[listname] + + if doreplacements: + retlist = filter( lambda x : x!=None and x!='' , + map(self.doReplacments,retlist) ) + if removeallentities: + retlist = filter( lambda x : x!=None and x!='' , + map(removeAllEntities,retlist) ) + + return retlist def getLists(self): lsts = {} for ln in self.listables.keys(): lsts[ln] = self.getList(ln) return lsts + + def isList(self,listname): + return self.listables.has_key(listname) def addChapter(self, title, html): self.chapters.append( (title,html) ) diff --git a/fanficdownloader/writers/base_writer.py b/fanficdownloader/writers/base_writer.py index 7f23084b..424aa933 100644 --- a/fanficdownloader/writers/base_writer.py +++ b/fanficdownloader/writers/base_writer.py @@ -134,7 +134,7 @@ class BaseStoryWriter(Configurable): names as Story.metadata, but ENTRY should use label and value. """ if self.getConfig("include_titlepage"): - self._write(out,START.substitute(self.story.metadata)) + self._write(out,START.substitute(self.story.getAllMetadata())) if WIDE_ENTRY==None: WIDE_ENTRY=ENTRY @@ -166,7 +166,7 @@ class BaseStoryWriter(Configurable): else: self._write(out, entry) - self._write(out,END.substitute(self.story.metadata)) + self._write(out,END.substitute(self.story.getAllMetadata())) def writeTOCPage(self, out, START, ENTRY, END): """ @@ -176,13 +176,13 @@ class BaseStoryWriter(Configurable): """ # Only do TOC if there's more than one chapter and it's configured. if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly : - self._write(out,START.substitute(self.story.metadata)) + self._write(out,START.substitute(self.story.getAllMetadata())) for index, (title,html) in enumerate(self.story.getChapters()): if html: self._write(out,ENTRY.substitute({'chapter':title, 'index':"%04d"%(index+1)})) - self._write(out,END.substitute(self.story.metadata)) + self._write(out,END.substitute(self.story.getAllMetadata())) # if no outstream is given, write to file. def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False): diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py index da734e9b..40a28feb 100644 --- a/fanficdownloader/writers/writer_epub.py +++ b/fanficdownloader/writers/writer_epub.py @@ -52,7 +52,7 @@ class EpubWriter(BaseStoryWriter): -