From f31fcc33410e6ba3ae546b21389a2d98c6611767 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 5 Sep 2012 12:20:35 -0500 Subject: [PATCH] Add extracategories, etc, Relationships, another Mac fix, fix for & in tags. --- calibre-plugin/config.py | 2 + calibre-plugin/ffdl_plugin.py | 43 ++-- calibre-plugin/jobs.py | 3 + defaults.ini | 184 +++++++++++++++++- .../adapters/adapter_adastrafanficcom.py | 1 - .../adapters/adapter_archiveofourownorg.py | 6 +- .../adapter_ashwindersycophanthexcom.py | 5 - .../adapters/adapter_castlefansorg.py | 5 - .../adapters/adapter_chaossycophanthexcom.py | 5 - .../adapters/adapter_checkmatedcom.py | 55 +++--- .../adapters/adapter_darksolaceorg.py | 5 - .../adapters/adapter_dokugacom.py | 5 - .../adapters/adapter_dracoandginnycom.py | 7 - .../adapters/adapter_dramioneorg.py | 5 - .../adapter_erosnsapphosycophanthexcom.py | 5 - .../adapters/adapter_fictionalleyorg.py | 1 - .../adapters/adapter_fimfictionnet.py | 3 - .../adapters/adapter_grangerenchantedcom.py | 5 - .../adapter_harrypotterfanfictioncom.py | 1 - .../adapters/adapter_hlfictionnet.py | 5 - .../adapters/adapter_hpfandomnet.py | 5 - .../adapters/adapter_hpfanficarchivecom.py | 5 - .../adapters/adapter_iketernalnet.py | 5 - .../adapters/adapter_ksarchivecom.py | 5 - .../adapters/adapter_libraryofmoriacom.py | 5 - .../adapters/adapter_lumossycophanthexcom.py | 5 - .../adapters/adapter_midnightwhispersca.py | 5 - .../adapters/adapter_mugglenetcom.py | 5 - .../adapters/adapter_nationallibrarynet.py | 9 +- .../adapters/adapter_ncisficcom.py | 9 +- .../adapters/adapter_ncisfictioncom.py | 10 +- .../adapters/adapter_nfacommunitycom.py | 5 - .../adapters/adapter_nhamagicalworldsus.py | 5 - .../adapter_occlumencysycophanthexcom.py | 5 - .../adapter_onedirectionfanfictioncom.py | 5 - .../adapters/adapter_phoenixsongnet.py | 5 - .../adapters/adapter_ponyfictionarchivenet.py | 5 - .../adapters/adapter_portkeyorg.py | 5 - .../adapters/adapter_potionsandsnitchesnet.py | 3 +- .../adapters/adapter_pretendercentrecom.py | 5 - .../adapters/adapter_prisonbreakficnet.py | 10 +- .../adapters/adapter_samdeanarchivenu.py | 5 - .../adapters/adapter_scarvesandcoffeenet.py | 7 - .../adapters/adapter_sg1heliopoliscom.py | 2 +- .../adapters/adapter_sinfuldesireorg.py | 5 - fanficdownloader/adapters/adapter_siyecouk.py | 5 - .../adapters/adapter_stargateatlantisorg.py | 5 - .../adapters/adapter_storiesofardacom.py | 5 - .../adapters/adapter_thealphagatecom.py | 5 - .../adapters/adapter_thehexfilesnet.py | 5 - .../adapters/adapter_thequidditchpitchorg.py | 5 - .../adapters/adapter_twilightarchivescom.py | 5 - .../adapters/adapter_twilightednet.py | 1 - .../adapters/adapter_twiwritenet.py | 1 - .../adapters/adapter_walkingtheplankorg.py | 5 - .../adapters/adapter_wolverineandroguecom.py | 7 - .../adapters/adapter_wraithbaitcom.py | 5 - fanficdownloader/story.py | 6 +- fanficdownloader/writers/base_writer.py | 25 ++- plugin-defaults.ini | 184 +++++++++++++++++- plugin-example.ini | 17 +- 61 files changed, 457 insertions(+), 320 deletions(-) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index fbaa6de5..63749743 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -715,6 +715,7 @@ permitted_values = { 'language', 'series', 'characters', + 'ships', 'status', 'datePublished', 'dateUpdated', @@ -751,6 +752,7 @@ titleLabels = { 'status-I':'Status:In-Progress', 'series':'Series', 'characters':'Characters', + 'ships':'Relationships', 'datePublished':'Published', 'dateUpdated':'Updated', 'dateCreated':'Packaged', diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index 5ae342a1..25ec2dd3 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Jim Miller' __docformat__ = 'restructuredtext en' -import time, os, copy, threading, re +import time, os, copy, threading, re, platform from ConfigParser import SafeConfigParser from StringIO import StringIO from functools import partial @@ -175,18 +175,21 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png', triggered=self.get_urls_from_page) - self.menu.addSeparator() - self.config_action = self.create_menu_item_ex(self.menu, '&Configure Plugin', #shortcut=False, # causes crashes on some Macs. - image= 'config.png', - unique_name='Configure FanFictionDownLoader', - shortcut_name='Configure FanFictionDownLoader', - triggered=partial(do_user_config,parent=self.gui)) + # print("platform.system():%s"%platform.system()) + # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0]) + if not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons. + self.menu.addSeparator() + self.config_action = self.create_menu_item_ex(self.menu, '&Configure Plugin', + image= 'config.png', + unique_name='Configure FanFictionDownLoader', + shortcut_name='Configure FanFictionDownLoader', + triggered=partial(do_user_config,parent=self.gui)) - self.about_action = self.create_menu_item_ex(self.menu, 'About Plugin', #shortcut=False, # causes crashes on some Macs. - image= 'images/icon.png', - unique_name='About FanFictionDownLoader', - shortcut_name='About FanFictionDownLoader', - triggered=self.about) + self.about_action = self.create_menu_item_ex(self.menu, 'About Plugin', + image= 'images/icon.png', + unique_name='About FanFictionDownLoader', + shortcut_name='About FanFictionDownLoader', + triggered=self.about) # Before we finalize, make sure we delete any actions for menus that are no longer displayed for menu_id, unique_name in self.old_actions_unique_map.iteritems(): @@ -472,9 +475,9 @@ make_firstimage_cover:true book['title'] = story.getMetadata("title", removeallentities=True) book['author_sort'] = book['author'] = story.getList("author", removeallentities=True) book['publisher'] = story.getMetadata("site") - book['tags'] = writer.getTags() # getTags could be moved up into adapter now. Adapter didn't used to know the fileform + book['tags'] = writer.getTags(removeallentities=True) # getTags could be moved up into adapter now. Adapter didn't used to know the fileform book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better. - book['series'] = story.getMetadata("series") + book['series'] = story.getMetadata("series", removeallentities=True) # adapter.opener is the element with a threadlock. But del # adapter.opener doesn't work--subproc fails when it tries @@ -896,18 +899,6 @@ make_firstimage_cover:true db.set_link_field_for_author(autid, unicode(authurls[i]), commit=False, notify=False) - # mi.title = oldmi.title - # mi.authors = oldmi.authors - # mi.publisher = oldmi.publisher - # mi.tags = oldmi.tags - # mi.languages = oldmi.languages - # mi.pubdate = oldmi.pubdate - # mi.timestamp = oldmi.timestamp - # mi.comments = oldmi.comments - # mi.series = oldmi.series - - # mi.set_identifiers(oldmi.get_identifiers()) - # implement 'newonly' flags here by setting to the current # value again. if not book['added']: diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index 6606a87a..83d5187e 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -109,6 +109,9 @@ def do_download_for_worker(book,options): when run as a worker job ''' try: + # import logging + # logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s") + book['comment'] = 'Download started...' ffdlconfig = SafeConfigParser() diff --git a/defaults.ini b/defaults.ini index 035de4fd..e73bf2cd 100644 --- a/defaults.ini +++ b/defaults.ini @@ -38,6 +38,7 @@ category_label:Category genre_label:Genre language_label:Language characters_label:Characters +ships_label:Relationships series_label:Series ## Completed/In-Progress status_label:Status @@ -70,7 +71,7 @@ version_label:FFDL Version ## You can include extra text or HTML that will be included as-is in ## the title page. Eg: titlepage_entries: ...,
,summary,
,... ## All current formats already include title and author. -titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description +titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description ## Try to collect series name and number of this story in series. ## Some sites (ab)use 'series' for reading lists and personal @@ -121,11 +122,20 @@ allow_unsafe_filename: false ## entries to make epub subjects and calibre tags ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d" -include_subject_tags: extratags, genre, category, characters, lastupdate, status +include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status ## extra tags (comma separated) to include, primarily for epub. extratags: FanFiction +## extra categories, genres, characters, ships and warnings can be +## configured. Used primarily for sites that are dedicated to a genre +## or 'ship and so don't included it for every story. +#extracategories: +#extragenres: +#extracharacters: +#extraships: +#extrawarnings: + ## number of seconds to sleep between calls to the story site. May by ## useful if pulling large numbers of stories or if the site is slow. #slow_down_sleep_time:0.5 @@ -318,6 +328,11 @@ nook_img_fix:true ## URLs like: http://test1.com?sid=12345 [test1.com] extratags: FanFiction,Testing +extracategories:Fafner +extragenres:Romance,Fluff +extracharacters:Reginald Smythe-Smythe,Mokona +extraships:Smythe-Smythe/Mokona +extrawarnings:Extreme Bogosity ## If necessary, you can define [:] sections to ## customize the formats differently for the same site. Overrides @@ -344,6 +359,11 @@ extratags: FanFiction,Testing,HTML #is_adult:true [ashwinder.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Severus Snape,Hermione Granger +extraships:Severus Snape/Hermione Granger + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -352,6 +372,9 @@ extratags: FanFiction,Testing,HTML #password:yourpassword [castlefans.org] +## Site dedicated to these categories/characters/ships +extracategories:Castle + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -375,6 +398,11 @@ extratags: FanFiction,Testing,HTML cover_exclusion_regexp:/images/.*?ribbon.gif [dark-solace.org] +## Site dedicated to these categories/characters/ships +extracategories:Buffy: The Vampire Slayer +extracharacters:Buffy, Spike +extraships:Spike/Buffy + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -383,6 +411,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #password:yourpassword [dramione.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Draco Malfoy,Hermione Granger +extraships:Draco Malfoy/Hermione Granger + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -396,6 +429,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [erosnsappho.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -406,6 +442,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif cover_exclusion_regexp:/images/.*?ribbon.gif [fanfiction.mugglenet.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -419,6 +458,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [fanfiction.portkey.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extraships:Harry Potter/Hermione Granger + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -441,6 +484,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #password:yourpassword [grangerenchanted.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Hermione Granger + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -454,26 +501,51 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [hlfiction.net] +## Site dedicated to these categories/characters/ships +extracategories:Highlander + +[ksarchive.com] +## Site dedicated to these categories/characters/ships +extracategories:Star Trek +extracharacters:Kirk,Spock +extraships:Kirk/Spock [lumos.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [national-library.net] +## Site dedicated to these categories/characters/ships +extracategories:West Wing [ncisfic.com] +## Site dedicated to these categories/characters/ships +extracategories:NCIS [nfacommunity.com] +## Site dedicated to these categories/characters/ships +extracategories:NCIS + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [nha.magical-worlds.us] +## Site dedicated to these categories/characters/ships +extracategories:Buffy: The Vampire Slayer +extracharacters:Willow [occlumency.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Severus Snape + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -482,6 +554,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #password:yourpassword [onedirectionfanfiction.com] +## Site dedicated to these categories/characters/ships +extracategories:One Direction + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -495,18 +570,28 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [ponyfictionarchive.net] +## Site dedicated to these categories/characters/ships +extracategories:My Little Pony: Friendship is Magic + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [pretendercentre.com] +## Site dedicated to these categories/characters/ships +extracategories:The Pretender + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [samdean.archive.nu] +## Site dedicated to these categories/characters/ships +extracategories:Supernatural +extracharacters:Sam,Dean +extraships:Sam/Dean [sg1-heliopolis.com] ## Some sites do not require a login, but do require the user to @@ -515,6 +600,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [stargate-atlantis.org] +## Site dedicated to these categories/characters/ships +extracategories:Stargate: Atlantis [svufiction.com] ## Some sites require login (or login for some rated stories) The @@ -530,6 +617,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [thehexfiles.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Draco Malfoy,Harry Potter +extraships:Harry Potter/Draco Malfoy [themasque.net] ## Some sites require login (or login for some rated stories) The @@ -545,6 +636,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [thequidditchpitch.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -558,12 +652,20 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [www.adastrafanfic.com] +## Site dedicated to these categories/characters/ships +extracategories:Star Trek + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.dracoandginny.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Draco Malfoy,Ginny Weasley +extraships:Draco Malfoy/Ginny Weasley + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -577,8 +679,13 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [www.thealphagate.com] +## Site dedicated to these categories/characters/ships +extracategories:Stargate: SG-1 [www.checkmated.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -593,6 +700,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [www.dokuga.com] +## Site dedicated to these categories/characters/ships +extracategories:InuYasha +extracharacters:Sesshoumaru,Kagome +extraships:Sesshoumaru/Kagome [www.fanfiction.net] ## fanfiction.net's 'cover' images are really just tiny thumbnails. @@ -610,6 +721,9 @@ never_make_cover: true [www.ficbook.net] [www.fictionalley.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -631,6 +745,9 @@ extratags: #password:yourpassword [www.fimfiction.net] +## Site dedicated to these categories/characters/ships +extracategories:My Little Pony: Friendship is Magic + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -642,20 +759,33 @@ extratags: #fail_on_password: false [www.harrypotterfanfiction.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.hpfandom.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.hpfanficarchive.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter [www.ik-eternal.net] +## Site dedicated to these categories/characters/ships +extracategories:InuYasha +extracharacters:InuYasha,Kagome +extraships:InuYasha/Kagome + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -669,6 +799,9 @@ extratags: #is_adult:true [www.libraryofmoria.com] +## Site dedicated to these categories/characters/ships +extracategories:Lord of the Rings + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -677,6 +810,9 @@ extratags: [www.mediaminer.org] [www.midnightwhispers.ca] +## Site dedicated to these categories/characters/ships +extracategories:Queer as Folk + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -687,8 +823,14 @@ extratags: cover_exclusion_regexp:/stories/999/images/.*?_trophy.png [www.ncisfiction.com] +## Site dedicated to these categories/characters/ships +extracategories:NCIS [www.phoenixsong.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extraships:Harry Potter/Ginny Weasley + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -697,28 +839,43 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png #password:yourpassword [www.potionsandsnitches.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter [www.prisonbreakfic.net] +## Site dedicated to these categories/characters/ships +extracategories:Prison Break [www.scarvesandcoffee.net] +## Site dedicated to these categories/characters/ships +extracategories:Glee +extracharacters:Kurt Hummel,Blaine Anderson + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.sinful-desire.org] +## Site dedicated to these categories/characters/ships +extracategories:Supernatural + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.siye.co.uk] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Harry Potter,Ginny Weasley +extraships:Harry Potter/Ginny Weasley [www.squidge.org/peja] # www.squidge.org/peja calls it Fandom category_label:Fandom # Remove numWords -- www.squidge.org/peja word counts are inaccurate -titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description +titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description [www.squidge.org/peja:txt] ## Add URLs since there aren't links. @@ -726,6 +883,8 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description [www.storiesofarda.com] +## Site dedicated to these categories/characters/ships +extracategories:Lord of the Rings [www.thepetulantpoetess.com] ## Some sites require login (or login for some rated stories) The @@ -765,8 +924,13 @@ collect_series: false #password:yourpassword [www.twilightarchives.com] +## Site dedicated to these categories/characters/ships +extracategories:Twilight [www.twilighted.net] +## Site dedicated to these categories/characters/ships +extracategories:Twilight + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -778,6 +942,9 @@ collect_series: false collect_series: false [www.twiwrite.net] +## Site dedicated to these categories/characters/ships +extracategories:Twilight + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -789,6 +956,11 @@ collect_series: false collect_series: false [www.walkingtheplank.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Severus Snape,Harry Potter +extraships:Severus Snape/Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -810,8 +982,14 @@ collect_series: false #is_adult:true [www.wolverineandrogue.com] +## Site dedicated to these categories/characters/ships +extracategories:X-Men Movie +extracharacters:Wolverine,Rogue [www.wraithbait.com] +## Site dedicated to these categories/characters/ships +extracategories:Stargate: Atlantis + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. diff --git a/fanficdownloader/adapters/adapter_adastrafanficcom.py b/fanficdownloader/adapters/adapter_adastrafanficcom.py index 353bee20..3aee4133 100644 --- a/fanficdownloader/adapters/adapter_adastrafanficcom.py +++ b/fanficdownloader/adapters/adapter_adastrafanficcom.py @@ -37,7 +37,6 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): # Most sites that claim to be # iso-8859-1 (and some that claim to be # utf8) are really windows-1252. - self.story.addToList("category","Star Trek") self.is_adult=False # get storyId from url--url validation guarantees query is only sid=1234 diff --git a/fanficdownloader/adapters/adapter_archiveofourownorg.py b/fanficdownloader/adapters/adapter_archiveofourownorg.py index 1e0e2643..25322bea 100644 --- a/fanficdownloader/adapters/adapter_archiveofourownorg.py +++ b/fanficdownloader/adapters/adapter_archiveofourownorg.py @@ -231,9 +231,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): self.story.addToList('characters',char.string) a = metasoup.find('dd',{'class':"relationship tags"}) if a != None: - chars = a.findAll('a',{'class':"tag"}) - for char in chars: - self.story.addToList('characters',char.string) + ships = a.findAll('a',{'class':"tag"}) + for ship in ships: + self.story.addToList('ships',ship.string) stats = metasoup.find('dl',{'class':'stats'}) diff --git a/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py b/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py index fd92931c..01d1c673 100644 --- a/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py @@ -56,11 +56,6 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','asph') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_castlefansorg.py b/fanficdownloader/adapters/adapter_castlefansorg.py index fe1158b5..a78f76e9 100644 --- a/fanficdownloader/adapters/adapter_castlefansorg.py +++ b/fanficdownloader/adapters/adapter_castlefansorg.py @@ -80,11 +80,6 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','cslf') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Castle") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%b %d, %Y" # XXX diff --git a/fanficdownloader/adapters/adapter_chaossycophanthexcom.py b/fanficdownloader/adapters/adapter_chaossycophanthexcom.py index b6cf945b..f9f6f50a 100644 --- a/fanficdownloader/adapters/adapter_chaossycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_chaossycophanthexcom.py @@ -56,11 +56,6 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','csph') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - #self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_checkmatedcom.py b/fanficdownloader/adapters/adapter_checkmatedcom.py index 874e3f6c..c529f0bd 100644 --- a/fanficdownloader/adapters/adapter_checkmatedcom.py +++ b/fanficdownloader/adapters/adapter_checkmatedcom.py @@ -44,25 +44,20 @@ class CheckmatedComAdapter(BaseSiteAdapter): self.username = "" # if left empty, site doesn't return any message at all. self.password = "" self.is_adult=False - + # get storyId from url--url validation guarantees query is only sid=1234 self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) - + self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId')) - + # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','chm') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%b %d, %Y" - + @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): # The site domain. Does have www here, if it uses it. @@ -82,7 +77,7 @@ class CheckmatedComAdapter(BaseSiteAdapter): return True else: return False - + def performLogin(self, url): params = {} @@ -93,8 +88,8 @@ class CheckmatedComAdapter(BaseSiteAdapter): params['name'] = self.getConfig("username") params['pass'] = self.getConfig("password") params['login'] = 'yes' - params['submit'] = 'login' - + params['submit'] = 'login' + loginUrl = 'http://' + self.getSiteDomain()+'/login.php' d = self._fetchUrl(loginUrl,params) e = self._fetchUrl(url) @@ -135,33 +130,33 @@ class CheckmatedComAdapter(BaseSiteAdapter): # The actual text that is used to announce you need to be an # adult varies from site to site. Again, print data before # the title search to troubleshoot. - + if "Access denied. This story has not been validated by the adminstrators of this site." in data: raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") - + # use BeautifulSoup HTML parser to make everything easier to find. soup = bs.BeautifulSoup(data) # print data # Now go hunting for all the meta data and the chapter list. - + ## Title a = soup.findAll('span', {'class' : 'storytitle'}) self.story.setMetadata('title',a[0].string) - + # Find authorid and URL from... author url. a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+")) self.story.setMetadata('authorId',a['href'].split('=')[1]) self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) self.story.setMetadata('author',a.string) - + a = soup.find('select', {'name' : 'chapter'}) if a == None: self.chapterUrls.append((self.story.getMetadata('title'),url)) else: for chapter in a.findAll('option'): self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])) - + self.story.setMetadata('numChapters',len(self.chapterUrls)) # eFiction sites don't help us out a lot with their meta data @@ -174,7 +169,7 @@ class CheckmatedComAdapter(BaseSiteAdapter): except: return "" - + # website does not keep track of word count, and there is no convenient way to calculate it summary = soup.find('fieldset') @@ -182,21 +177,23 @@ class CheckmatedComAdapter(BaseSiteAdapter): summary.name='div' self.setDescription(url,summary) - + # Rated: NC-17
etc table = soup.findAll('div', {'class' : 'text'})[1] for labels in table.findAll('tr'): value = labels.findAll('td')[1] label = labels.findAll('td')[0] - + if 'Rating' in stripHTML(label): self.story.setMetadata('rating', stripHTML(value)) if 'Ship' in stripHTML(label): + if value.string != "none/none": + self.story.addToList('ships',value.string) for char in value.string.split('/'): if char != 'none': - self.story.addToList('characters',char) + self.story.addToList('characters',char) if 'Status' in stripHTML(label): if value.find('img', {'src' : 'img/incomplete.gif'}) == None: @@ -206,11 +203,11 @@ class CheckmatedComAdapter(BaseSiteAdapter): if 'Published' in stripHTML(label): self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) - + if 'Updated' in stripHTML(label): self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) - - a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories') + + a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories') for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}): a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view')) if a != None: @@ -220,8 +217,8 @@ class CheckmatedComAdapter(BaseSiteAdapter): if 'genre' in stripHTML(label): for genre in value.findAll('img'): self.story.addToList('genre',genre['title']) - - + + # grab the text for an individual chapter. def getChapterText(self, url): @@ -229,11 +226,11 @@ class CheckmatedComAdapter(BaseSiteAdapter): soup = bs.BeautifulSoup(self._fetchUrl(url), selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. - + div = soup.find('div', {'id' : 'resizeableText'}) div.find('div', {'class' : 'storyTools'}).extract() if None == div: raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_darksolaceorg.py b/fanficdownloader/adapters/adapter_darksolaceorg.py index fc90c744..9ece865b 100644 --- a/fanficdownloader/adapters/adapter_darksolaceorg.py +++ b/fanficdownloader/adapters/adapter_darksolaceorg.py @@ -56,11 +56,6 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','dksl') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Buffy the Vampire Slayer") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d %B %Y" diff --git a/fanficdownloader/adapters/adapter_dokugacom.py b/fanficdownloader/adapters/adapter_dokugacom.py index 8436a574..b70b16a2 100644 --- a/fanficdownloader/adapters/adapter_dokugacom.py +++ b/fanficdownloader/adapters/adapter_dokugacom.py @@ -56,11 +56,6 @@ class DokugaComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','dkg') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","InuYasha") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior if 'fanfiction' in self.story.getMetadata('section'): diff --git a/fanficdownloader/adapters/adapter_dracoandginnycom.py b/fanficdownloader/adapters/adapter_dracoandginnycom.py index 9c186c25..6d825223 100644 --- a/fanficdownloader/adapters/adapter_dracoandginnycom.py +++ b/fanficdownloader/adapters/adapter_dracoandginnycom.py @@ -55,13 +55,6 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','dcagn') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - self.story.addToList("characters","Ginny Weasley") - self.story.addToList("characters","Draco Malfoy") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%b %d, %Y" diff --git a/fanficdownloader/adapters/adapter_dramioneorg.py b/fanficdownloader/adapters/adapter_dramioneorg.py index db299c69..e296ae0c 100644 --- a/fanficdownloader/adapters/adapter_dramioneorg.py +++ b/fanficdownloader/adapters/adapter_dramioneorg.py @@ -55,11 +55,6 @@ class DramioneOrgAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','drmn') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d %B %Y" diff --git a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py index b7613447..6ff8fff8 100644 --- a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py @@ -56,11 +56,6 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','essph') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d/%m/%y" diff --git a/fanficdownloader/adapters/adapter_fictionalleyorg.py b/fanficdownloader/adapters/adapter_fictionalleyorg.py index b374c5d6..d87a7994 100644 --- a/fanficdownloader/adapters/adapter_fictionalleyorg.py +++ b/fanficdownloader/adapters/adapter_fictionalleyorg.py @@ -37,7 +37,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter): # Most sites that claim to be # iso-8859-1 (and some that claim to be # utf8) are really windows-1252. - self.story.addToList("category","Harry Potter") self.is_adult=False # get storyId from url--url validation guarantees query correct diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index 9fbe5bde..4b61d91f 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -133,11 +133,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): if storyMetadata["categories"][category]: self.story.addToList("genre", category) - self.story.addToList("category", "My Little Pony") - self.story.setMetadata("numWords", str(storyMetadata["words"])) - # fimfic is the first site with an explicit cover image. if self.getConfig('include_images') and "image" in storyMetadata.keys(): coverurl = storyMetadata["image"] diff --git a/fanficdownloader/adapters/adapter_grangerenchantedcom.py b/fanficdownloader/adapters/adapter_grangerenchantedcom.py index e513d892..7df58bf8 100644 --- a/fanficdownloader/adapters/adapter_grangerenchantedcom.py +++ b/fanficdownloader/adapters/adapter_grangerenchantedcom.py @@ -60,11 +60,6 @@ class GrangerEnchantedCom(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','gech') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d/%b/%Y" diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py index 0bcef6ed..8c5ad0a7 100644 --- a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py @@ -37,7 +37,6 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter): # Most sites that claim to be # iso-8859-1 (and some that claim to be # utf8) are really windows-1252. - self.story.addToList("category","Harry Potter") self.is_adult=False # get storyId from url--url validation guarantees query is only psid=1234 diff --git a/fanficdownloader/adapters/adapter_hlfictionnet.py b/fanficdownloader/adapters/adapter_hlfictionnet.py index be15585b..c3203ee2 100644 --- a/fanficdownloader/adapters/adapter_hlfictionnet.py +++ b/fanficdownloader/adapters/adapter_hlfictionnet.py @@ -55,11 +55,6 @@ class HLFictionNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','hlf') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Highlander") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%y" diff --git a/fanficdownloader/adapters/adapter_hpfandomnet.py b/fanficdownloader/adapters/adapter_hpfandomnet.py index 54c530e1..dfc7c055 100644 --- a/fanficdownloader/adapters/adapter_hpfandomnet.py +++ b/fanficdownloader/adapters/adapter_hpfandomnet.py @@ -60,11 +60,6 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','hpfdm') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%Y.%m.%d" # XXX diff --git a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py index 5305f673..9e078c72 100644 --- a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py +++ b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py @@ -55,11 +55,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','hpffa') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d, %Y" diff --git a/fanficdownloader/adapters/adapter_iketernalnet.py b/fanficdownloader/adapters/adapter_iketernalnet.py index 39127875..2b673d7d 100644 --- a/fanficdownloader/adapters/adapter_iketernalnet.py +++ b/fanficdownloader/adapters/adapter_iketernalnet.py @@ -55,11 +55,6 @@ class IkEternalNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','ike') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","InuYasha") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d, %Y" diff --git a/fanficdownloader/adapters/adapter_ksarchivecom.py b/fanficdownloader/adapters/adapter_ksarchivecom.py index 5a770bb8..11b9b96e 100644 --- a/fanficdownloader/adapters/adapter_ksarchivecom.py +++ b/fanficdownloader/adapters/adapter_ksarchivecom.py @@ -60,11 +60,6 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','ksa') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Star Trek") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%b/%d/%Y" # XXX diff --git a/fanficdownloader/adapters/adapter_libraryofmoriacom.py b/fanficdownloader/adapters/adapter_libraryofmoriacom.py index 942d7ea9..d6851a85 100644 --- a/fanficdownloader/adapters/adapter_libraryofmoriacom.py +++ b/fanficdownloader/adapters/adapter_libraryofmoriacom.py @@ -56,11 +56,6 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','lom') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Lord of the Rings") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d, %Y" diff --git a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py index aa995f75..878a73ac 100644 --- a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py @@ -56,11 +56,6 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','lsph') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_midnightwhispersca.py b/fanficdownloader/adapters/adapter_midnightwhispersca.py index da7ad4c7..cbdce553 100644 --- a/fanficdownloader/adapters/adapter_midnightwhispersca.py +++ b/fanficdownloader/adapters/adapter_midnightwhispersca.py @@ -60,11 +60,6 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','mw') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Queer as Folk") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d, %Y" # XXX diff --git a/fanficdownloader/adapters/adapter_mugglenetcom.py b/fanficdownloader/adapters/adapter_mugglenetcom.py index f9edda48..26c468f7 100644 --- a/fanficdownloader/adapters/adapter_mugglenetcom.py +++ b/fanficdownloader/adapters/adapter_mugglenetcom.py @@ -59,11 +59,6 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','mgln') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%y" # XXX diff --git a/fanficdownloader/adapters/adapter_nationallibrarynet.py b/fanficdownloader/adapters/adapter_nationallibrarynet.py index 683bcd5c..530da234 100644 --- a/fanficdownloader/adapters/adapter_nationallibrarynet.py +++ b/fanficdownloader/adapters/adapter_nationallibrarynet.py @@ -55,11 +55,6 @@ class NationalLibraryNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','ntlb') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","West Wing") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m-%d-%y" @@ -154,6 +149,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter): for char in value.string.split(', '): self.story.addToList('characters',char) + if 'Pairing' in label: + for char in value.string.split(', '): + self.story.addToList('ships',char) + if 'Warnings' in label: for warning in value.string.split(', '): self.story.addToList('warnings',warning) diff --git a/fanficdownloader/adapters/adapter_ncisficcom.py b/fanficdownloader/adapters/adapter_ncisficcom.py index 91295f15..e0fcb3dd 100644 --- a/fanficdownloader/adapters/adapter_ncisficcom.py +++ b/fanficdownloader/adapters/adapter_ncisficcom.py @@ -55,11 +55,6 @@ class NCISFicComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','ncisf') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","NCIS") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m-%d-%y" @@ -154,6 +149,10 @@ class NCISFicComAdapter(BaseSiteAdapter): for char in value.string.split(', '): self.story.addToList('characters',char) + if 'Pairing' in label: + for char in value.string.split(', '): + self.story.addToList('ships',char) + if 'Warnings' in label: for warning in value.string.split(', '): self.story.addToList('warnings',warning) diff --git a/fanficdownloader/adapters/adapter_ncisfictioncom.py b/fanficdownloader/adapters/adapter_ncisfictioncom.py index 1f44ce92..6099bada 100644 --- a/fanficdownloader/adapters/adapter_ncisfictioncom.py +++ b/fanficdownloader/adapters/adapter_ncisfictioncom.py @@ -56,11 +56,6 @@ class NCISFictionComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','ncisfn') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","NCIS") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d/%m/%Y" @@ -157,6 +152,11 @@ class NCISFictionComAdapter(BaseSiteAdapter): for char in chars: self.story.addToList('characters',char.string) + if 'Pairing' in label: + ships = value.findAll('a') + for ship in ships: + self.story.addToList('ships',ship.string) + if 'Genre' in label: genres = value.findAll('a') for genre in genres: diff --git a/fanficdownloader/adapters/adapter_nfacommunitycom.py b/fanficdownloader/adapters/adapter_nfacommunitycom.py index e33d83da..1c497740 100644 --- a/fanficdownloader/adapters/adapter_nfacommunitycom.py +++ b/fanficdownloader/adapters/adapter_nfacommunitycom.py @@ -60,11 +60,6 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','nfa') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","NCIS") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" # XXX diff --git a/fanficdownloader/adapters/adapter_nhamagicalworldsus.py b/fanficdownloader/adapters/adapter_nhamagicalworldsus.py index f1889d54..6f4aea46 100644 --- a/fanficdownloader/adapters/adapter_nhamagicalworldsus.py +++ b/fanficdownloader/adapters/adapter_nhamagicalworldsus.py @@ -56,11 +56,6 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','nha') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Buffy the Vampire Slayer") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = " %m/%d/%y" diff --git a/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py b/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py index c31a674c..ea1119f1 100644 --- a/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py @@ -56,11 +56,6 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','osph') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py index d3aafd4d..cc57de5a 100644 --- a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py @@ -55,11 +55,6 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','odf') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","One Direction") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%y" diff --git a/fanficdownloader/adapters/adapter_phoenixsongnet.py b/fanficdownloader/adapters/adapter_phoenixsongnet.py index 928c6f5b..00f1a5de 100644 --- a/fanficdownloader/adapters/adapter_phoenixsongnet.py +++ b/fanficdownloader/adapters/adapter_phoenixsongnet.py @@ -55,11 +55,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','phs') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d %Y" diff --git a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py index 47986c22..c40fb2ba 100644 --- a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py +++ b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py @@ -60,11 +60,6 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','pffa') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","My Little Pony") - @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): # The site domain. Does have www here, if it uses it. diff --git a/fanficdownloader/adapters/adapter_portkeyorg.py b/fanficdownloader/adapters/adapter_portkeyorg.py index b13381b6..b14a101b 100644 --- a/fanficdownloader/adapters/adapter_portkeyorg.py +++ b/fanficdownloader/adapters/adapter_portkeyorg.py @@ -62,11 +62,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','prtky') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d/%m/%y" # XXX diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py index 72b411d4..a8dd85ea 100644 --- a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py +++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py @@ -37,7 +37,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter): # Most sites that claim to be # iso-8859-1 (and some that claim to be # utf8) are really windows-1252. - self.story.addToList("category","Harry Potter") # get storyId from url--url validation guarantees query is only sid=1234 self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) @@ -142,7 +141,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter): chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) charstext = [char.string for char in chars] for char in charstext: - if char == "!Snape and Harry (required)": + if "Snape and Harry (required)" in char: self.story.addToList('characters',"Snape") self.story.addToList('characters',"Harry") else: diff --git a/fanficdownloader/adapters/adapter_pretendercentrecom.py b/fanficdownloader/adapters/adapter_pretendercentrecom.py index db200bca..022efa1a 100644 --- a/fanficdownloader/adapters/adapter_pretendercentrecom.py +++ b/fanficdownloader/adapters/adapter_pretendercentrecom.py @@ -55,11 +55,6 @@ class PretenderCenterComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','ptdc') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","The Pretender") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d/%m/%y" diff --git a/fanficdownloader/adapters/adapter_prisonbreakficnet.py b/fanficdownloader/adapters/adapter_prisonbreakficnet.py index ed5df814..0c4f066e 100644 --- a/fanficdownloader/adapters/adapter_prisonbreakficnet.py +++ b/fanficdownloader/adapters/adapter_prisonbreakficnet.py @@ -55,11 +55,6 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','pbf') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Prison Break") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d, %Y" @@ -157,6 +152,11 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter): for char in chars: self.story.addToList('characters',char.string) + if 'Pairing' in label: + ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX + for ship in ships: + self.story.addToList('ships',ship.string.replace(" and ","/")) + if 'Genre' in label: genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX for genre in genres: diff --git a/fanficdownloader/adapters/adapter_samdeanarchivenu.py b/fanficdownloader/adapters/adapter_samdeanarchivenu.py index 38c6f27e..c79645ae 100644 --- a/fanficdownloader/adapters/adapter_samdeanarchivenu.py +++ b/fanficdownloader/adapters/adapter_samdeanarchivenu.py @@ -55,11 +55,6 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','sda') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Supernatural") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%y" diff --git a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py index 8c0ccf28..9d415bb9 100644 --- a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py +++ b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py @@ -55,13 +55,6 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','scacf') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Glee") - self.story.addToList("characters","Kurt Hummel") - self.story.addToList("characters","Blaine Anderson") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py index 093443fd..b067ec6d 100644 --- a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py +++ b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py @@ -58,7 +58,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter): # If all stories from the site fall into the same category, # the site itself isn't likely to label them as such, so we - # do. + # do. Can't use extracategories, could be Atlantis or SG-1 if 'atlantis' in self.story.getMetadata('section'): self.story.addToList("category","Stargate: Atlantis") else: diff --git a/fanficdownloader/adapters/adapter_sinfuldesireorg.py b/fanficdownloader/adapters/adapter_sinfuldesireorg.py index 2255a599..d4874082 100644 --- a/fanficdownloader/adapters/adapter_sinfuldesireorg.py +++ b/fanficdownloader/adapters/adapter_sinfuldesireorg.py @@ -55,11 +55,6 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','snds') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Supernatural") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_siyecouk.py b/fanficdownloader/adapters/adapter_siyecouk.py index 510b6833..dc95559b 100644 --- a/fanficdownloader/adapters/adapter_siyecouk.py +++ b/fanficdownloader/adapters/adapter_siyecouk.py @@ -56,11 +56,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','siye') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%Y.%m.%d" # XXX diff --git a/fanficdownloader/adapters/adapter_stargateatlantisorg.py b/fanficdownloader/adapters/adapter_stargateatlantisorg.py index e40acb02..5cc156c5 100644 --- a/fanficdownloader/adapters/adapter_stargateatlantisorg.py +++ b/fanficdownloader/adapters/adapter_stargateatlantisorg.py @@ -55,11 +55,6 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','stat') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Stargate: Atlantis") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%B %d %Y" diff --git a/fanficdownloader/adapters/adapter_storiesofardacom.py b/fanficdownloader/adapters/adapter_storiesofardacom.py index a26f7343..7bb999b4 100644 --- a/fanficdownloader/adapters/adapter_storiesofardacom.py +++ b/fanficdownloader/adapters/adapter_storiesofardacom.py @@ -55,11 +55,6 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','soa') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Lord of the Rings") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_thealphagatecom.py b/fanficdownloader/adapters/adapter_thealphagatecom.py index 17955e22..98ec91d4 100644 --- a/fanficdownloader/adapters/adapter_thealphagatecom.py +++ b/fanficdownloader/adapters/adapter_thealphagatecom.py @@ -55,11 +55,6 @@ class TheAlphaGateComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','tag') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Stargate: SG-1") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d %b %Y" diff --git a/fanficdownloader/adapters/adapter_thehexfilesnet.py b/fanficdownloader/adapters/adapter_thehexfilesnet.py index b6fc120e..bc515cb7 100644 --- a/fanficdownloader/adapters/adapter_thehexfilesnet.py +++ b/fanficdownloader/adapters/adapter_thehexfilesnet.py @@ -55,11 +55,6 @@ class TheHexFilesNetAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','thf') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%Y.%m.%d" diff --git a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py index 0cf66c05..2c3533d9 100644 --- a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py +++ b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py @@ -60,11 +60,6 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','tqdpch') # XXX - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") # XXX - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" # XXX diff --git a/fanficdownloader/adapters/adapter_twilightarchivescom.py b/fanficdownloader/adapters/adapter_twilightarchivescom.py index 704b2e14..f9003260 100644 --- a/fanficdownloader/adapters/adapter_twilightarchivescom.py +++ b/fanficdownloader/adapters/adapter_twilightarchivescom.py @@ -55,11 +55,6 @@ class TwilightArchivesComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','twa') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Twilight") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d %b %y" diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py index f1cc6589..4e1e2dd1 100644 --- a/fanficdownloader/adapters/adapter_twilightednet.py +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -37,7 +37,6 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): # Most sites that claim to be # iso-8859-1 (and some that claim to be # utf8) are really windows-1252. - self.story.addToList("category","Twilight") self.username = "NoneGiven" # if left empty, site doesn't return any message at all. self.password = "" diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py index 1c70b5ce..415b7f5d 100644 --- a/fanficdownloader/adapters/adapter_twiwritenet.py +++ b/fanficdownloader/adapters/adapter_twiwritenet.py @@ -37,7 +37,6 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter): # Most sites that claim to be # iso-8859-1 (and some that claim to be # utf8) are really windows-1252. - self.story.addToList("category","Twilight") self.is_adult = False self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all. self.password = "" diff --git a/fanficdownloader/adapters/adapter_walkingtheplankorg.py b/fanficdownloader/adapters/adapter_walkingtheplankorg.py index 60784052..e3885387 100644 --- a/fanficdownloader/adapters/adapter_walkingtheplankorg.py +++ b/fanficdownloader/adapters/adapter_walkingtheplankorg.py @@ -53,11 +53,6 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','wtp') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Harry Potter") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%b %d, %Y" diff --git a/fanficdownloader/adapters/adapter_wolverineandroguecom.py b/fanficdownloader/adapters/adapter_wolverineandroguecom.py index 25d664ea..37b09a73 100644 --- a/fanficdownloader/adapters/adapter_wolverineandroguecom.py +++ b/fanficdownloader/adapters/adapter_wolverineandroguecom.py @@ -55,13 +55,6 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','wrfa') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","X-Men Movie") - self.story.addToList("characters","Wolverine") - self.story.addToList("characters","Rogue") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%m/%d/%Y" diff --git a/fanficdownloader/adapters/adapter_wraithbaitcom.py b/fanficdownloader/adapters/adapter_wraithbaitcom.py index c9b7d9ef..9f077a8a 100644 --- a/fanficdownloader/adapters/adapter_wraithbaitcom.py +++ b/fanficdownloader/adapters/adapter_wraithbaitcom.py @@ -56,11 +56,6 @@ class WraithBaitComAdapter(BaseSiteAdapter): # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','wb') - # If all stories from the site fall into the same category, - # the site itself isn't likely to label them as such, so we - # do. - self.story.addToList("category","Stargate") - # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior self.dateformat = "%d %b %Y" diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index 666a0178..0451d558 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -225,7 +225,7 @@ class Story: def getMetadata(self, key, removeallentities=False, doreplacements=True): value = None if self.isList(key): - value = ', '.join(self.getList(key)) + value = ', '.join(self.getList(key, removeallentities)) elif self.metadata.has_key(key): value = self.metadata[key] if value: @@ -310,10 +310,10 @@ class Story: return retlist - def getLists(self): + def getLists(self, removeallentities=False): lsts = {} for ln in self.listables.keys(): - lsts[ln] = self.getList(ln) + lsts[ln] = self.getList(ln, removeallentities) return lsts def isList(self,listname): diff --git a/fanficdownloader/writers/base_writer.py b/fanficdownloader/writers/base_writer.py index 424aa933..7d72120f 100644 --- a/fanficdownloader/writers/base_writer.py +++ b/fanficdownloader/writers/base_writer.py @@ -50,6 +50,7 @@ class BaseStoryWriter(Configurable): 'genre', 'language', 'characters', + 'ships', 'series', 'status', 'datePublished', @@ -81,6 +82,7 @@ class BaseStoryWriter(Configurable): 'status':'Status', 'series':'Series', 'characters':'Characters', + 'ships':'Relationships', 'datePublished':'Published', 'dateUpdated':'Updated', 'dateCreated':'Packaged', @@ -104,12 +106,19 @@ class BaseStoryWriter(Configurable): } self.story.setMetadata('formatname',self.getFormatName()) self.story.setMetadata('formatext',self.getFormatExt()) - - for tag in self.getConfigList("extratags"): - self.story.addToList("extratags",tag) - def getMetadata(self,key): - return stripHTML(self.story.getMetadata(key)) + ## Look for config parameter, split and add each to metadata field. + for (config,metadata) in [("extratags","extratags"), + ("extracategories","category"), + ("extragenres","genre"), + ("extracharacters","characters"), + ("extraships","ships"), + ("extrawarnings","warnings")]: + for val in self.getConfigList(config): + self.story.addToList(metadata,val) + + def getMetadata(self,key, removeallentities=False): + return stripHTML(self.story.getMetadata(key, removeallentities)) def getOutputFileName(self): if self.getConfig('zip_output'): @@ -252,7 +261,7 @@ class BaseStoryWriter(Configurable): if close: outstream.close() - def getTags(self): + def getTags(self, removeallentities=False): # set to avoid duplicates subject tags. subjectset = set() @@ -265,10 +274,10 @@ class BaseStoryWriter(Configurable): if entry in self.getConfigList("include_subject_tags") and \ entry not in self.story.getLists() and \ self.story.getMetadata(entry): - subjectset.add(self.getMetadata(entry)) + subjectset.add(self.getMetadata(entry, removeallentities)) # listables all go into dc:subject tags, but only if they are configured. - for (name,lst) in self.story.getLists().iteritems(): + for (name,lst) in self.story.getLists(removeallentities).iteritems(): if name in self.getConfigList("include_subject_tags"): for tag in lst: subjectset.add(tag) diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 352f9d7a..49ca8e46 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -42,6 +42,7 @@ category_label:Category genre_label:Genre language_label:Language characters_label:Characters +ships_label:Relationships series_label:Series ## Completed/In-Progress status_label:Status @@ -74,7 +75,7 @@ version_label:FFDL Version ## You can include extra text or HTML that will be included as-is in ## the title page. Eg: titlepage_entries: ...,
,summary,
,... ## All current formats already include title and author. -titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description +titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description ## Try to collect series name and number of this story in series. ## Some sites (ab)use 'series' for reading lists and personal @@ -100,11 +101,20 @@ include_tocpage: true ## entries to make epub subjects and calibre tags ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d" -include_subject_tags: extratags, genre, category, characters, status +include_subject_tags: extratags, genre, category, characters, ships, status ## extra tags (comma separated) to include, primarily for epub. extratags: FanFiction +## extra categories, genres, characters, ships and warnings can be +## configured. Used primarily for sites that are dedicated to a genre +## or 'ship and so don't included it for every story. +#extracategories: +#extragenres: +#extracharacters: +#extraships: +#extrawarnings: + ## number of seconds to sleep between calls to the story site. May by ## useful if pulling large numbers of stories or if the site is slow. #slow_down_sleep_time:0.5 @@ -303,6 +313,11 @@ nook_img_fix:true ## URLs like: http://test1.com?sid=12345 [test1.com] extratags: FanFiction,Testing +extracategories:Fafner +extragenres:Romance,Fluff +extracharacters:Reginald Smythe-Smythe,Mokona +extraships:Smythe-Smythe/Mokona +extrawarnings:Extreme Bogosity ## If necessary, you can define [:] sections to ## customize the formats differently for the same site. Overrides @@ -329,6 +344,11 @@ extratags: FanFiction,Testing,HTML #is_adult:true [ashwinder.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Severus Snape,Hermione Granger +extraships:Severus Snape/Hermione Granger + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -337,6 +357,9 @@ extratags: FanFiction,Testing,HTML #password:yourpassword [castlefans.org] +## Site dedicated to these categories/characters/ships +extracategories:Castle + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -360,6 +383,11 @@ extratags: FanFiction,Testing,HTML cover_exclusion_regexp:/images/.*?ribbon.gif [dark-solace.org] +## Site dedicated to these categories/characters/ships +extracategories:Buffy: The Vampire Slayer +extracharacters:Buffy, Spike +extraships:Spike/Buffy + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -368,6 +396,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #password:yourpassword [dramione.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Draco Malfoy,Hermione Granger +extraships:Draco Malfoy/Hermione Granger + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -381,6 +414,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [erosnsappho.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -391,6 +427,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif cover_exclusion_regexp:/images/.*?ribbon.gif [fanfiction.mugglenet.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -404,6 +443,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [fanfiction.portkey.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extraships:Harry Potter/Hermione Granger + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -426,6 +469,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #password:yourpassword [grangerenchanted.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Hermione Granger + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -439,26 +486,51 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [hlfiction.net] +## Site dedicated to these categories/characters/ships +extracategories:Highlander + +[ksarchive.com] +## Site dedicated to these categories/characters/ships +extracategories:Star Trek +extracharacters:Kirk,Spock +extraships:Kirk/Spock [lumos.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [national-library.net] +## Site dedicated to these categories/characters/ships +extracategories:West Wing [ncisfic.com] +## Site dedicated to these categories/characters/ships +extracategories:NCIS [nfacommunity.com] +## Site dedicated to these categories/characters/ships +extracategories:NCIS + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [nha.magical-worlds.us] +## Site dedicated to these categories/characters/ships +extracategories:Buffy: The Vampire Slayer +extracharacters:Willow [occlumency.sycophanthex.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Severus Snape + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -467,6 +539,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #password:yourpassword [onedirectionfanfiction.com] +## Site dedicated to these categories/characters/ships +extracategories:One Direction + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -480,18 +555,28 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [ponyfictionarchive.net] +## Site dedicated to these categories/characters/ships +extracategories:My Little Pony: Friendship is Magic + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [pretendercentre.com] +## Site dedicated to these categories/characters/ships +extracategories:The Pretender + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [samdean.archive.nu] +## Site dedicated to these categories/characters/ships +extracategories:Supernatural +extracharacters:Sam,Dean +extraships:Sam/Dean [sg1-heliopolis.com] ## Some sites do not require a login, but do require the user to @@ -500,6 +585,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [stargate-atlantis.org] +## Site dedicated to these categories/characters/ships +extracategories:Stargate: Atlantis [svufiction.com] ## Some sites require login (or login for some rated stories) The @@ -515,6 +602,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [thehexfiles.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Draco Malfoy,Harry Potter +extraships:Harry Potter/Draco Malfoy [themasque.net] ## Some sites require login (or login for some rated stories) The @@ -530,6 +621,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [thequidditchpitch.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -543,12 +637,20 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [www.adastrafanfic.com] +## Site dedicated to these categories/characters/ships +extracategories:Star Trek + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.dracoandginny.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Draco Malfoy,Ginny Weasley +extraships:Draco Malfoy/Ginny Weasley + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -562,8 +664,13 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [www.thealphagate.com] +## Site dedicated to these categories/characters/ships +extracategories:Stargate: SG-1 [www.checkmated.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -578,6 +685,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #is_adult:true [www.dokuga.com] +## Site dedicated to these categories/characters/ships +extracategories:InuYasha +extracharacters:Sesshoumaru,Kagome +extraships:Sesshoumaru/Kagome [www.fanfiction.net] ## fanfiction.net's 'cover' images are really just tiny thumbnails. @@ -595,6 +706,9 @@ never_make_cover: true [www.ficbook.net] [www.fictionalley.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -613,6 +727,9 @@ extratags: #password:yourpassword [www.fimfiction.net] +## Site dedicated to these categories/characters/ships +extracategories:My Little Pony: Friendship is Magic + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -624,20 +741,33 @@ extratags: #fail_on_password: false [www.harrypotterfanfiction.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.hpfandom.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.hpfanficarchive.com] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter [www.ik-eternal.net] +## Site dedicated to these categories/characters/ships +extracategories:InuYasha +extracharacters:InuYasha,Kagome +extraships:InuYasha/Kagome + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -651,6 +781,9 @@ extratags: #is_adult:true [www.libraryofmoria.com] +## Site dedicated to these categories/characters/ships +extracategories:Lord of the Rings + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -659,6 +792,9 @@ extratags: [www.mediaminer.org] [www.midnightwhispers.ca] +## Site dedicated to these categories/characters/ships +extracategories:Queer as Folk + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -669,8 +805,14 @@ extratags: cover_exclusion_regexp:/stories/999/images/.*?_trophy.png [www.ncisfiction.com] +## Site dedicated to these categories/characters/ships +extracategories:NCIS [www.phoenixsong.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extraships:Harry Potter/Ginny Weasley + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -679,28 +821,43 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png #password:yourpassword [www.potionsandsnitches.net] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter [www.prisonbreakfic.net] +## Site dedicated to these categories/characters/ships +extracategories:Prison Break [www.scarvesandcoffee.net] +## Site dedicated to these categories/characters/ships +extracategories:Glee +extracharacters:Kurt Hummel,Blaine Anderson + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.sinful-desire.org] +## Site dedicated to these categories/characters/ships +extracategories:Supernatural + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. #is_adult:true [www.siye.co.uk] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Harry Potter,Ginny Weasley +extraships:Harry Potter/Ginny Weasley [www.squidge.org/peja] # www.squidge.org/peja calls it Fandom category_label:Fandom # Remove numWords -- www.squidge.org/peja word counts are inaccurate -titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description +titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description [www.squidge.org/peja:txt] ## Add URLs since there aren't links. @@ -708,6 +865,8 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description [www.storiesofarda.com] +## Site dedicated to these categories/characters/ships +extracategories:Lord of the Rings [www.thepetulantpoetess.com] ## Some sites require login (or login for some rated stories) The @@ -747,8 +906,13 @@ collect_series: false #password:yourpassword [www.twilightarchives.com] +## Site dedicated to these categories/characters/ships +extracategories:Twilight [www.twilighted.net] +## Site dedicated to these categories/characters/ships +extracategories:Twilight + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -760,6 +924,9 @@ collect_series: false collect_series: false [www.twiwrite.net] +## Site dedicated to these categories/characters/ships +extracategories:Twilight + ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In ## commandline version, this should go in your personal.ini, not @@ -771,6 +938,11 @@ collect_series: false collect_series: false [www.walkingtheplank.org] +## Site dedicated to these categories/characters/ships +extracategories:Harry Potter +extracharacters:Severus Snape,Harry Potter +extraships:Severus Snape/Harry Potter + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. @@ -792,8 +964,14 @@ collect_series: false #is_adult:true [www.wolverineandrogue.com] +## Site dedicated to these categories/characters/ships +extracategories:X-Men Movie +extracharacters:Wolverine,Rogue [www.wraithbait.com] +## Site dedicated to these categories/characters/ships +extracategories:Stargate: Atlantis + ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, ## this should go in your personal.ini, not defaults.ini. diff --git a/plugin-example.ini b/plugin-example.ini index 5b60e964..a08ea973 100644 --- a/plugin-example.ini +++ b/plugin-example.ini @@ -1,9 +1,9 @@ ## This is an example of what your personal configuration might look -## like. +## like. Uncomment options by removing the '#' in front of them. [defaults] ## Some sites also require the user to confirm they are adult for -## adult content. +## adult content. Uncomment by removing '#' in front of is_adult. #is_adult:true ## include images from img tags in the body and summary of @@ -28,18 +28,9 @@ ## space. #grayscale_images: false -## Try to collect series name and number of this story in series. -## Some sites (ab)use 'series' for reading lists and personal -## collections. This lets us turn it on and off by site without -## keeping a lengthy titlepage_entries per site and prevents it -## updating in the plugin. -## Turn off in [defaults] or [overrides] to prevent all sites from -## updating series column. -## default is true -#collect_series: false - ## Most common, I expect will be using this to save username/passwords -## for different sites. +## for different sites. Here are a few examples. + [www.twilighted.net] #username:YourPenname #password:YourPassword