diff --git a/README.markdown b/README.markdown index 84e69ae..3869073 100644 --- a/README.markdown +++ b/README.markdown @@ -19,10 +19,20 @@ My recommended setup process is: Usage --- +Basic + $ python3 leech.py [[URL]] A new file will appear named `Title of the Story.epub`. +This is equivalent to the slightly longer + + $ python3 leech.py download [[URL]] + +Flushing the cache + + $ python3 leech.py flush + If you want to put it on a Kindle you'll have to convert it. I'd recommend [Calibre](http://calibre-ebook.com/), though you could also try using [kindlegen](http://www.amazon.com/gp/feature.html?docId=1000765211) directly. Supports diff --git a/leech.py b/leech.py index b73c26e..da74ffe 100755 --- a/leech.py +++ b/leech.py @@ -1,79 +1,99 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import argparse -import sys -import json +import click +from click_default_group import DefaultGroup + +import requests +import requests_cache import http.cookiejar +import json import sites import ebook -import requests -import requests_cache - -__version__ = 1 +__version__ = 2 USER_AGENT = 'Leech/%s +http://davidlynch.org' % __version__ -def leech(url, session, filename=None, args=None): - # we have: a page, which could be absolutely any part of a story, or not a story at all - # check a bunch of things which are completely ff.n specific, to get text from it - site, url = sites.get(url) - if not site: - raise Exception("No site handler found") +def uses_session(command): + """Decorator for click commands that need a session.""" + @click.option('--cache/--no-cache', default=True) + def wrapper(cache, **kwargs): + if cache: + session = requests_cache.CachedSession('leech', expire_after=4 * 3600) + else: + session = requests.Session() - print("Handler", site, url) + lwp_cookiejar = http.cookiejar.LWPCookieJar() + try: + lwp_cookiejar.load('leech.cookies', ignore_discard=True) + except Exception as e: + pass + session.cookies = lwp_cookiejar + session.headers.update({ + 'User-agent': USER_AGENT + }) + return command(session=session, **kwargs) + wrapper.__name__ = command.__name__ + return wrapper - handler = site(session, args=args) - with open('leech.json') as config_file: - config = json.load(config_file) +def uses_story(command): + """Decorator for click commands that need a story.""" + @click.argument('url') + @click.option('--include-index', default=False, help='[Xenforo only] Should the chapter marked as an index be included?') + @click.option('--offset', type=int, default=None, help='[Xenforo only] The chapter to start from.') + @click.option('--limit', type=int, default=None, help='[Xenforo only] The chapter to end with.') + @click.option('--skip-spoilers/--include-spoilers', default=True, help='[Xenforo only] If the story should include content enclosed in spoiler tags.') + @uses_session + def wrapper(url, session, include_index, offset, limit, skip_spoilers, **kwargs): + site, url = sites.get(url) + if not site: + raise Exception("No site handler found") - login = config.get('logins', {}).get(site.__name__, False) - if login: - handler.login(login) + handler = site(session, options={ + 'offset': offset, + 'limit': limit, + 'skip_spoilers': skip_spoilers, + 'include_index': include_index, + }) - cover_options = config.get('cover', {}) + with open('leech.json') as store_file: + store = json.load(store_file) + login = store.get('logins', {}).get(site.__name__, False) + if login: + handler.login(login) - story = handler.extract(url) - if not story: - raise Exception("Couldn't extract story") + story = handler.extract(url) + if not story: + raise Exception("Couldn't extract story") - return ebook.generate_epub(story, filename, cover_options=cover_options) + command(story=story, **kwargs) + wrapper.__name__ = command.__name__ + return wrapper + + +@click.group(cls=DefaultGroup, default='download', default_if_no_args=True) +def cli(): + """Top level click group. Uses click-default-group to preserve most behavior from leech v1.""" + pass + + +@cli.command() +def flush(): + """"Flushes the contents of the cache.""" + requests_cache.install_cache('leech') + requests_cache.clear() + print("Flushed cache") + + +@cli.command() +@uses_story +def download(story): + """Downloads a story and saves it on disk as a ebpub ebook.""" + filename = ebook.generate_epub(story) + print("File created:", filename) if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('url', help="url of a story to fetch", nargs='?') - parser.add_argument('--filename', help="output filename (the title is used if this isn't provided)") - parser.add_argument('--no-cache', dest='cache', action='store_false') - parser.add_argument('--flush', dest='flush', action='store_true') - parser.set_defaults(cache=True, flush=False) - args, extra_args = parser.parse_known_args() - - if args.flush: - requests_cache.install_cache('leech') - requests_cache.clear() - print("Flushed cache") - sys.exit() - - if not args.url: - sys.exit("URL is required") - - if args.cache: - session = requests_cache.CachedSession('leech', expire_after=4 * 3600) - else: - session = requests.Session() - - lwp_cookiejar = http.cookiejar.LWPCookieJar() - try: - lwp_cookiejar.load('leech.cookies', ignore_discard=True) - except Exception as e: - pass - session.cookies = lwp_cookiejar - session.headers.update({ - 'User-agent': USER_AGENT - }) - - filename = leech(args.url, filename=args.filename, session=session, args=extra_args) - print("File created:", filename) + cli() diff --git a/requirements.txt b/requirements.txt index 666a0ec..7bab54f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,5 @@ requests-cache==0.4.13 six==1.10.0 urllib3==1.22 webencodings==0.5.1 +click==6.7 +click-default-group==1.2 diff --git a/sites/__init__.py b/sites/__init__.py index 70ab656..efd82df 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -1,7 +1,6 @@ import glob import os -import argparse import uuid import attr from bs4 import BeautifulSoup @@ -62,11 +61,8 @@ class Site: extracting the content of a story from said site. """ session = attr.ib() - args = attr.ib() footnotes = attr.ib(default=attr.Factory(list), init=False) - - def __attrs_post_init__(self): - self.options = self._parse_args(self.args) + options = attr.ib(default=attr.Factory(dict)) @staticmethod def matches(url): @@ -88,14 +84,6 @@ class Site: def login(self, login_details): raise NotImplementedError() - def _parse_args(self, args): - parser = argparse.ArgumentParser() - self._add_arguments(parser) - return parser.parse_args(args) - - def _add_arguments(self, parser): - pass - def _soup(self, url, method='html5lib', **kw): page = self.session.get(url, **kw) if not page: diff --git a/sites/xenforo.py b/sites/xenforo.py index cb0e8a1..974b1a9 100644 --- a/sites/xenforo.py +++ b/sites/xenforo.py @@ -40,7 +40,7 @@ class XenForo(Site): mark for mark in self._chapter_list(url) if '/members' not in mark.get('href') and '/threadmarks' not in mark.get('href') ] - marks = marks[self.options.offset:self.options.limit] + marks = marks[self.options['offset']:self.options['limit']] for idx, mark in enumerate(marks, 1): href = mark.get('href') @@ -95,7 +95,7 @@ class XenForo(Site): if not links: raise SiteException("No links in index?") - if self.options.include_index: + if self.options['include_index']: fake_link = self._new_tag('a', href=url) fake_link.string = "Index" links.insert(0, fake_link) @@ -151,7 +151,7 @@ class XenForo(Site): # spoilers don't work well, so turn them into epub footnotes for idx, spoiler in enumerate(post.find_all(class_='ToggleTriggerAnchor')): spoiler_title = spoiler.find(class_='SpoilerTitle') - if self.options.spoilers: + if self.options['skip_spoilers']: link = self._footnote(spoiler.find(class_='SpoilerTarget').extract(), chapterid) if spoiler_title: link.string = spoiler_title.get_text() @@ -174,12 +174,6 @@ class XenForo(Site): return datetime.datetime.strptime(maybe_date['title'], "%b %d, %Y at %I:%M %p") raise SiteException("No date", maybe_date) - def _add_arguments(self, parser): - parser.add_argument('--include-index', dest='include_index', action='store_true', default=False) - parser.add_argument('--offset', dest='offset', type=int, default=None) - parser.add_argument('--limit', dest='limit', type=int, default=None) - parser.add_argument('--skip-spoilers', dest='spoilers', action='store_false', default=True) - class XenForoIndex(XenForo): @classmethod @@ -198,8 +192,8 @@ class SpaceBattles(XenForo): @register -class SpaceBattlesIndex(XenForoIndex): - domain = 'forums.spacebattles.com' +class SpaceBattlesIndex(SpaceBattles, XenForoIndex): + pass @register