From db48233cf44b6f147b62e95d74d3e813cbc30e21 Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Sat, 7 Oct 2017 13:01:44 -0400 Subject: [PATCH 1/7] Switch from using raw argparser to using click. Preserves the existing interface, except leech --flush becomes leech flush --- README.markdown | 10 ++++ leech.py | 140 ++++++++++++++++++++++++++-------------------- requirements.txt | 2 + sites/__init__.py | 14 +---- sites/xenforo.py | 16 ++---- 5 files changed, 98 insertions(+), 84 deletions(-) diff --git a/README.markdown b/README.markdown index 84e69ae..3869073 100644 --- a/README.markdown +++ b/README.markdown @@ -19,10 +19,20 @@ My recommended setup process is: Usage --- +Basic + $ python3 leech.py [[URL]] A new file will appear named `Title of the Story.epub`. +This is equivalent to the slightly longer + + $ python3 leech.py download [[URL]] + +Flushing the cache + + $ python3 leech.py flush + If you want to put it on a Kindle you'll have to convert it. I'd recommend [Calibre](http://calibre-ebook.com/), though you could also try using [kindlegen](http://www.amazon.com/gp/feature.html?docId=1000765211) directly. Supports diff --git a/leech.py b/leech.py index b73c26e..da74ffe 100755 --- a/leech.py +++ b/leech.py @@ -1,79 +1,99 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import argparse -import sys -import json +import click +from click_default_group import DefaultGroup + +import requests +import requests_cache import http.cookiejar +import json import sites import ebook -import requests -import requests_cache - -__version__ = 1 +__version__ = 2 USER_AGENT = 'Leech/%s +http://davidlynch.org' % __version__ -def leech(url, session, filename=None, args=None): - # we have: a page, which could be absolutely any part of a story, or not a story at all - # check a bunch of things which are completely ff.n specific, to get text from it - site, url = sites.get(url) - if not site: - raise Exception("No site handler found") +def uses_session(command): + """Decorator for click commands that need a session.""" + @click.option('--cache/--no-cache', default=True) + def wrapper(cache, **kwargs): + if cache: + session = requests_cache.CachedSession('leech', expire_after=4 * 3600) + else: + session = requests.Session() - print("Handler", site, url) + lwp_cookiejar = http.cookiejar.LWPCookieJar() + try: + lwp_cookiejar.load('leech.cookies', ignore_discard=True) + except Exception as e: + pass + session.cookies = lwp_cookiejar + session.headers.update({ + 'User-agent': USER_AGENT + }) + return command(session=session, **kwargs) + wrapper.__name__ = command.__name__ + return wrapper - handler = site(session, args=args) - with open('leech.json') as config_file: - config = json.load(config_file) +def uses_story(command): + """Decorator for click commands that need a story.""" + @click.argument('url') + @click.option('--include-index', default=False, help='[Xenforo only] Should the chapter marked as an index be included?') + @click.option('--offset', type=int, default=None, help='[Xenforo only] The chapter to start from.') + @click.option('--limit', type=int, default=None, help='[Xenforo only] The chapter to end with.') + @click.option('--skip-spoilers/--include-spoilers', default=True, help='[Xenforo only] If the story should include content enclosed in spoiler tags.') + @uses_session + def wrapper(url, session, include_index, offset, limit, skip_spoilers, **kwargs): + site, url = sites.get(url) + if not site: + raise Exception("No site handler found") - login = config.get('logins', {}).get(site.__name__, False) - if login: - handler.login(login) + handler = site(session, options={ + 'offset': offset, + 'limit': limit, + 'skip_spoilers': skip_spoilers, + 'include_index': include_index, + }) - cover_options = config.get('cover', {}) + with open('leech.json') as store_file: + store = json.load(store_file) + login = store.get('logins', {}).get(site.__name__, False) + if login: + handler.login(login) - story = handler.extract(url) - if not story: - raise Exception("Couldn't extract story") + story = handler.extract(url) + if not story: + raise Exception("Couldn't extract story") - return ebook.generate_epub(story, filename, cover_options=cover_options) + command(story=story, **kwargs) + wrapper.__name__ = command.__name__ + return wrapper + + +@click.group(cls=DefaultGroup, default='download', default_if_no_args=True) +def cli(): + """Top level click group. Uses click-default-group to preserve most behavior from leech v1.""" + pass + + +@cli.command() +def flush(): + """"Flushes the contents of the cache.""" + requests_cache.install_cache('leech') + requests_cache.clear() + print("Flushed cache") + + +@cli.command() +@uses_story +def download(story): + """Downloads a story and saves it on disk as a ebpub ebook.""" + filename = ebook.generate_epub(story) + print("File created:", filename) if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('url', help="url of a story to fetch", nargs='?') - parser.add_argument('--filename', help="output filename (the title is used if this isn't provided)") - parser.add_argument('--no-cache', dest='cache', action='store_false') - parser.add_argument('--flush', dest='flush', action='store_true') - parser.set_defaults(cache=True, flush=False) - args, extra_args = parser.parse_known_args() - - if args.flush: - requests_cache.install_cache('leech') - requests_cache.clear() - print("Flushed cache") - sys.exit() - - if not args.url: - sys.exit("URL is required") - - if args.cache: - session = requests_cache.CachedSession('leech', expire_after=4 * 3600) - else: - session = requests.Session() - - lwp_cookiejar = http.cookiejar.LWPCookieJar() - try: - lwp_cookiejar.load('leech.cookies', ignore_discard=True) - except Exception as e: - pass - session.cookies = lwp_cookiejar - session.headers.update({ - 'User-agent': USER_AGENT - }) - - filename = leech(args.url, filename=args.filename, session=session, args=extra_args) - print("File created:", filename) + cli() diff --git a/requirements.txt b/requirements.txt index 666a0ec..7bab54f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,5 @@ requests-cache==0.4.13 six==1.10.0 urllib3==1.22 webencodings==0.5.1 +click==6.7 +click-default-group==1.2 diff --git a/sites/__init__.py b/sites/__init__.py index 70ab656..efd82df 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -1,7 +1,6 @@ import glob import os -import argparse import uuid import attr from bs4 import BeautifulSoup @@ -62,11 +61,8 @@ class Site: extracting the content of a story from said site. """ session = attr.ib() - args = attr.ib() footnotes = attr.ib(default=attr.Factory(list), init=False) - - def __attrs_post_init__(self): - self.options = self._parse_args(self.args) + options = attr.ib(default=attr.Factory(dict)) @staticmethod def matches(url): @@ -88,14 +84,6 @@ class Site: def login(self, login_details): raise NotImplementedError() - def _parse_args(self, args): - parser = argparse.ArgumentParser() - self._add_arguments(parser) - return parser.parse_args(args) - - def _add_arguments(self, parser): - pass - def _soup(self, url, method='html5lib', **kw): page = self.session.get(url, **kw) if not page: diff --git a/sites/xenforo.py b/sites/xenforo.py index cb0e8a1..974b1a9 100644 --- a/sites/xenforo.py +++ b/sites/xenforo.py @@ -40,7 +40,7 @@ class XenForo(Site): mark for mark in self._chapter_list(url) if '/members' not in mark.get('href') and '/threadmarks' not in mark.get('href') ] - marks = marks[self.options.offset:self.options.limit] + marks = marks[self.options['offset']:self.options['limit']] for idx, mark in enumerate(marks, 1): href = mark.get('href') @@ -95,7 +95,7 @@ class XenForo(Site): if not links: raise SiteException("No links in index?") - if self.options.include_index: + if self.options['include_index']: fake_link = self._new_tag('a', href=url) fake_link.string = "Index" links.insert(0, fake_link) @@ -151,7 +151,7 @@ class XenForo(Site): # spoilers don't work well, so turn them into epub footnotes for idx, spoiler in enumerate(post.find_all(class_='ToggleTriggerAnchor')): spoiler_title = spoiler.find(class_='SpoilerTitle') - if self.options.spoilers: + if self.options['skip_spoilers']: link = self._footnote(spoiler.find(class_='SpoilerTarget').extract(), chapterid) if spoiler_title: link.string = spoiler_title.get_text() @@ -174,12 +174,6 @@ class XenForo(Site): return datetime.datetime.strptime(maybe_date['title'], "%b %d, %Y at %I:%M %p") raise SiteException("No date", maybe_date) - def _add_arguments(self, parser): - parser.add_argument('--include-index', dest='include_index', action='store_true', default=False) - parser.add_argument('--offset', dest='offset', type=int, default=None) - parser.add_argument('--limit', dest='limit', type=int, default=None) - parser.add_argument('--skip-spoilers', dest='spoilers', action='store_false', default=True) - class XenForoIndex(XenForo): @classmethod @@ -198,8 +192,8 @@ class SpaceBattles(XenForo): @register -class SpaceBattlesIndex(XenForoIndex): - domain = 'forums.spacebattles.com' +class SpaceBattlesIndex(SpaceBattles, XenForoIndex): + pass @register From c702337040ac57c452a764212d9f92aa5766fdb9 Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Fri, 13 Oct 2017 19:37:13 -0400 Subject: [PATCH 2/7] Reworks how site-specific options work. --- leech.py | 39 ++++++++++++++++++++++++++------------- sites/__init__.py | 4 ++++ sites/xenforo.py | 9 +++++++++ 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/leech.py b/leech.py index da74ffe..2f89753 100755 --- a/leech.py +++ b/leech.py @@ -41,28 +41,41 @@ def uses_session(command): def uses_story(command): """Decorator for click commands that need a story.""" @click.argument('url') - @click.option('--include-index', default=False, help='[Xenforo only] Should the chapter marked as an index be included?') - @click.option('--offset', type=int, default=None, help='[Xenforo only] The chapter to start from.') - @click.option('--limit', type=int, default=None, help='[Xenforo only] The chapter to end with.') - @click.option('--skip-spoilers/--include-spoilers', default=True, help='[Xenforo only] If the story should include content enclosed in spoiler tags.') + @click.option( + '--site-options', + default='{}', + help='JSON object encoding any site specific option.' + ) @uses_session - def wrapper(url, session, include_index, offset, limit, skip_spoilers, **kwargs): + def wrapper(url, session, site_options, **kwargs): site, url = sites.get(url) if not site: raise Exception("No site handler found") - handler = site(session, options={ - 'offset': offset, - 'limit': limit, - 'skip_spoilers': skip_spoilers, - 'include_index': include_index, - }) + default_site_options = site.get_default_options() with open('leech.json') as store_file: store = json.load(store_file) login = store.get('logins', {}).get(site.__name__, False) - if login: - handler.login(login) + configured_site_options = store.get('site_options', {}).get(site.__name__, {}) + + overridden_site_options = json.loads(site_options) + + # The final options dictionary is computed by layering the default, configured, + # and overridden options together in that order. + options = dict( + list(default_site_options.items()) + + list(configured_site_options.items()) + + list(overridden_site_options.items()) + ) + + handler = site( + session, + options=options + ) + + if login: + handler.login(login) story = handler.extract(url) if not story: diff --git a/sites/__init__.py b/sites/__init__.py index efd82df..a1687be 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -64,6 +64,10 @@ class Site: footnotes = attr.ib(default=attr.Factory(list), init=False) options = attr.ib(default=attr.Factory(dict)) + @staticmethod + def get_default_options(): + return {} + @staticmethod def matches(url): raise NotImplementedError() diff --git a/sites/xenforo.py b/sites/xenforo.py index 974b1a9..3b9382b 100644 --- a/sites/xenforo.py +++ b/sites/xenforo.py @@ -10,6 +10,15 @@ class XenForo(Site): domain = False + @staticmethod + def get_default_options(): + return { + 'offset': None, + 'limit': None, + 'skip_spoilers': True, + 'include_index': False, + } + @classmethod def matches(cls, url): match = re.match(r'^(https?://%s/threads/[^/]*\d+)/?.*' % cls.domain, url) From 9b4d2a09986ff1a55e0eaa5955c29266d262f3ee Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Fri, 13 Oct 2017 19:43:38 -0400 Subject: [PATCH 3/7] Adds a more sensible default for options in the Site base class. --- sites/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sites/__init__.py b/sites/__init__.py index a1687be..b2556a4 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -62,7 +62,10 @@ class Site: """ session = attr.ib() footnotes = attr.ib(default=attr.Factory(list), init=False) - options = attr.ib(default=attr.Factory(dict)) + options = attr.ib(default=attr.Factory( + lambda site: site.get_default_options(), + True + )) @staticmethod def get_default_options(): From fd7998ea272006197485a59902323c719860910f Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Fri, 13 Apr 2018 17:57:08 -0400 Subject: [PATCH 4/7] Clean up how verbose log output works... --- leech.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/leech.py b/leech.py index 8644c55..19a93e8 100755 --- a/leech.py +++ b/leech.py @@ -19,7 +19,10 @@ logger = logging.getLogger(__name__) def configure_logging(verbose): if verbose: - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig( + level=logging.DEBUG, + format="[%(name)s @ %(levelname)s] %(message)s" + ) else: logging.basicConfig( level=logging.INFO, From 379c087086687cc6630c7780e4d6799a1fb5c06e Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Fri, 13 Apr 2018 18:10:46 -0400 Subject: [PATCH 5/7] Merge a few things I missed in by hand. --- leech.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/leech.py b/leech.py index 19a93e8..25deab9 100755 --- a/leech.py +++ b/leech.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import click -from click_default_group import DefaultGroup - +import http.cookiejar +import json +import logging import requests import requests_cache -import http.cookiejar -import logging -import json +import sqlite3 +from click_default_group import DefaultGroup import sites import ebook @@ -52,6 +52,8 @@ def open_story(url, session, site_options): if not site: raise Exception("No site handler found") + logger.info("Handler: %s (%s)", site, url) + default_site_options = site.get_default_options() with open('leech.json') as store_file: @@ -95,6 +97,11 @@ def flush(verbose): configure_logging(verbose) requests_cache.install_cache('leech') requests_cache.clear() + + conn = sqlite3.connect('leech.sqlite') + conn.execute("VACUUM") + conn.close() + logger.info("Flushed cache") From 7c1702e6ffc95d67351def09a9ed8cf85b6f6013 Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Fri, 13 Apr 2018 18:18:37 -0400 Subject: [PATCH 6/7] Fixes whitespace issues. --- leech.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/leech.py b/leech.py index 25deab9..91b14b8 100755 --- a/leech.py +++ b/leech.py @@ -17,6 +17,7 @@ USER_AGENT = 'Leech/%s +http://davidlynch.org' % __version__ logger = logging.getLogger(__name__) + def configure_logging(verbose): if verbose: logging.basicConfig( @@ -29,6 +30,7 @@ def configure_logging(verbose): format="[%(name)s] %(message)s" ) + def create_session(cache): if cache: session = requests_cache.CachedSession('leech', expire_after=4 * 3600) @@ -46,6 +48,7 @@ def create_session(cache): }) return session + def open_story(url, session, site_options): site, url = sites.get(url) @@ -84,6 +87,7 @@ def open_story(url, session, site_options): raise Exception("Couldn't extract story") return story + @click.group(cls=DefaultGroup, default='download', default_if_no_args=True) def cli(): """Top level click group. Uses click-default-group to preserve most behavior from leech v1.""" From d1842e2bf10552d3bf98efd56d2c8d4bb49571be Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Sat, 14 Apr 2018 12:56:31 -0400 Subject: [PATCH 7/7] Adds a system for site options to be included as click.options on commands. --- leech.py | 51 ++++++++++++++++++++++++--------- sites/__init__.py | 73 +++++++++++++++++++++++++++++++++++++++++++++-- sites/xenforo.py | 36 +++++++++++++++++------ 3 files changed, 136 insertions(+), 24 deletions(-) diff --git a/leech.py b/leech.py index 91b14b8..bcf12d5 100755 --- a/leech.py +++ b/leech.py @@ -8,6 +8,7 @@ import requests import requests_cache import sqlite3 from click_default_group import DefaultGroup +from functools import reduce import sites import ebook @@ -49,31 +50,43 @@ def create_session(cache): return session -def open_story(url, session, site_options): - site, url = sites.get(url) +def load_on_disk_options(site): + try: + with open('leech.json') as store_file: + store = json.load(store_file) + login = store.get('logins', {}).get(site.__name__, False) + configured_site_options = store.get('site_options', {}).get(site.__name__, {}) + except FileNotFoundError: + logger.info("Unable to locate leech.json. Continuing assuming it does not exist.") + login = False + configured_site_options = {} + return configured_site_options, login - if not site: - raise Exception("No site handler found") - - logger.info("Handler: %s (%s)", site, url) +def create_options(site, site_options, unused_flags): + """Compiles options provided from multiple different sources + (e.g. on disk, via flags, via defaults, via JSON provided as a flag value) + into a single options object.""" default_site_options = site.get_default_options() - with open('leech.json') as store_file: - store = json.load(store_file) - login = store.get('logins', {}).get(site.__name__, False) - configured_site_options = store.get('site_options', {}).get(site.__name__, {}) + flag_specified_site_options = site.interpret_site_specific_options(**unused_flags) + + configured_site_options, login = load_on_disk_options(site) overridden_site_options = json.loads(site_options) # The final options dictionary is computed by layering the default, configured, - # and overridden options together in that order. + # and overridden, and flag-specified options together in that order. options = dict( list(default_site_options.items()) + list(configured_site_options.items()) + - list(overridden_site_options.items()) + list(overridden_site_options.items()) + + list(flag_specified_site_options.items()) ) + return options, login + +def open_story(site, url, session, login, options): handler = site( session, options=options @@ -88,6 +101,11 @@ def open_story(url, session, site_options): return story +def site_specific_options(f): + option_list = sites.list_site_specific_options() + return reduce(lambda cmd, decorator: decorator(cmd), [f] + option_list) + + @click.group(cls=DefaultGroup, default='download', default_if_no_args=True) def cli(): """Top level click group. Uses click-default-group to preserve most behavior from leech v1.""" @@ -118,11 +136,16 @@ def flush(verbose): ) @click.option('--cache/--no-cache', default=True) @click.option('--verbose', '-v', is_flag=True, help="Verbose debugging output") -def download(url, site_options, cache, verbose): +@site_specific_options # Includes other click.options specific to sites +def download(url, site_options, cache, verbose, **other_flags): """Downloads a story and saves it on disk as a ebpub ebook.""" configure_logging(verbose) session = create_session(cache) - story = open_story(url, session, site_options) + + site, url = sites.get(url) + options, login = create_options(site, site_options, other_flags) + story = open_story(site, url, session, login, options) + filename = ebook.generate_epub(story) logger.info("File created: " + filename) diff --git a/sites/__init__.py b/sites/__init__.py index 8e9b67f..63bb0bb 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -1,4 +1,5 @@ +import click import glob import os import uuid @@ -72,8 +73,37 @@ class Site: )) @staticmethod - def get_default_options(): - return {} + def get_site_specific_option_defs(): + """Returns a list of click.option objects to add to CLI commands. + + It is best practice to ensure that these names are reasonably unique + to ensure that they do not conflict with the core options, or other + sites' options. It is OK for different site's options to have the + same name, but pains should be taken to ensure they remain semantically + similar in meaning. + """ + return [] + + @classmethod + def get_default_options(cls): + options = {} + for option in cls.get_site_specific_option_defs(): + options[option.name] = option.default + return options + + @classmethod + def interpret_site_specific_options(cls, **kwargs): + """Returns options summarizing CLI flags provided. + + Only includes entries the user has explicitly provided as flags + / will not contain default values. For that, use get_default_options(). + """ + options = {} + for option in cls.get_site_specific_option_defs(): + option_value = kwargs[option.name] + if option_value is not None: + options[option.name] = option_value + return options @staticmethod def matches(url): @@ -148,6 +178,32 @@ class Site: return spoiler_link +@attr.s(hash=True) +class SiteSpecificOption: + """Represents a site-specific option that can be configured. + + Will be added to the CLI as a click.option -- many of these + fields correspond to click.option arguments.""" + name = attr.ib() + flag_pattern = attr.ib() + type = attr.ib(default=None) + help = attr.ib(default=None) + default = attr.ib(default=None) + + def as_click_option(self): + return click.option( + str(self.name), + str(self.flag_pattern), + type=self.type, + # Note: This default not matching self.default is intentional. + # It ensures that we know if a flag was explicitly provided, + # which keeps it from overriding options set in leech.json etc. + # Instead, default is used in site_cls.get_default_options() + default=None, + help=self.help if self.help is not None else "" + ) + + class SiteException(Exception): pass @@ -161,10 +217,23 @@ def get(url): for site_class in _sites: match = site_class.matches(url) if match: + logger.info("Handler: %s (%s)", site_class, match) return site_class, match raise NotImplementedError("Could not find a handler for " + url) +def list_site_specific_options(): + """Returns a list of all site's click options, which will be presented to the user.""" + + # Ensures that duplicate options are not added twice. + # Especially important for subclassed sites (e.g. Xenforo sites) + options = set() + + for site_class in _sites: + options.update(site_class.get_site_specific_option_defs()) + return [option.as_click_option() for option in options] + + # And now, a particularly hacky take on a plugin system: # Make an __all__ out of all the python files in this directory that don't start # with __. Then import * them. diff --git a/sites/xenforo.py b/sites/xenforo.py index b5497c4..83cefff 100644 --- a/sites/xenforo.py +++ b/sites/xenforo.py @@ -3,7 +3,7 @@ import datetime import re import logging -from . import register, Site, SiteException, Section, Chapter +from . import register, Site, SiteException, SiteSpecificOption, Section, Chapter logger = logging.getLogger(__name__) @@ -14,13 +14,33 @@ class XenForo(Site): domain = False @staticmethod - def get_default_options(): - return { - 'offset': None, - 'limit': None, - 'skip_spoilers': True, - 'include_index': False, - } + def get_site_specific_option_defs(): + return [ + SiteSpecificOption( + 'include_index', + '--include-index/--no-include-index', + default=False, + help="If true, the post marked as an index will be included as a chapter." + ), + SiteSpecificOption( + 'skip_spoilers', + '--skip-spoilers/--include-spoilers', + default=True, + help="If true, do not transcribe any tags that are marked as a spoiler." + ), + SiteSpecificOption( + 'offset', + '--offset', + type=int, + help="The chapter index to start in the chapter marks." + ), + SiteSpecificOption( + 'limit', + '--limit', + type=int, + help="The chapter to end at at in the chapter marks." + ), + ] @classmethod def matches(cls, url):