From d1842e2bf10552d3bf98efd56d2c8d4bb49571be Mon Sep 17 00:00:00 2001 From: Will Oursler Date: Sat, 14 Apr 2018 12:56:31 -0400 Subject: [PATCH] Adds a system for site options to be included as click.options on commands. --- leech.py | 51 ++++++++++++++++++++++++--------- sites/__init__.py | 73 +++++++++++++++++++++++++++++++++++++++++++++-- sites/xenforo.py | 36 +++++++++++++++++------ 3 files changed, 136 insertions(+), 24 deletions(-) diff --git a/leech.py b/leech.py index 91b14b8..bcf12d5 100755 --- a/leech.py +++ b/leech.py @@ -8,6 +8,7 @@ import requests import requests_cache import sqlite3 from click_default_group import DefaultGroup +from functools import reduce import sites import ebook @@ -49,31 +50,43 @@ def create_session(cache): return session -def open_story(url, session, site_options): - site, url = sites.get(url) +def load_on_disk_options(site): + try: + with open('leech.json') as store_file: + store = json.load(store_file) + login = store.get('logins', {}).get(site.__name__, False) + configured_site_options = store.get('site_options', {}).get(site.__name__, {}) + except FileNotFoundError: + logger.info("Unable to locate leech.json. Continuing assuming it does not exist.") + login = False + configured_site_options = {} + return configured_site_options, login - if not site: - raise Exception("No site handler found") - - logger.info("Handler: %s (%s)", site, url) +def create_options(site, site_options, unused_flags): + """Compiles options provided from multiple different sources + (e.g. on disk, via flags, via defaults, via JSON provided as a flag value) + into a single options object.""" default_site_options = site.get_default_options() - with open('leech.json') as store_file: - store = json.load(store_file) - login = store.get('logins', {}).get(site.__name__, False) - configured_site_options = store.get('site_options', {}).get(site.__name__, {}) + flag_specified_site_options = site.interpret_site_specific_options(**unused_flags) + + configured_site_options, login = load_on_disk_options(site) overridden_site_options = json.loads(site_options) # The final options dictionary is computed by layering the default, configured, - # and overridden options together in that order. + # and overridden, and flag-specified options together in that order. options = dict( list(default_site_options.items()) + list(configured_site_options.items()) + - list(overridden_site_options.items()) + list(overridden_site_options.items()) + + list(flag_specified_site_options.items()) ) + return options, login + +def open_story(site, url, session, login, options): handler = site( session, options=options @@ -88,6 +101,11 @@ def open_story(url, session, site_options): return story +def site_specific_options(f): + option_list = sites.list_site_specific_options() + return reduce(lambda cmd, decorator: decorator(cmd), [f] + option_list) + + @click.group(cls=DefaultGroup, default='download', default_if_no_args=True) def cli(): """Top level click group. Uses click-default-group to preserve most behavior from leech v1.""" @@ -118,11 +136,16 @@ def flush(verbose): ) @click.option('--cache/--no-cache', default=True) @click.option('--verbose', '-v', is_flag=True, help="Verbose debugging output") -def download(url, site_options, cache, verbose): +@site_specific_options # Includes other click.options specific to sites +def download(url, site_options, cache, verbose, **other_flags): """Downloads a story and saves it on disk as a ebpub ebook.""" configure_logging(verbose) session = create_session(cache) - story = open_story(url, session, site_options) + + site, url = sites.get(url) + options, login = create_options(site, site_options, other_flags) + story = open_story(site, url, session, login, options) + filename = ebook.generate_epub(story) logger.info("File created: " + filename) diff --git a/sites/__init__.py b/sites/__init__.py index 8e9b67f..63bb0bb 100644 --- a/sites/__init__.py +++ b/sites/__init__.py @@ -1,4 +1,5 @@ +import click import glob import os import uuid @@ -72,8 +73,37 @@ class Site: )) @staticmethod - def get_default_options(): - return {} + def get_site_specific_option_defs(): + """Returns a list of click.option objects to add to CLI commands. + + It is best practice to ensure that these names are reasonably unique + to ensure that they do not conflict with the core options, or other + sites' options. It is OK for different site's options to have the + same name, but pains should be taken to ensure they remain semantically + similar in meaning. + """ + return [] + + @classmethod + def get_default_options(cls): + options = {} + for option in cls.get_site_specific_option_defs(): + options[option.name] = option.default + return options + + @classmethod + def interpret_site_specific_options(cls, **kwargs): + """Returns options summarizing CLI flags provided. + + Only includes entries the user has explicitly provided as flags + / will not contain default values. For that, use get_default_options(). + """ + options = {} + for option in cls.get_site_specific_option_defs(): + option_value = kwargs[option.name] + if option_value is not None: + options[option.name] = option_value + return options @staticmethod def matches(url): @@ -148,6 +178,32 @@ class Site: return spoiler_link +@attr.s(hash=True) +class SiteSpecificOption: + """Represents a site-specific option that can be configured. + + Will be added to the CLI as a click.option -- many of these + fields correspond to click.option arguments.""" + name = attr.ib() + flag_pattern = attr.ib() + type = attr.ib(default=None) + help = attr.ib(default=None) + default = attr.ib(default=None) + + def as_click_option(self): + return click.option( + str(self.name), + str(self.flag_pattern), + type=self.type, + # Note: This default not matching self.default is intentional. + # It ensures that we know if a flag was explicitly provided, + # which keeps it from overriding options set in leech.json etc. + # Instead, default is used in site_cls.get_default_options() + default=None, + help=self.help if self.help is not None else "" + ) + + class SiteException(Exception): pass @@ -161,10 +217,23 @@ def get(url): for site_class in _sites: match = site_class.matches(url) if match: + logger.info("Handler: %s (%s)", site_class, match) return site_class, match raise NotImplementedError("Could not find a handler for " + url) +def list_site_specific_options(): + """Returns a list of all site's click options, which will be presented to the user.""" + + # Ensures that duplicate options are not added twice. + # Especially important for subclassed sites (e.g. Xenforo sites) + options = set() + + for site_class in _sites: + options.update(site_class.get_site_specific_option_defs()) + return [option.as_click_option() for option in options] + + # And now, a particularly hacky take on a plugin system: # Make an __all__ out of all the python files in this directory that don't start # with __. Then import * them. diff --git a/sites/xenforo.py b/sites/xenforo.py index b5497c4..83cefff 100644 --- a/sites/xenforo.py +++ b/sites/xenforo.py @@ -3,7 +3,7 @@ import datetime import re import logging -from . import register, Site, SiteException, Section, Chapter +from . import register, Site, SiteException, SiteSpecificOption, Section, Chapter logger = logging.getLogger(__name__) @@ -14,13 +14,33 @@ class XenForo(Site): domain = False @staticmethod - def get_default_options(): - return { - 'offset': None, - 'limit': None, - 'skip_spoilers': True, - 'include_index': False, - } + def get_site_specific_option_defs(): + return [ + SiteSpecificOption( + 'include_index', + '--include-index/--no-include-index', + default=False, + help="If true, the post marked as an index will be included as a chapter." + ), + SiteSpecificOption( + 'skip_spoilers', + '--skip-spoilers/--include-spoilers', + default=True, + help="If true, do not transcribe any tags that are marked as a spoiler." + ), + SiteSpecificOption( + 'offset', + '--offset', + type=int, + help="The chapter index to start in the chapter marks." + ), + SiteSpecificOption( + 'limit', + '--limit', + type=int, + help="The chapter to end at at in the chapter marks." + ), + ] @classmethod def matches(cls, url):