Adds a system for site options to be included as click.options on commands.

leech.py

@@ -8,6 +8,7 @@ import requests
 import requests_cache
 import sqlite3
 from click_default_group import DefaultGroup
+from functools import reduce
 
 import sites
 import ebook
@@ -49,31 +50,43 @@ def create_session(cache):
     return session
 
 
-def open_story(url, session, site_options):
-    site, url = sites.get(url)
+def load_on_disk_options(site):
+    try:
+        with open('leech.json') as store_file:
+            store = json.load(store_file)
+            login = store.get('logins', {}).get(site.__name__, False)
+            configured_site_options = store.get('site_options', {}).get(site.__name__, {})
+    except FileNotFoundError:
+        logger.info("Unable to locate leech.json. Continuing assuming it does not exist.")
+        login = False
+        configured_site_options = {}
+    return configured_site_options, login
 
-    if not site:
-        raise Exception("No site handler found")
-
-    logger.info("Handler: %s (%s)", site, url)
 
+def create_options(site, site_options, unused_flags):
+    """Compiles options provided from multiple different sources
+    (e.g. on disk, via flags, via defaults, via JSON provided as a flag value)
+    into a single options object."""
     default_site_options = site.get_default_options()
 
-    with open('leech.json') as store_file:
-        store = json.load(store_file)
-        login = store.get('logins', {}).get(site.__name__, False)
-        configured_site_options = store.get('site_options', {}).get(site.__name__, {})
+    flag_specified_site_options = site.interpret_site_specific_options(**unused_flags)
+
+    configured_site_options, login = load_on_disk_options(site)
 
     overridden_site_options = json.loads(site_options)
 
     # The final options dictionary is computed by layering the default, configured,
-    # and overridden options together in that order.
+    # and overridden, and flag-specified options together in that order.
     options = dict(
         list(default_site_options.items()) +
         list(configured_site_options.items()) +
-        list(overridden_site_options.items())
+        list(overridden_site_options.items()) +
+        list(flag_specified_site_options.items())
     )
+    return options, login
 
+
+def open_story(site, url, session, login, options):
     handler = site(
         session,
         options=options
@@ -88,6 +101,11 @@ def open_story(url, session, site_options):
     return story
 
 
+def site_specific_options(f):
+    option_list = sites.list_site_specific_options()
+    return reduce(lambda cmd, decorator: decorator(cmd), [f] + option_list)
+
+
 @click.group(cls=DefaultGroup, default='download', default_if_no_args=True)
 def cli():
     """Top level click group. Uses click-default-group to preserve most behavior from leech v1."""
@@ -118,11 +136,16 @@ def flush(verbose):
 )
 @click.option('--cache/--no-cache', default=True)
 @click.option('--verbose', '-v', is_flag=True, help="Verbose debugging output")
-def download(url, site_options, cache, verbose):
+@site_specific_options  # Includes other click.options specific to sites
+def download(url, site_options, cache, verbose, **other_flags):
     """Downloads a story and saves it on disk as a ebpub ebook."""
     configure_logging(verbose)
     session = create_session(cache)
-    story = open_story(url, session, site_options)
+
+    site, url = sites.get(url)
+    options, login = create_options(site, site_options, other_flags)
+    story = open_story(site, url, session, login, options)
+
     filename = ebook.generate_epub(story)
     logger.info("File created: " + filename)
 

sites/__init__.py

@@ -1,4 +1,5 @@
 
+import click
 import glob
 import os
 import uuid
@@ -72,8 +73,37 @@ class Site:
     ))
 
     @staticmethod
-    def get_default_options():
-        return {}
+    def get_site_specific_option_defs():
+        """Returns a list of click.option objects to add to CLI commands.
+
+        It is best practice to ensure that these names are reasonably unique
+        to ensure that they do not conflict with the core options, or other
+        sites' options. It is OK for different site's options to have the
+        same name, but pains should be taken to ensure they remain semantically
+        similar in meaning.
+        """
+        return []
+
+    @classmethod
+    def get_default_options(cls):
+        options = {}
+        for option in cls.get_site_specific_option_defs():
+            options[option.name] = option.default
+        return options
+
+    @classmethod
+    def interpret_site_specific_options(cls, **kwargs):
+        """Returns options summarizing CLI flags provided.
+
+        Only includes entries the user has explicitly provided as flags
+        / will not contain default values. For that, use get_default_options().
+        """
+        options = {}
+        for option in cls.get_site_specific_option_defs():
+            option_value = kwargs[option.name]
+            if option_value is not None:
+                options[option.name] = option_value
+        return options
 
     @staticmethod
     def matches(url):
@@ -148,6 +178,32 @@ class Site:
         return spoiler_link
 
 
+@attr.s(hash=True)
+class SiteSpecificOption:
+    """Represents a site-specific option that can be configured.
+
+    Will be added to the CLI as a click.option -- many of these
+    fields correspond to click.option arguments."""
+    name = attr.ib()
+    flag_pattern = attr.ib()
+    type = attr.ib(default=None)
+    help = attr.ib(default=None)
+    default = attr.ib(default=None)
+
+    def as_click_option(self):
+        return click.option(
+            str(self.name),
+            str(self.flag_pattern),
+            type=self.type,
+            # Note: This default not matching self.default is intentional.
+            # It ensures that we know if a flag was explicitly provided,
+            # which keeps it from overriding options set in leech.json etc.
+            # Instead, default is used in site_cls.get_default_options()
+            default=None,
+            help=self.help if self.help is not None else ""
+        )
+
+
 class SiteException(Exception):
     pass
 
@@ -161,10 +217,23 @@ def get(url):
     for site_class in _sites:
         match = site_class.matches(url)
         if match:
+            logger.info("Handler: %s (%s)", site_class, match)
             return site_class, match
     raise NotImplementedError("Could not find a handler for " + url)
 
 
+def list_site_specific_options():
+    """Returns a list of all site's click options, which will be presented to the user."""
+
+    # Ensures that duplicate options are not added twice.
+    # Especially important for subclassed sites (e.g. Xenforo sites)
+    options = set()
+
+    for site_class in _sites:
+        options.update(site_class.get_site_specific_option_defs())
+    return [option.as_click_option() for option in options]
+
+
 # And now, a particularly hacky take on a plugin system:
 # Make an __all__ out of all the python files in this directory that don't start
 # with __. Then import * them.

sites/xenforo.py

@@ -3,7 +3,7 @@
 import datetime
 import re
 import logging
-from . import register, Site, SiteException, Section, Chapter
+from . import register, Site, SiteException, SiteSpecificOption, Section, Chapter
 
 logger = logging.getLogger(__name__)
 
@@ -14,13 +14,33 @@ class XenForo(Site):
     domain = False
 
     @staticmethod
-    def get_default_options():
-        return {
-            'offset': None,
-            'limit': None,
-            'skip_spoilers': True,
-            'include_index': False,
-        }
+    def get_site_specific_option_defs():
+        return [
+            SiteSpecificOption(
+                'include_index',
+                '--include-index/--no-include-index',
+                default=False,
+                help="If true, the post marked as an index will be included as a chapter."
+            ),
+            SiteSpecificOption(
+                'skip_spoilers',
+                '--skip-spoilers/--include-spoilers',
+                default=True,
+                help="If true, do not transcribe any tags that are marked as a spoiler."
+            ),
+            SiteSpecificOption(
+                'offset',
+                '--offset',
+                type=int,
+                help="The chapter index to start in the chapter marks."
+            ),
+            SiteSpecificOption(
+                'limit',
+                '--limit',
+                type=int,
+                help="The chapter to end at at in the chapter marks."
+            ),
+        ]
 
     @classmethod
     def matches(cls, url):