-i/import_incremental to only import new directories (#99)

NEWS

@@ -17,6 +17,11 @@
 * Relatedly, the -L flag to the "import" command makes it take a query
   as its argument instead of a list of directories. The matched albums
   (or items, depending on the -s flag) are then re-imported.
+* A new flag -i to the import command runs incremental imports, keeping
+  track of and skipping previously-imported directories. This has the
+  effect of making repeated import commands pick up only newly-added
+  directories. The "import_incremental" config option makes this the
+  default.
 * When pruning directories, "clutter" files such as .DS_Store and
   Thumbs.db are ignored (and removed with otherwise-empty
   directories).

beets/importer.py

@@ -187,6 +187,18 @@ def _infer_album_fields(task):
         for k, v in changes.iteritems():
             setattr(item, k, v)
 
+def _open_state():
+    """Reads the state file, returning a dictionary."""
+    try:
+        with open(STATE_FILE) as f:
+            return pickle.load(f)
+    except IOError:
+        return {}
+def _save_state(state):
+    """Writes the state dictionary out to disk."""
+    with open(STATE_FILE, 'w') as f:
+        pickle.dump(state, f)
+
 
 # Utilities for reading and writing the beets progress file, which
 # allows long tagging tasks to be resumed when they pause (or crash).
@@ -196,11 +208,9 @@ def progress_set(toppath, path):
     `path`. If path is None, then clear the progress value (indicating
     that the tagging completed).
     """
-    try:
-        with open(STATE_FILE) as f:
-            state = pickle.load(f)
-    except IOError:
-        state = {PROGRESS_KEY: {}}
+    state = _open_state()
+    if PROGRESS_KEY not in state:
+        state[PROGRESS_KEY] = {}
 
     if path is None:
         # Remove progress from file.
@@ -209,20 +219,41 @@ def progress_set(toppath, path):
     else:
         state[PROGRESS_KEY][toppath] = path
 
-    with open(STATE_FILE, 'w') as f:
-        pickle.dump(state, f)
+    _save_state(state)
 def progress_get(toppath):
     """Get the last successfully tagged subpath of toppath. If toppath
     has no progress information, returns None.
     """
-    try:
-        with open(STATE_FILE) as f:
-            state = pickle.load(f)
-    except IOError:
+    state = _open_state()
+    if PROGRESS_KEY not in state:
         return None
     return state[PROGRESS_KEY].get(toppath)
 
 
+# Similarly, utilities for manipulating the "incremental" import log.
+# This keeps track of all directories that were ever imported, which
+# allows the importer to only import new stuff.
+HISTORY_KEY = 'taghistory'
+def history_add(path):
+    """Indicate that the import of `path` is completed and should not
+    be repeated in incremental imports.
+    """
+    state = _open_state()
+    if HISTORY_KEY not in state:
+        state[HISTORY_KEY] = set()
+
+    state[HISTORY_KEY].add(path)
+
+    _save_state(state)
+def history_get():
+    """Get the set of completed paths in incremental imports.
+    """
+    state = _open_state()
+    if HISTORY_KEY not in state:
+        return set()
+    return state[HISTORY_KEY]
+
+
 # The configuration structure.
 
 class ImportConfig(object):
@@ -234,7 +265,7 @@ class ImportConfig(object):
                'quiet_fallback', 'copy', 'write', 'art', 'delete',
                'choose_match_func', 'should_resume_func', 'threaded',
                'autot', 'singletons', 'timid', 'choose_item_func',
-               'query']
+               'query', 'incremental']
     def __init__(self, **kwargs):
         for slot in self._fields:
             setattr(self, slot, kwargs[slot])
@@ -243,11 +274,16 @@ class ImportConfig(object):
         if self.paths:
             self.paths = map(normpath, self.paths)
 
+        # Incremental and progress are mutually exclusive.
+        if self.incremental:
+            self.resume = False
+
         # When based on a query instead of directories, never
         # save progress or try to resume.
         if self.query is not None:
             self.paths = None
             self.resume = False
+            self.incremental = False
 
 
 # The importer task class.
@@ -352,6 +388,12 @@ class ImportTask(object):
             # album task, which implies the same.
             progress_set(self.toppath, self.path)
 
+    def save_history(self):
+        """Save the directory in the history for incremental imports.
+        """
+        if self.sentinel or self.is_album:
+            history_add(self.path)
+
     # Logical decisions.
     def should_write_tags(self):
         """Should new info be written to the files' metadata?"""
@@ -398,6 +440,10 @@ def read_tasks(config):
                 else:
                     # Clear progress; we're starting from the top.
                     progress_set(path, None)
+
+    # Look for saved incremental directories.
+    if config.incremental:
+        history_dirs = history_get()
     
     for toppath in config.paths:
         # Check whether the path is to a file.
@@ -410,6 +456,7 @@ def read_tasks(config):
         if progress:
             resume_dir = resume_dirs.get(toppath)
         for path, items in autotag.albums_in_dir(toppath):
+            # Skip according to progress.
             if progress and resume_dir:
                 # We're fast-forwarding to resume a previous tagging.
                 if path == resume_dir:
@@ -418,6 +465,10 @@ def read_tasks(config):
                     resume_dir = None
                 continue
 
+            # When incremental, skip paths in the history.
+            if config.incremental and path in history_dirs:
+                continue
+
             # Yield all the necessary tasks.
             if config.singletons:
                 for item in items:
@@ -634,6 +685,8 @@ def finalize(config):
         if task.should_skip():
             if config.resume is not False:
                 task.save_progress()
+            if config.incremental:
+                task.save_history()
             continue
 
         items = task.items if task.is_album else [task.item]
@@ -657,6 +710,8 @@ def finalize(config):
         # Update progress.
         if config.resume is not False:
             task.save_progress()
+        if config.incremental:
+            task.save_history()
 
 
 # Singleton pipeline stages.

beets/ui/commands.py

@@ -84,6 +84,7 @@ DEFAULT_IMPORT_ART            = True
 DEFAULT_IMPORT_QUIET          = False
 DEFAULT_IMPORT_QUIET_FALLBACK = 'skip'
 DEFAULT_IMPORT_RESUME         = None # "ask"
+DEFAULT_IMPORT_INCREMENTAL    = False
 DEFAULT_THREADED              = True
 DEFAULT_COLOR                 = True
 
@@ -483,7 +484,7 @@ def choose_item(task, config):
 
 def import_files(lib, paths, copy, write, autot, logpath, art, threaded,
                  color, delete, quiet, resume, quiet_fallback, singletons,
-                 timid, query):
+                 timid, query, incremental):
     """Import the files in the given list of paths, tagging each leaf
     directory as an album. If copy, then the files are copied into
     the library folder. If write, then new metadata is written to the
@@ -544,6 +545,7 @@ def import_files(lib, paths, copy, write, autot, logpath, art, threaded,
         timid = timid,
         choose_item_func = choose_item,
         query = query,
+        incremental = incremental,
     )
     
     # If we were logging, close the file.
@@ -587,6 +589,8 @@ import_cmd.parser.add_option('-t', '--timid', dest='timid',
     action='store_true', help='always confirm all actions')
 import_cmd.parser.add_option('-L', '--library', dest='library',
     action='store_true', help='retag items matching a query')
+import_cmd.parser.add_option('-i', '--incremental', dest='incremental',
+    action='store_true', help='skip already-imported directories')
 def import_func(lib, config, opts, args):
     copy  = opts.copy  if opts.copy  is not None else \
         ui.config_val(config, 'beets', 'import_copy',
@@ -612,6 +616,9 @@ def import_func(lib, config, opts, args):
             DEFAULT_IMPORT_TIMID, bool)
     logpath = opts.logpath if opts.logpath is not None else \
         ui.config_val(config, 'beets', 'import_log', None)
+    incremental = opts.incremental if opts.incremental is not None else \
+        ui.config_val(config, 'beets', 'import_incremental',
+            DEFAULT_IMPORT_INCREMENTAL, bool)
 
     # Resume has three options: yes, no, and "ask" (None).
     resume = opts.resume if opts.resume is not None else \
@@ -638,7 +645,7 @@ def import_func(lib, config, opts, args):
 
     import_files(lib, paths, copy, write, autot, logpath, art, threaded,
                  color, delete, quiet, resume, quiet_fallback, singletons,
-                 timid, query)
+                 timid, query, incremental)
 import_cmd.func = import_func
 default_commands.append(import_cmd)
 

test/_common.py

@@ -87,6 +87,7 @@ def iconfig(lib, **kwargs):
         choose_item_func = lambda x, y: importer.action.SKIP,
         timid = False,
         query = None,
+        incremental = False,
     )
     for k, v in kwargs.items():
         setattr(config, k, v)

test/test_importer.py

@@ -101,6 +101,7 @@ class NonAutotaggedImportTest(unittest.TestCase):
                 choose_item_func = None,
                 timid = False,
                 query = None,
+                incremental = False,
         )
 
         return paths

test/test_ui.py

@@ -428,7 +428,8 @@ class ImportTest(unittest.TestCase):
     def test_quiet_timid_disallowed(self):
         self.assertRaises(ui.UserError, commands.import_files,
                           None, [], False, False, False, None, False, False,
-                          False, False, True, False, None, False, True, None)
+                          False, False, True, False, None, False, True, None,
+                          False)
 
 class InputTest(unittest.TestCase):
     def setUp(self):