configurable pathname substitution (#115)

beets/importer.py

@@ -77,6 +77,8 @@ def _reopen_lib(lib):
             lib.directory,
             lib.path_formats,
             lib.art_filename,
+            lib.timeout,
+            lib.replacements,
         )
     else:
         return lib

beets/library.py

@@ -701,6 +701,7 @@ class Library(BaseLibrary):
                        path_formats=None,
                        art_filename='cover',
                        timeout=5.0,
+                       replacements=None,
                        item_fields=ITEM_FIELDS,
                        album_fields=ALBUM_FIELDS):
         if path == ':memory:':
@@ -714,6 +715,7 @@ class Library(BaseLibrary):
             path_formats = {'default': path_formats}
         self.path_formats = path_formats
         self.art_filename = bytestring_path(art_filename)
+        self.replacements = replacements
         
         self.timeout = timeout
         self.conn = sqlite3.connect(self.path, timeout)
@@ -828,7 +830,7 @@ class Library(BaseLibrary):
             subpath = subpath.encode(encoding, 'replace')
         
         # Truncate components and remove forbidden characters.
-        subpath = util.sanitize_path(subpath)
+        subpath = util.sanitize_path(subpath, pathmod, self.replacements)
         
         # Preserve extension.
         _, extension = pathmod.splitext(item.path)

beets/ui/__init__.py

@@ -26,6 +26,7 @@ from difflib import SequenceMatcher
 import logging
 import sqlite3
 import errno
+import re
 
 from beets import library
 from beets import plugins
@@ -404,6 +405,30 @@ def default_paths(pathmod=None):
 
     return config, libpath, libdir
 
+def _get_replacements(config):
+    """Given a ConfigParser, get the list of replacement pairs. If no
+    replacements are specified, returns None. Otherwise, returns a list
+    of (compiled regex, replacement string) pairs.
+    """
+    repl_string = config_val(config, 'beets', 'replace', None)
+    if not repl_string:
+        return
+
+    parts = repl_string.strip().split()
+    if not parts:
+        return
+    if len(parts) % 2 != 0:
+        # Must have an even number of parts.
+        raise UserError(u'"replace" config value must consist of'
+                        u' pattern/replacement pairs')
+
+    out = []
+    for index in xrange(0, len(parts), 2):
+        pattern = parts[index]
+        replacement = parts[index+1]
+        out.append((re.compile(pattern), replacement))
+    return out
+
 
 # Subcommand parsing infrastructure.
 
@@ -635,6 +660,7 @@ def main(args=None, configfh=None):
     art_filename = \
         config_val(config, 'beets', 'art_filename', DEFAULT_ART_FILENAME)
     lib_timeout = config_val(config, 'beets', 'timeout', DEFAULT_TIMEOUT)
+    replacements = _get_replacements(config)
     try:
         lib_timeout = float(lib_timeout)
     except ValueError:
@@ -645,7 +671,8 @@ def main(args=None, configfh=None):
                               directory,
                               path_formats,
                               art_filename,
-                              lib_timeout)
+                              lib_timeout,
+                              replacements)
     except sqlite3.OperationalError:
         raise UserError("database file %s could not be opened" % db_path)
     

beets/util/__init__.py

@@ -251,25 +251,33 @@ CHAR_REPLACE = [
     (re.compile(r'[\\/\?"]|^\.'), '_'),
     (re.compile(r':'), '-'),
 ]
-CHAR_REPLACE_WINDOWS = re.compile(r'["\*<>\|]|^\.|\.$| +$'), '_'
-def sanitize_path(path, pathmod=None):
+CHAR_REPLACE_WINDOWS = [
+    (re.compile(r'["\*<>\|]|^\.|\.$|\s+$'), '_'),
+]
+def sanitize_path(path, pathmod=None, replacements=None):
     """Takes a path and makes sure that it is legal. Returns a new path.
     Only works with fragments; won't work reliably on Windows when a
     path begins with a drive letter. Path separators (including altsep!)
-    should already be cleaned from the path components.
+    should already be cleaned from the path components. If replacements
+    is specified, it is used *instead* of the default set of
+    replacements for the platform; it must be a list of (compiled regex,
+    replacement string) pairs.
     """
     pathmod = pathmod or os.path
     windows = pathmod.__name__ == 'ntpath'
+
+    # Choose the appropriate replacements.
+    if not replacements:
+        replacements = list(CHAR_REPLACE)
+        if windows:
+            replacements += CHAR_REPLACE_WINDOWS
     
     comps = components(path, pathmod)
     if not comps:
         return ''
     for i, comp in enumerate(comps):
         # Replace special characters.
-        for regex, repl in CHAR_REPLACE:
-            comp = regex.sub(repl, comp)
-        if windows:
-            regex, repl = CHAR_REPLACE_WINDOWS
+        for regex, repl in replacements:
             comp = regex.sub(repl, comp)
         
         # Truncate each component.

docs/changelog.rst

@@ -12,6 +12,9 @@ This release focuses on making beets' path formatting vastly more powerful.
   :doc:`/reference/pathformat`. If you're interested in adding your own template
   functions via a plugin, see :ref:`writing-plugins`.
 * Plugins can also now define new path *fields* in addition to functions.
+* **Filename substitutions are now configurable** via the ``replace`` config
+  value. You can choose which characters you think should be allowed in your
+  directory and music file names.  See :doc:`/reference/config`.
 * Fix an incompatibility in BPD with libmpc (the library that powers mpc and
   ncmpc).
 * Fix a crash when importing a partial match whose first track was missing.

docs/reference/config.rst

@@ -73,6 +73,33 @@ section header:
     A space-separated list of glob patterns specifying file and directory names
     to be ignored when importing. Defaults to ``.AppleDouble ._* *~ .DS_Store``.
 
+``replace``
+    A set of regular expression/replacement pairs to be applied to all filenames
+    created by beets. Typically, these replacements are used to avoid confusing
+    problems or errors with the filesystem (for example, leading ``.``
+    characters are replaced on Unix and the ``*<>|`` characters are removed on
+    Windows). To override these substitutions, specify a sequence of
+    whitespace-separated terms; the first term is a regular expression and the
+    second is a string that should replace anything matching that regex. For
+    example, ``replace = [xy] z`` will make beets replace all instances of the
+    characters ``x`` or ``y`` with the character ``z``.
+
+    If you do change this value, be certain that you include at least enough
+    substitutions to avoid causing errors on your operating system. Here are
+    some recommended base replacements for Unix-like OSes::
+
+        replace = [\\/\?"]|^\.' _
+                  : -
+
+    And, on Windows::
+
+        replace = [\\/\?"]|^\.' _
+                  ["\*<>\|]|^\.|\.$|\s+$ _
+                  : -
+
+    Note that the above examples are, in fact, the default substitutions used by
+    beets.
+
 ``art_filename``
     When importing album art, the name of the file (without extension) where the
     cover art image should be placed. Defaults to ``cover`` (i.e., images will

test/test_db.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2010, Adrian Sampson.
+# Copyright 2011, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -14,13 +14,13 @@
 
 """Tests for non-query database functions of Item.
 """
-
 import unittest
 import os
 import sqlite3
 import ntpath
 import posixpath
 import shutil
+import re
 
 import _common
 from _common import item
@@ -231,7 +231,7 @@ class DestinationTest(unittest.TestCase):
         self.assertFalse('<' in p)
         self.assertFalse('>' in p)
         self.assertFalse('|' in p)
-    
+
     def test_sanitize_replaces_colon_with_dash(self):
         p = util.sanitize_path(u':', posixpath)
         self.assertEqual(p, u'-')
@@ -363,6 +363,18 @@ class DestinationTest(unittest.TestCase):
         p = util.sanitize_path('', posixpath)
         self.assertEqual(p, '')
 
+    def test_sanitize_with_custom_replace_overrides_built_in_sub(self):
+        p = util.sanitize_path('a/.?/b', posixpath, [
+            (re.compile(r'foo'), 'bar'),
+        ])
+        self.assertEqual(p, 'a/.?/b')
+
+    def test_sanitize_with_custom_replace_adds_replacements(self):
+        p = util.sanitize_path('foo/bar', posixpath, [
+            (re.compile(r'foo'), 'bar'),
+        ])
+        self.assertEqual(p, 'bar/bar')
+
 class DestinationFunctionTest(unittest.TestCase):
     def setUp(self):
         self.lib = beets.library.Library(':memory:')

test/test_ui.py

@@ -19,8 +19,10 @@ import os
 import shutil
 import textwrap
 import logging
+import re
 from StringIO import StringIO
 
+import _common
 from beets import library
 from beets import ui
 from beets.ui import commands
@@ -28,8 +30,6 @@ from beets import autotag
 from beets import importer
 from beets.mediafile import MediaFile
 
-import _common
-
 class ListTest(unittest.TestCase):
     def setUp(self):
         self.io = _common.DummyIO()
@@ -499,14 +499,6 @@ class ConfigTest(unittest.TestCase):
             [beets]
             path_format=x"""), func)
 
-    def test_paths_section_overriden_by_cli_switch(self):
-        def func(lib, config, opts, args):
-            self.assertEqual(lib.path_formats['default'], 'z')
-            self.assertEqual(len(lib.path_formats), 1)
-        self._run_main(['-p', 'z'], textwrap.dedent("""
-            [paths]
-            x=y"""), func)
-
     def test_nonexistant_config_file(self):
         os.environ['BEETSCONFIG'] = '/xxxxx'
         ui.main(['version'])
@@ -520,6 +512,34 @@ class ConfigTest(unittest.TestCase):
                 library: /xxx/yyy/not/a/real/path
             """), func)
 
+    def test_replacements_parsed(self):
+        def func(lib, config, opts, args):
+            replacements = lib.replacements
+            self.assertEqual(replacements, [(re.compile(r'[xy]'), 'z')])
+        self._run_main([], textwrap.dedent("""
+            [beets]
+            replace=[xy] z"""), func)
+
+    def test_empty_replacements_produce_none(self):
+        def func(lib, config, opts, args):
+            replacements = lib.replacements
+            self.assertFalse(replacements)
+        self._run_main([], textwrap.dedent("""
+            [beets]
+            """), func)
+
+    def test_multiple_replacements_parsed(self):
+        def func(lib, config, opts, args):
+            replacements = lib.replacements
+            self.assertEqual(replacements, [
+                (re.compile(r'[xy]'), 'z'),
+                (re.compile(r'foo'), 'bar'),
+            ])
+        self._run_main([], textwrap.dedent("""
+            [beets]
+            replace=[xy] z
+                foo bar"""), func)
+
 class ShowdiffTest(unittest.TestCase):
     def setUp(self):
         self.io = _common.DummyIO()