From b493bc70049ded8e8d990bd1d6100a60697aa935 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 19 Dec 2011 18:37:35 -0800 Subject: [PATCH] configurable pathname substitution (#115) --- beets/importer.py | 2 ++ beets/library.py | 4 +++- beets/ui/__init__.py | 29 +++++++++++++++++++++++++++- beets/util/__init__.py | 22 ++++++++++++++------- docs/changelog.rst | 3 +++ docs/reference/config.rst | 27 ++++++++++++++++++++++++++ test/test_db.py | 18 +++++++++++++++--- test/test_ui.py | 40 +++++++++++++++++++++++++++++---------- 8 files changed, 123 insertions(+), 22 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index 0d076f5ba..8afcaf816 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -77,6 +77,8 @@ def _reopen_lib(lib): lib.directory, lib.path_formats, lib.art_filename, + lib.timeout, + lib.replacements, ) else: return lib diff --git a/beets/library.py b/beets/library.py index 2dc22ce55..b75870e09 100644 --- a/beets/library.py +++ b/beets/library.py @@ -701,6 +701,7 @@ class Library(BaseLibrary): path_formats=None, art_filename='cover', timeout=5.0, + replacements=None, item_fields=ITEM_FIELDS, album_fields=ALBUM_FIELDS): if path == ':memory:': @@ -714,6 +715,7 @@ class Library(BaseLibrary): path_formats = {'default': path_formats} self.path_formats = path_formats self.art_filename = bytestring_path(art_filename) + self.replacements = replacements self.timeout = timeout self.conn = sqlite3.connect(self.path, timeout) @@ -828,7 +830,7 @@ class Library(BaseLibrary): subpath = subpath.encode(encoding, 'replace') # Truncate components and remove forbidden characters. - subpath = util.sanitize_path(subpath) + subpath = util.sanitize_path(subpath, pathmod, self.replacements) # Preserve extension. _, extension = pathmod.splitext(item.path) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 8b5fd931e..30522fe9d 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -26,6 +26,7 @@ from difflib import SequenceMatcher import logging import sqlite3 import errno +import re from beets import library from beets import plugins @@ -404,6 +405,30 @@ def default_paths(pathmod=None): return config, libpath, libdir +def _get_replacements(config): + """Given a ConfigParser, get the list of replacement pairs. If no + replacements are specified, returns None. Otherwise, returns a list + of (compiled regex, replacement string) pairs. + """ + repl_string = config_val(config, 'beets', 'replace', None) + if not repl_string: + return + + parts = repl_string.strip().split() + if not parts: + return + if len(parts) % 2 != 0: + # Must have an even number of parts. + raise UserError(u'"replace" config value must consist of' + u' pattern/replacement pairs') + + out = [] + for index in xrange(0, len(parts), 2): + pattern = parts[index] + replacement = parts[index+1] + out.append((re.compile(pattern), replacement)) + return out + # Subcommand parsing infrastructure. @@ -635,6 +660,7 @@ def main(args=None, configfh=None): art_filename = \ config_val(config, 'beets', 'art_filename', DEFAULT_ART_FILENAME) lib_timeout = config_val(config, 'beets', 'timeout', DEFAULT_TIMEOUT) + replacements = _get_replacements(config) try: lib_timeout = float(lib_timeout) except ValueError: @@ -645,7 +671,8 @@ def main(args=None, configfh=None): directory, path_formats, art_filename, - lib_timeout) + lib_timeout, + replacements) except sqlite3.OperationalError: raise UserError("database file %s could not be opened" % db_path) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index ea5d4a692..a70148ca1 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -251,25 +251,33 @@ CHAR_REPLACE = [ (re.compile(r'[\\/\?"]|^\.'), '_'), (re.compile(r':'), '-'), ] -CHAR_REPLACE_WINDOWS = re.compile(r'["\*<>\|]|^\.|\.$| +$'), '_' -def sanitize_path(path, pathmod=None): +CHAR_REPLACE_WINDOWS = [ + (re.compile(r'["\*<>\|]|^\.|\.$|\s+$'), '_'), +] +def sanitize_path(path, pathmod=None, replacements=None): """Takes a path and makes sure that it is legal. Returns a new path. Only works with fragments; won't work reliably on Windows when a path begins with a drive letter. Path separators (including altsep!) - should already be cleaned from the path components. + should already be cleaned from the path components. If replacements + is specified, it is used *instead* of the default set of + replacements for the platform; it must be a list of (compiled regex, + replacement string) pairs. """ pathmod = pathmod or os.path windows = pathmod.__name__ == 'ntpath' + + # Choose the appropriate replacements. + if not replacements: + replacements = list(CHAR_REPLACE) + if windows: + replacements += CHAR_REPLACE_WINDOWS comps = components(path, pathmod) if not comps: return '' for i, comp in enumerate(comps): # Replace special characters. - for regex, repl in CHAR_REPLACE: - comp = regex.sub(repl, comp) - if windows: - regex, repl = CHAR_REPLACE_WINDOWS + for regex, repl in replacements: comp = regex.sub(repl, comp) # Truncate each component. diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ae18bc94..40ba8a9e7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,9 @@ This release focuses on making beets' path formatting vastly more powerful. :doc:`/reference/pathformat`. If you're interested in adding your own template functions via a plugin, see :ref:`writing-plugins`. * Plugins can also now define new path *fields* in addition to functions. +* **Filename substitutions are now configurable** via the ``replace`` config + value. You can choose which characters you think should be allowed in your + directory and music file names. See :doc:`/reference/config`. * Fix an incompatibility in BPD with libmpc (the library that powers mpc and ncmpc). * Fix a crash when importing a partial match whose first track was missing. diff --git a/docs/reference/config.rst b/docs/reference/config.rst index b1a5a98a8..8caf0be68 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -73,6 +73,33 @@ section header: A space-separated list of glob patterns specifying file and directory names to be ignored when importing. Defaults to ``.AppleDouble ._* *~ .DS_Store``. +``replace`` + A set of regular expression/replacement pairs to be applied to all filenames + created by beets. Typically, these replacements are used to avoid confusing + problems or errors with the filesystem (for example, leading ``.`` + characters are replaced on Unix and the ``*<>|`` characters are removed on + Windows). To override these substitutions, specify a sequence of + whitespace-separated terms; the first term is a regular expression and the + second is a string that should replace anything matching that regex. For + example, ``replace = [xy] z`` will make beets replace all instances of the + characters ``x`` or ``y`` with the character ``z``. + + If you do change this value, be certain that you include at least enough + substitutions to avoid causing errors on your operating system. Here are + some recommended base replacements for Unix-like OSes:: + + replace = [\\/\?"]|^\.' _ + : - + + And, on Windows:: + + replace = [\\/\?"]|^\.' _ + ["\*<>\|]|^\.|\.$|\s+$ _ + : - + + Note that the above examples are, in fact, the default substitutions used by + beets. + ``art_filename`` When importing album art, the name of the file (without extension) where the cover art image should be placed. Defaults to ``cover`` (i.e., images will diff --git a/test/test_db.py b/test/test_db.py index 863dee49c..68eaf9175 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,5 +1,5 @@ # This file is part of beets. -# Copyright 2010, Adrian Sampson. +# Copyright 2011, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -14,13 +14,13 @@ """Tests for non-query database functions of Item. """ - import unittest import os import sqlite3 import ntpath import posixpath import shutil +import re import _common from _common import item @@ -231,7 +231,7 @@ class DestinationTest(unittest.TestCase): self.assertFalse('<' in p) self.assertFalse('>' in p) self.assertFalse('|' in p) - + def test_sanitize_replaces_colon_with_dash(self): p = util.sanitize_path(u':', posixpath) self.assertEqual(p, u'-') @@ -363,6 +363,18 @@ class DestinationTest(unittest.TestCase): p = util.sanitize_path('', posixpath) self.assertEqual(p, '') + def test_sanitize_with_custom_replace_overrides_built_in_sub(self): + p = util.sanitize_path('a/.?/b', posixpath, [ + (re.compile(r'foo'), 'bar'), + ]) + self.assertEqual(p, 'a/.?/b') + + def test_sanitize_with_custom_replace_adds_replacements(self): + p = util.sanitize_path('foo/bar', posixpath, [ + (re.compile(r'foo'), 'bar'), + ]) + self.assertEqual(p, 'bar/bar') + class DestinationFunctionTest(unittest.TestCase): def setUp(self): self.lib = beets.library.Library(':memory:') diff --git a/test/test_ui.py b/test/test_ui.py index 80b5bce56..23db3100e 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -19,8 +19,10 @@ import os import shutil import textwrap import logging +import re from StringIO import StringIO +import _common from beets import library from beets import ui from beets.ui import commands @@ -28,8 +30,6 @@ from beets import autotag from beets import importer from beets.mediafile import MediaFile -import _common - class ListTest(unittest.TestCase): def setUp(self): self.io = _common.DummyIO() @@ -499,14 +499,6 @@ class ConfigTest(unittest.TestCase): [beets] path_format=x"""), func) - def test_paths_section_overriden_by_cli_switch(self): - def func(lib, config, opts, args): - self.assertEqual(lib.path_formats['default'], 'z') - self.assertEqual(len(lib.path_formats), 1) - self._run_main(['-p', 'z'], textwrap.dedent(""" - [paths] - x=y"""), func) - def test_nonexistant_config_file(self): os.environ['BEETSCONFIG'] = '/xxxxx' ui.main(['version']) @@ -520,6 +512,34 @@ class ConfigTest(unittest.TestCase): library: /xxx/yyy/not/a/real/path """), func) + def test_replacements_parsed(self): + def func(lib, config, opts, args): + replacements = lib.replacements + self.assertEqual(replacements, [(re.compile(r'[xy]'), 'z')]) + self._run_main([], textwrap.dedent(""" + [beets] + replace=[xy] z"""), func) + + def test_empty_replacements_produce_none(self): + def func(lib, config, opts, args): + replacements = lib.replacements + self.assertFalse(replacements) + self._run_main([], textwrap.dedent(""" + [beets] + """), func) + + def test_multiple_replacements_parsed(self): + def func(lib, config, opts, args): + replacements = lib.replacements + self.assertEqual(replacements, [ + (re.compile(r'[xy]'), 'z'), + (re.compile(r'foo'), 'bar'), + ]) + self._run_main([], textwrap.dedent(""" + [beets] + replace=[xy] z + foo bar"""), func) + class ShowdiffTest(unittest.TestCase): def setUp(self): self.io = _common.DummyIO()