diff --git a/beets/library.py b/beets/library.py index 05ef98f48..bd88136f1 100644 --- a/beets/library.py +++ b/beets/library.py @@ -24,6 +24,7 @@ import unicodedata import time import re from unidecode import unidecode +import platform from beets import logging from beets.mediafile import MediaFile, MutagenError, UnreadableFileError @@ -42,30 +43,55 @@ log = logging.getLogger('beets') # Library-specific query types. class PathQuery(dbcore.FieldQuery): - """A query that matches all items under a given path.""" + """A query that matches all items under a given path. + + Matching can either be case-insensitive or case-sensitive. By + default, the behavior depends on the OS: case-insensitive on Windows + and case-sensitive otherwise. + """ escape_re = re.compile(r'[\\_%]') escape_char = b'\\' - def __init__(self, field, pattern, fast=True): + def __init__(self, field, pattern, fast=True, case_sensitive=None): + """Create a path query. + + `case_sensitive` can be a bool or `None`, indicating that the + behavior should depend on the platform (the default). + """ super(PathQuery, self).__init__(field, pattern, fast) + # By default, the case sensitivity depends on the platform. + if case_sensitive is None: + case_sensitive = platform.system() != 'Windows' + self.case_sensitive = case_sensitive + + # Use a normalized-case pattern for case-insensitive matches. + if not case_sensitive: + pattern = pattern.lower() + # Match the path as a single file. self.file_path = util.bytestring_path(util.normpath(pattern)) # As a directory (prefix). self.dir_path = util.bytestring_path(os.path.join(self.file_path, b'')) def match(self, item): - return (item.path == self.file_path) or \ - item.path.startswith(self.dir_path) + path = item.path if self.case_sensitive else item.path.lower() + return (path == self.file_path) or path.startswith(self.dir_path) def col_clause(self): + file_blob = buffer(self.file_path) + + if self.case_sensitive: + dir_blob = buffer(self.dir_path) + return '({0} = ?) || (substr({0}, 1, ?) = ?)'.format(self.field), \ + (file_blob, len(dir_blob), dir_blob) + escape = lambda m: self.escape_char + m.group(0) dir_pattern = self.escape_re.sub(escape, self.dir_path) - dir_pattern = buffer(dir_pattern + b'%') - file_blob = buffer(self.file_path) + dir_blob = buffer(dir_pattern + b'%') return '({0} = ?) || ({0} LIKE ? ESCAPE ?)'.format(self.field), \ - (file_blob, dir_pattern, self.escape_char) + (file_blob, dir_blob, self.escape_char) # Library-specific field types. @@ -1092,11 +1118,12 @@ def parse_query_string(s, model_cls): The string is split into components using shell-like syntax. """ + assert isinstance(s, unicode), "Query is not unicode: {0!r}".format(s) + # A bug in Python < 2.7.3 prevents correct shlex splitting of # Unicode strings. # http://bugs.python.org/issue6988 - if isinstance(s, unicode): - s = s.encode('utf8') + s = s.encode('utf8') try: parts = [p.decode('utf8') for p in shlex.split(s)] except ValueError as exc: diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 2d6968e13..53156143f 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -624,7 +624,7 @@ def cpu_count(): num = 0 elif sys.platform == b'darwin': try: - num = int(command_output(['sysctl', '-n', 'hw.ncpu'])) + num = int(command_output([b'sysctl', b'-n', b'hw.ncpu'])) except ValueError: num = 0 else: @@ -641,8 +641,8 @@ def cpu_count(): def command_output(cmd, shell=False): """Runs the command and returns its output after it has exited. - ``cmd`` is a list of arguments starting with the command names. If - ``shell`` is true, ``cmd`` is assumed to be a string and passed to a + ``cmd`` is a list of byte string arguments starting with the command names. + If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a shell to execute. If the process exits with a non-zero return code diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index b1920d8ac..bce888209 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -91,8 +91,8 @@ def im_resize(maxwidth, path_in, path_out=None): # compatibility. try: util.command_output([ - 'convert', util.syspath(path_in), - '-resize', '{0}x^>'.format(maxwidth), path_out + b'convert', util.syspath(path_in), + b'-resize', b'{0}x^>'.format(maxwidth), path_out ]) except subprocess.CalledProcessError: log.warn(u'artresizer: IM convert failed for {0}', @@ -187,7 +187,7 @@ class ArtResizer(object): # Try invoking ImageMagick's "convert". try: - out = util.command_output(['identify', '--version']) + out = util.command_output([b'identify', b'--version']) if 'imagemagick' in out.lower(): pattern = r".+ (\d+)\.(\d+)\.(\d+).*" diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 35feb4d9c..d44095687 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -196,13 +196,13 @@ class EmbedCoverArtPlugin(BeetsPlugin): # Converting images to grayscale tends to minimize the weight # of colors in the diff score. convert_proc = subprocess.Popen( - ['convert', syspath(imagepath), syspath(art), - '-colorspace', 'gray', 'MIFF:-'], + [b'convert', syspath(imagepath), syspath(art), + b'-colorspace', b'gray', b'MIFF:-'], stdout=subprocess.PIPE, close_fds=not is_windows, ) compare_proc = subprocess.Popen( - ['compare', '-metric', 'PHASH', '-', 'null:'], + [b'compare', b'-metric', b'PHASH', b'-', b'null:'], stdin=convert_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index d5b6ae8d4..11f8c794c 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -21,6 +21,7 @@ import collections import itertools import sys import warnings +import re from beets import logging from beets import ui @@ -68,6 +69,7 @@ AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") class Backend(object): """An abstract class representing engine for calculating RG values. """ + def __init__(self, config, log): """Initialize the backend with the configuration view for the plugin. @@ -83,10 +85,112 @@ class Backend(object): raise NotImplementedError() +# bsg1770gain backend +class Bs1770gainBackend(Backend): + """bs1770gain is a loudness scanner compliant with ITU-R BS.1770 and + its flavors EBU R128, ATSC A/85 and Replaygain 2.0. + """ + + def __init__(self, config, log): + super(Bs1770gainBackend, self).__init__(config, log) + cmd = b'bs1770gain' + + try: + self.method = b'--' + config['method'].get(str) + except: + self.method = b'--replaygain' + + try: + call([cmd, self.method]) + self.command = cmd + except OSError: + raise FatalReplayGainError( + 'Is bs1770gain installed? Is your method in config correct?' + ) + if not self.command: + raise FatalReplayGainError( + 'no replaygain command found: install bs1770gain' + ) + + def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of TrackGain objects. + """ + + output = self.compute_gain(items, False) + return output + + def compute_album_gain(self, album): + """Computes the album gain of the given album, returns an + AlbumGain object. + """ + # TODO: What should be done when not all tracks in the album are + # supported? + + supported_items = album.items() + output = self.compute_gain(supported_items, True) + + return AlbumGain(output[-1], output[:-1]) + + def compute_gain(self, items, is_album): + """Computes the track or album gain of a list of items, returns + a list of TrackGain objects. + When computing album gain, the last TrackGain object returned is + the album gain + """ + + if len(items) == 0: + return [] + + """Compute ReplayGain values and return a list of results + dictionaries as given by `parse_tool_output`. + """ + # Construct shell command. + cmd = [self.command] + cmd = cmd + [self.method] + cmd = cmd + ['-it'] + cmd = cmd + [syspath(i.path) for i in items] + + self._log.debug(u'analyzing {0} files', len(items)) + self._log.debug(u"executing {0}", " ".join(map(displayable_path, cmd))) + output = call(cmd) + self._log.debug(u'analysis finished') + results = self.parse_tool_output(output, + len(items) + is_album) + return results + + def parse_tool_output(self, text, num_lines): + """Given the output from bs1770gain, parse the text and + return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + data = text.decode('utf8', errors='ignore') + regex = ("(\s{2,2}\[\d+\/\d+\].*?|\[ALBUM\].*?)(?=\s{2,2}\[\d+\/\d+\]" + "|\s{2,2}\[ALBUM\]:|done\.$)") + + results = re.findall(regex, data, re.S | re.M) + for ll in results[0:num_lines]: + parts = ll.split(b'\n') + if len(parts) == 0: + self._log.debug(u'bad tool output: {0!r}', text) + raise ReplayGainError('bs1770gain failed') + + d = { + 'file': parts[0], + 'gain': float((parts[1].split('/'))[1].split('LU')[0]), + 'peak': float(parts[2].split('/')[1]), + } + + self._log.info('analysed {}gain={};peak={}', + d['file'].rstrip(), d['gain'], d['peak']) + out.append(Gain(d['gain'], d['peak'])) + return out + + # mpgain/aacgain CLI tool backend. - - class CommandBackend(Backend): + def __init__(self, config, log): super(CommandBackend, self).__init__(config, log) config.add({ @@ -106,9 +210,9 @@ class CommandBackend(Backend): ) else: # Check whether the program is in $PATH. - for cmd in ('mp3gain', 'aacgain'): + for cmd in (b'mp3gain', b'aacgain'): try: - call([cmd, '-v']) + call([cmd, b'-v']) self.command = cmd except OSError: pass @@ -172,14 +276,14 @@ class CommandBackend(Backend): # tag-writing; this turns the mp3gain/aacgain tool into a gain # calculator rather than a tag manipulator because we take care # of changing tags ourselves. - cmd = [self.command, '-o', '-s', 's'] + cmd = [self.command, b'-o', b'-s', b's'] if self.noclip: # Adjust to avoid clipping. - cmd = cmd + ['-k'] + cmd = cmd + [b'-k'] else: # Disable clipping warning. - cmd = cmd + ['-c'] - cmd = cmd + ['-d', bytes(self.gain_offset)] + cmd = cmd + [b'-c'] + cmd = cmd + [b'-d', bytes(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] self._log.debug(u'analyzing {0} files', len(items)) @@ -218,6 +322,7 @@ class CommandBackend(Backend): # GStreamer-based backend. class GStreamerBackend(Backend): + def __init__(self, config, log): super(GStreamerBackend, self).__init__(config, log) self._import_gst() @@ -470,8 +575,9 @@ class AudioToolsBackend(Backend): `_ and its capabilities to read more file formats and compute ReplayGain values using it replaygain module. """ + def __init__(self, config, log): - super(CommandBackend, self).__init__(config, log) + super(AudioToolsBackend, self).__init__(config, log) self._import_audiotools() def _import_audiotools(self): @@ -598,9 +704,10 @@ class ReplayGainPlugin(BeetsPlugin): """ backends = { - "command": CommandBackend, + "command": CommandBackend, "gstreamer": GStreamerBackend, - "audiotools": AudioToolsBackend + "audiotools": AudioToolsBackend, + "bs1770gain": Bs1770gainBackend } def __init__(self): diff --git a/beetsplug/scrub.py b/beetsplug/scrub.py index f6a3bed27..ac0017474 100644 --- a/beetsplug/scrub.py +++ b/beetsplug/scrub.py @@ -139,7 +139,7 @@ class ScrubPlugin(BeetsPlugin): self._log.error(u'could not scrub {0}: {1}', util.displayable_path(path), exc) - def write_item(self, path): + def write_item(self, item, path, tags): """Automatically embed art into imported albums.""" if not scrubbing and self.config['auto']: self._log.debug(u'auto-scrubbing {0}', util.displayable_path(path)) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4da0bb141..9af5d0a34 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Features: * Beets now accept top-level options ``--format-item`` and ``--format-album`` before any subcommand to control how items and albums are displayed. :bug:`1271`: +* :doc:`/plugins/replaygain`: There is a new backend for the `bs1770gain`_ + tool. Thanks to :user:`jmwatte`. :bug:`1343` * There are now multiple levels of verbosity. On the command line, you can make beets somewhat verbose with ``-v`` or very verbose with ``-vv``. :bug:`1244` @@ -82,6 +84,7 @@ Fixes: * :doc:`/plugins/replaygain`: Stop applying replaygain directly to source files when using the mp3gain backend. :bug:`1316` +* Path queries are case-sensitive on non-Windows OSes. :bug:`1165` * :doc:`/plugins/lyrics`: Silence a warning about insecure requests in the new MusixMatch backend. :bug:`1204` * Fix a crash when ``beet`` is invoked without arguments. :bug:`1205` @@ -137,6 +140,8 @@ For developers: immediately after they are initialized. It's also possible to replace the originally created tasks by returning new ones using this event. +.. _bs1770gain: http://bs1770gain.sourceforge.net + 1.3.10 (January 5, 2015) ------------------------ diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 9961ad03c..0c1f7017f 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -385,7 +385,7 @@ Multiple stages run in parallel but each stage processes only one task at a time and each task is processed by only one stage at a time. Plugins provide stages as functions that take two arguments: ``config`` and -``task``, which are ``ImportConfig`` and ``ImportTask`` objects (both defined in +``task``, which are ``ImportSession`` and ``ImportTask`` objects (both defined in ``beets.importer``). Add such a function to the plugin's ``import_stages`` field to register it:: @@ -394,7 +394,7 @@ to register it:: def __init__(self): super(ExamplePlugin, self).__init__() self.import_stages = [self.stage] - def stage(self, config, task): + def stage(self, session, task): print('Importing something!') .. _extend-query: diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index d2584a648..b8f385df8 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -10,9 +10,9 @@ playback levels. Installation ------------ -This plugin can use one of three backends to compute the ReplayGain values: -GStreamer, mp3gain (and its cousin, aacgain), and Python Audio Tools. mp3gain -can be easier to install but GStreamer and Audio Tools support more audio +This plugin can use one of four backends to compute the ReplayGain values: +GStreamer, mp3gain (and its cousin, aacgain), Python Audio Tools and bs1770gain. mp3gain +can be easier to install but GStreamer, Audio Tools and bs1770gain support more audio formats. Once installed, this plugin analyzes all files during the import process. This @@ -75,6 +75,23 @@ On OS X, most of the dependencies can be installed with `Homebrew`_:: .. _Python Audio Tools: http://audiotools.sourceforge.net +bs1770gain +`````````` + +In order to use this backend, you will need to install the bs1770gain command-line tool. Here are some hints: + +* goto `bs1770gain`_ and follow the download instructions +* make sure it is in your $PATH + +.. _bs1770gain: bs1770gain.sourceforge.net + +Then, enable the plugin (see :ref:`using-plugins`) and specify the +backend in your configuration file:: + + replaygain: + backend: bs1770gain + + Configuration ------------- @@ -100,6 +117,14 @@ These options only work with the "command" backend: would keep clipping from occurring. Default: ``yes``. +This option only works with the "bs1770gain" backend: + +- **method**: The loudness scanning standard: either `replaygain` for + ReplayGain 2.0, `ebu` for EBU R128, or `atsc` for ATSC A/85. This dictates + the reference level: -18, -23, or -24 LUFS respectively. Default: + `replaygain` + + Manual Analysis --------------- diff --git a/docs/reference/query.rst b/docs/reference/query.rst index 7dc79461a..20c5360f8 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -184,6 +184,8 @@ Note that this only matches items that are *already in your library*, so a path query won't necessarily find *all* the audio files in a directory---just the ones you've already added to your beets library. +Path queries are case-sensitive on most platforms but case-insensitive on +Windows. .. _query-sort: diff --git a/test/helper.py b/test/helper.py index 8d0dbf8a6..bb32c1c87 100644 --- a/test/helper.py +++ b/test/helper.py @@ -51,6 +51,7 @@ from beets.library import Library, Item, Album from beets import importer from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.mediafile import MediaFile, Image +from beets.ui import _encoding # TODO Move AutotagMock here from test import _common @@ -117,9 +118,13 @@ def capture_stdout(): def has_program(cmd, args=['--version']): """Returns `True` if `cmd` can be executed. """ + full_cmd = [cmd] + args + for i, elem in enumerate(full_cmd): + if isinstance(elem, unicode): + full_cmd[i] = elem.encode(_encoding()) try: with open(os.devnull, 'wb') as devnull: - subprocess.check_call([cmd] + args, stderr=devnull, + subprocess.check_call(full_cmd, stderr=devnull, stdout=devnull, stdin=devnull) except OSError: return False diff --git a/test/test_library.py b/test/test_library.py index a496ef6c0..c3807637e 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -452,6 +452,22 @@ class DestinationTest(_common.TestCase): self.assertEqual(self.i.destination(), np('base/one/_.mp3')) + @unittest.skip('unimplemented: #496') + def test_truncation_does_not_conflict_with_replacement(self): + # Use a replacement that should always replace the last X in any + # path component with a Z. + self.lib.replacements = [ + (re.compile(r'X$'), u'Z'), + ] + + # Construct an item whose untruncated path ends with a Y but whose + # truncated version ends with an X. + self.i.title = 'X' * 300 + 'Y' + + # The final path should reflect the replacement. + dest = self.i.destination() + self.assertTrue('XZ' in dest) + class ItemFormattedMappingTest(_common.LibTestCase): def test_formatted_item_value(self): @@ -1082,6 +1098,10 @@ class ParseQueryTest(unittest.TestCase): self.assertIsInstance(raised.exception, beets.dbcore.query.ParsingError) + def test_parse_bytes(self): + with self.assertRaises(AssertionError): + beets.library.parse_query_string(b"query", None) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/test/test_query.py b/test/test_query.py index a9b1058bd..d512e02b8 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -17,6 +17,8 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +from functools import partial + from test import _common from test._common import unittest from test import helper @@ -461,6 +463,26 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, ['album with backslash']) + def test_case_sensitivity(self): + self.add_album(path='/A/B/C2.mp3', title='caps path') + + makeq = partial(beets.library.PathQuery, 'path', '/A/B') + + results = self.lib.items(makeq(case_sensitive=True)) + self.assert_items_matched(results, ['caps path']) + + results = self.lib.items(makeq(case_sensitive=False)) + self.assert_items_matched(results, ['path item', 'caps path']) + + # test platform-aware default sensitivity + with _common.system_mock('Darwin'): + q = makeq() + self.assertEqual(q.case_sensitive, True) + + with _common.system_mock('Windows'): + q = makeq() + self.assertEqual(q.case_sensitive, False) + class IntQueryTest(unittest.TestCase, TestHelper): diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 64d65b006..fe1bdfea5 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -25,7 +25,7 @@ try: import gi gi.require_version('Gst', '1.0') GST_AVAILABLE = True -except ImportError, ValueError: +except (ImportError, ValueError): GST_AVAILABLE = False if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): @@ -42,9 +42,18 @@ class ReplayGainCliTestBase(TestHelper): try: self.load_plugins('replaygain') except: - self.teardown_beets() - self.unload_plugins() - raise + import sys + # store exception info so an error in teardown does not swallow it + exc_info = sys.exc_info() + try: + self.teardown_beets() + self.unload_plugins() + except: + # if load_plugins() failed then setup is incomplete and + # teardown operations may fail. In particular # {Item,Album} + # may not have the _original_types attribute in unload_plugins + pass + raise exc_info[1], None, exc_info[2] self.config['replaygain']['backend'] = self.backend album = self.add_album_fixture(2)