From 5f7d280987115c38c6cb02fcc1420355444d8ddf Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 30 Sep 2012 14:06:21 -0700 Subject: [PATCH 01/85] changelog note about @KraYmer's lastgenre feature --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ceec1f286..2321b5618 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,8 @@ Changelog * Null values in the database can now be matched with the empty-string regular expression, ``^$``. * Queries now correctly match non-string values in path format predicates. +* :doc:`/plugins/lastgenre`: Use the albums' existing genre tags if they pass + the whitelist (thank to Fabrice Laporte). * :doc:`/plugins/fetchart`: Fix a bug where cover art filenames could lack a ``.jpg`` extension. * :doc:`/plugins/lyrics`: Fix an exception with non-ASCII lyrics. From 1372e42dec85755ebaa1574f3624b4cd08b56160 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 30 Sep 2012 14:16:30 -0700 Subject: [PATCH 02/85] docs: warn that incremental must be enabled early --- docs/reference/cli.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 71ad8aaaa..c1eb446bf 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -51,11 +51,11 @@ right now; this is something we need to work on. Read the configuration file (below). * Also, you can disable the autotagging behavior entirely using ``-A`` - (don't autotag) -- then your music will be imported with its existing + (don't autotag)---then your music will be imported with its existing metadata. * During a long tagging import, it can be useful to keep track of albums - that weren't tagged successfully -- either because they're not in the + that weren't tagged successfully---either because they're not in the MusicBrainz database or because something's wrong with the files. Use the ``-l`` option to specify a filename to log every time you skip and album or import it "as-is" or an album gets skipped as a duplicate. @@ -77,7 +77,11 @@ right now; this is something we need to work on. Read the option to run an *incremental* import. With this flag, beets will keep track of every directory it ever imports and avoid importing them again. This is useful if you have an "incoming" directory that you periodically - add things to. (The ``-I`` flag disables incremental imports.) + add things to. + To get this to work correctly, you'll need to use an incremental import *every + time* you run an import on the directory in question---including the first + time, when no subdirectories will be skipped. So consider enabling the + ``import_incremental`` configuration option. * By default, beets will proceed without asking if it finds a very close metadata match. To disable this and have the importer as you every time, From ba140a3f9797c1c333fa287c7f3d5850073b652e Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Thu, 4 Oct 2012 10:47:51 +0200 Subject: [PATCH 03/85] Added command to the lastgenre plugin --- beetsplug/lastgenre/__init__.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index 4d5cec51a..b915f815c 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -31,6 +31,7 @@ import os from beets import plugins from beets import ui from beets.util import normpath +from beets.ui import commands log = logging.getLogger('beets') @@ -166,6 +167,35 @@ class LastGenrePlugin(plugins.BeetsPlugin): fallback_str = ui.config_val(config, 'lastgenre', 'fallback_str', None) + def commands(self): + lastgenre_cmd = ui.Subcommand('lastgenre', help='fetch genres') + def lastgenre_func(lib, config, opts, args): + # The "write to files" option corresponds to the + # import_write config value. + write = ui.config_val(config, 'beets', 'import_write', + commands.DEFAULT_IMPORT_WRITE, bool) + for album in lib.albums(ui.decargs(args)): + tags = [] + lastfm_obj = LASTFM.get_album(album.albumartist, album.album) + if album.genre: + tags.append(album.genre) + + tags.extend(_tags_for(lastfm_obj)) + genre = _tags_to_genre(tags) + + if not genre and fallback_str != None: + genre = fallback_str + log.debug(u'no last.fm genre found: fallback to %s' % genre) + + if genre is not None: + log.debug(u'adding last.fm album genre: %s' % genre) + album.genre = genre + if write: + for item in album.items(): + item.write() + lastgenre_cmd.func = lastgenre_func + return [lastgenre_cmd] + def imported(self, config, task): tags = [] if task.is_album: From 56958d175b408e789a0a307fbcac4ec254b3588e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 4 Oct 2012 09:47:59 -0700 Subject: [PATCH 04/85] docs + changelog for GH-50 (lastgenre command) --- docs/changelog.rst | 4 +++- docs/plugins/lastgenre.rst | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2321b5618..c54a9ce47 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,7 +21,9 @@ Changelog expression, ``^$``. * Queries now correctly match non-string values in path format predicates. * :doc:`/plugins/lastgenre`: Use the albums' existing genre tags if they pass - the whitelist (thank to Fabrice Laporte). + the whitelist (thanks to Fabrice Laporte). +* :doc:`/plugins/lastgenre`: Add a ``lastgenre`` command for fetching genres + post facto. * :doc:`/plugins/fetchart`: Fix a bug where cover art filenames could lack a ``.jpg`` extension. * :doc:`/plugins/lyrics`: Fix an exception with non-ASCII lyrics. diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index accd7c819..aad040076 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -65,3 +65,11 @@ tree. .. _YAML: http://www.yaml.org/ .. _pyyaml: http://pyyaml.org/ + + +Running Manually +---------------- + +In addition to running automatically on import, the plugin can also run manually +from the command line. Use the command ``beet lastgenre [QUERY]`` to fetch +genres for albums matching a certain query. From a4033faf3b0728174cc369a81688f795ecd70803 Mon Sep 17 00:00:00 2001 From: Blemjhoo Tezoulbr Date: Fri, 5 Oct 2012 00:47:46 +0300 Subject: [PATCH 05/85] zero plugin: ver 0.10 - code cleanup --- beetsplug/zero.py | 53 ++++++++++++++--------------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/beetsplug/zero.py b/beetsplug/zero.py index 01e0cd102..f28ae95ab 100644 --- a/beetsplug/zero.py +++ b/beetsplug/zero.py @@ -14,9 +14,8 @@ """ Clears tag fields in media files.""" -from __future__ import print_function -import sys import re +import logging from beets.plugins import BeetsPlugin from beets import ui from beets.library import ITEM_KEYS @@ -24,14 +23,14 @@ from beets.importer import action __author__ = 'baobab@heresiarch.info' -__version__ = '0.9' +__version__ = '0.10' class ZeroPlugin(BeetsPlugin): _instance = None + _log = logging.getLogger('beets') - debug = False fields = [] patterns = {} warned = False @@ -43,25 +42,16 @@ class ZeroPlugin(BeetsPlugin): return cls._instance def __str__(self): - return ('[zero]\n debug = {0}\n fields = {1}\n patterns = {2}\n' - ' warned = {3}'.format(self.debug, self.fields, self.patterns, - self.warned)) - - def dbg(self, *args): - """Prints message to stderr.""" - if self.debug: - print('[zero]', *args, file=sys.stderr) + return ('[zero]\n fields = {0}\n patterns = {1}\n warned = {2}' + .format(self.fields, self.patterns, self.warned)) def configure(self, config): if not config.has_section('zero'): - self.dbg('plugin is not configured') + self._log.warn('[zero] plugin is not configured') return - self.debug = ui.config_val(config, 'zero', 'debug', True, bool) for f in ui.config_val(config, 'zero', 'fields', '').split(): if f not in ITEM_KEYS: - self.dbg( - 'invalid field \"{0}\" (try \'beet fields\')'.format(f) - ) + self._log.error('[zero] invalid field: {0}'.format(f)) else: self.fields.append(f) p = ui.config_val(config, 'zero', f, '').split() @@ -69,15 +59,11 @@ class ZeroPlugin(BeetsPlugin): self.patterns[f] = p else: self.patterns[f] = ['.'] - if self.debug: - print(self, file=sys.stderr) def import_task_choice_event(self, task, config): """Listen for import_task_choice event.""" - if self.debug: - self.dbg('listen: import_task_choice') if task.choice_flag == action.ASIS and not self.warned: - self.dbg('cannot zero in \"as-is\" mode') + self._log.warn('[zero] cannot zero in \"as-is\" mode') self.warned = True # TODO request write in as-is mode @@ -93,25 +79,24 @@ class ZeroPlugin(BeetsPlugin): def write_event(self, item): """Listen for write event.""" - if self.debug: - self.dbg('listen: write') if not self.fields: - self.dbg('no fields, nothing to do') + self._log.warn('[zero] no fields, nothing to do') return for fn in self.fields: try: fval = getattr(item, fn) except AttributeError: - self.dbg('? no such field: {0}'.format(fn)) + self._log.error('[zero] no such field: {0}'.format(fn)) else: if not self.match_patterns(fval, self.patterns[fn]): - self.dbg('\"{0}\" ({1}) is not match any of: {2}' - .format(fval, fn, ' '.join(self.patterns[fn]))) + self._log.debug('[zero] \"{0}\" ({1}) not match: {2}' + .format(fval, fn, + ' '.join(self.patterns[fn]))) continue - self.dbg('\"{0}\" ({1}) match: {2}' - .format(fval, fn, ' '.join(self.patterns[fn]))) + self._log.debug('[zero] \"{0}\" ({1}) match: {2}' + .format(fval, fn, ' '.join(self.patterns[fn]))) setattr(item, fn, type(fval)()) - self.dbg('{0}={1}'.format(fn, getattr(item, fn))) + self._log.debug('[zero] {0}={1}'.format(fn, getattr(item, fn))) @ZeroPlugin.listen('import_task_choice') @@ -121,9 +106,3 @@ def zero_choice(task, config): @ZeroPlugin.listen('write') def zero_write(item): ZeroPlugin().write_event(item) - - -# simple test -if __name__ == '__main__': - print(ZeroPlugin().match_patterns('test', ['[0-9]'])) - print(ZeroPlugin().match_patterns('test', ['.'])) From c9fafb8379febc2859f45dba6e493e99bc6798d8 Mon Sep 17 00:00:00 2001 From: Blemjhoo Tezoulbr Date: Fri, 5 Oct 2012 02:04:51 +0300 Subject: [PATCH 06/85] plugin the: ver 1.1 - singleton mode, code cleanup --- beetsplug/the.py | 154 ++++++++++++++++++++----------------------- docs/plugins/the.rst | 2 - test/test_the.py | 77 ++++++++++++---------- 3 files changed, 114 insertions(+), 119 deletions(-) diff --git a/beetsplug/the.py b/beetsplug/the.py index ab3e9f0a4..7c333b566 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -14,118 +14,106 @@ """Moves patterns in path formats (suitable for moving articles).""" -from __future__ import print_function -import sys import re +import logging from beets.plugins import BeetsPlugin from beets import ui __author__ = 'baobab@heresiarch.info' -__version__ = '1.0' +__version__ = '1.1' PATTERN_THE = u'^[the]{3}\s' PATTERN_A = u'^[a][n]?\s' FORMAT = u'{0}, {1}' -the_options = { - 'debug': False, - 'the': True, - 'a': True, - 'format': FORMAT, - 'strip': False, - 'silent': False, - 'patterns': [PATTERN_THE, PATTERN_A], -} - - class ThePlugin(BeetsPlugin): + _instance = None + _log = logging.getLogger('beets') + + the = True + a = True + format = u'' + strip = False + patterns = [] + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(ThePlugin, + cls).__new__(cls, *args, **kwargs) + return cls._instance + + def __str__(self): + return ('[the]\n the = {0}\n a = {1}\n format = {2}\n' + ' strip = {3}\n patterns = {4}' + .format(self.the, self.a, self.format, self.strip, + self.patterns)) + def configure(self, config): if not config.has_section('the'): - print('[the] plugin is not configured, using defaults', - file=sys.stderr) + self._log.warn(u'[the] plugin is not configured, using defaults') return - self.in_config = True - the_options['debug'] = ui.config_val(config, 'the', 'debug', False, - bool) - the_options['the'] = ui.config_val(config, 'the', 'the', True, bool) - the_options['a'] = ui.config_val(config, 'the', 'a', True, bool) - the_options['format'] = ui.config_val(config, 'the', 'format', - FORMAT) - the_options['strip'] = ui.config_val(config, 'the', 'strip', False, - bool) - the_options['silent'] = ui.config_val(config, 'the', 'silent', False, - bool) - the_options['patterns'] = ui.config_val(config, 'the', 'patterns', - '').split() - for p in the_options['patterns']: + self.the = ui.config_val(config, 'the', 'the', True, bool) + self.a = ui.config_val(config, 'the', 'a', True, bool) + self.format = ui.config_val(config, 'the', 'format', FORMAT) + self.strip = ui.config_val(config, 'the', 'strip', False, bool) + self.patterns = ui.config_val(config, 'the', 'patterns', '').split() + for p in self.patterns: if p: try: re.compile(p) except re.error: - print(u'[the] invalid pattern: {0}'.format(p), - file=sys.stderr) + self._log.error(u'[the] invalid pattern: {0}'.format(p)) else: if not (p.startswith('^') or p.endswith('$')): - if not the_options['silent']: - print(u'[the] warning: pattern \"{0}\" will not ' - 'match string start/end'.format(p), - file=sys.stderr) - if the_options['a']: - the_options['patterns'] = [PATTERN_A] + the_options['patterns'] - if the_options['the']: - the_options['patterns'] = [PATTERN_THE] + the_options['patterns'] - if not the_options['patterns'] and not the_options['silent']: - print('[the] no patterns defined!') - if the_options['debug']: - print(u'[the] patterns: {0}' - .format(' '.join(the_options['patterns'])), file=sys.stderr) + self._log.warn(u'[the] warning: \"{0}\" will not ' + 'match string start/end'.format(p)) + if self.a: + self.patterns = [PATTERN_A] + self.patterns + if self.the: + self.patterns = [PATTERN_THE] + self.patterns + if not self.patterns: + self._log.warn(u'[the] no patterns defined!') -def unthe(text, pattern, strip=False): - """Moves pattern in the path format string or strips it + def unthe(self, text, pattern): + """Moves pattern in the path format string or strips it - text -- text to handle - pattern -- regexp pattern (case ignore is already on) - strip -- if True, pattern will be removed + text -- text to handle + pattern -- regexp pattern (case ignore is already on) + strip -- if True, pattern will be removed - """ - if text: - r = re.compile(pattern, flags=re.IGNORECASE) - try: - t = r.findall(text)[0] - except IndexError: - return text - else: - r = re.sub(r, '', text).strip() - if strip: - return r + """ + if text: + r = re.compile(pattern, flags=re.IGNORECASE) + try: + t = r.findall(text)[0] + except IndexError: + return text else: - return the_options['format'].format(r, t.strip()).strip() - else: - return u'' + r = re.sub(r, '', text).strip() + if self.strip: + return r + else: + return self.format.format(r, t.strip()).strip() + else: + return u'' + def the_template_func(self, text): + if not self.patterns: + return text + if text: + for p in self.patterns: + r = self.unthe(text, p) + if r != text: + break + self._log.debug(u'[the] \"{0}\" -> \"{1}\"'.format(text, r)) + return r + else: + return u'' @ThePlugin.template_func('the') def func_the(text): """Provides beets template function %the""" - if not the_options['patterns']: - return text - if text: - for p in the_options['patterns']: - r = unthe(text, p, the_options['strip']) - if r != text: - break - if the_options['debug']: - print(u'[the] \"{0}\" -> \"{1}\"'.format(text, r), file=sys.stderr) - return r - else: - return u'' - - -# simple tests -if __name__ == '__main__': - print(unthe('The The', PATTERN_THE)) - print(unthe('An Apple', PATTERN_A)) - print(unthe('A Girl', PATTERN_A, strip=True)) + return ThePlugin().the_template_func(text) diff --git a/docs/plugins/the.rst b/docs/plugins/the.rst index c137a1216..7fb1903de 100644 --- a/docs/plugins/the.rst +++ b/docs/plugins/the.rst @@ -36,8 +36,6 @@ can add plugin section into config file:: format={0}, {1} # strip instead of moving to the end, default is off strip=no - # do not print warnings, default is off - silent=no # custom regexp patterns, separated by space patterns= diff --git a/test/test_the.py b/test/test_the.py index efdd81d9e..5ed30a858 100644 --- a/test/test_the.py +++ b/test/test_the.py @@ -1,50 +1,59 @@ """Tests for the 'the' plugin""" from _common import unittest -from beetsplug import the +from beetsplug.the import ThePlugin, PATTERN_A, PATTERN_THE, FORMAT class ThePluginTest(unittest.TestCase): - - + def test_unthe_with_default_patterns(self): - self.assertEqual(the.unthe('', the.PATTERN_THE), '') - self.assertEqual(the.unthe('The Something', the.PATTERN_THE), + self.assertEqual(ThePlugin().unthe('', PATTERN_THE), '') + self.assertEqual(ThePlugin().unthe('The Something', PATTERN_THE), 'Something, The') - self.assertEqual(the.unthe('The The', the.PATTERN_THE), 'The, The') - self.assertEqual(the.unthe('The The', the.PATTERN_THE), 'The, The') - self.assertEqual(the.unthe('The The X', the.PATTERN_THE), - u'The X, The') - self.assertEqual(the.unthe('the The', the.PATTERN_THE), 'The, the') - self.assertEqual(the.unthe('Protected The', the.PATTERN_THE), + self.assertEqual(ThePlugin().unthe('The The', PATTERN_THE), + 'The, The') + self.assertEqual(ThePlugin().unthe('The The', PATTERN_THE), + 'The, The') + self.assertEqual(ThePlugin().unthe('The The X', PATTERN_THE), + 'The X, The') + self.assertEqual(ThePlugin().unthe('the The', PATTERN_THE), + 'The, the') + self.assertEqual(ThePlugin().unthe('Protected The', PATTERN_THE), 'Protected The') - self.assertEqual(the.unthe('A Boy', the.PATTERN_A), 'Boy, A') - self.assertEqual(the.unthe('a girl', the.PATTERN_A), 'girl, a') - self.assertEqual(the.unthe('An Apple', the.PATTERN_A), 'Apple, An') - self.assertEqual(the.unthe('An A Thing', the.PATTERN_A), 'A Thing, An') - self.assertEqual(the.unthe('the An Arse', the.PATTERN_A), + self.assertEqual(ThePlugin().unthe('A Boy', PATTERN_A), + 'Boy, A') + self.assertEqual(ThePlugin().unthe('a girl', PATTERN_A), + 'girl, a') + self.assertEqual(ThePlugin().unthe('An Apple', PATTERN_A), + 'Apple, An') + self.assertEqual(ThePlugin().unthe('An A Thing', PATTERN_A), + 'A Thing, An') + self.assertEqual(ThePlugin().unthe('the An Arse', PATTERN_A), 'the An Arse') - self.assertEqual(the.unthe('The Something', the.PATTERN_THE, - strip=True), 'Something') - self.assertEqual(the.unthe('An A', the.PATTERN_A, strip=True), 'A') - + ThePlugin().strip = True + self.assertEqual(ThePlugin().unthe('The Something', PATTERN_THE), + 'Something') + self.assertEqual(ThePlugin().unthe('An A', PATTERN_A), 'A') + ThePlugin().strip = False + def test_template_function_with_defaults(self): - the.the_options['patterns'] = [the.PATTERN_THE, the.PATTERN_A] - the.the_options['format'] = the.FORMAT - self.assertEqual(the.func_the('The The'), 'The, The') - self.assertEqual(the.func_the('An A'), 'A, An') - + ThePlugin().patterns = [PATTERN_THE, PATTERN_A] + ThePlugin().format = FORMAT + self.assertEqual(ThePlugin().the_template_func('The The'), 'The, The') + self.assertEqual(ThePlugin().the_template_func('An A'), 'A, An') + def test_custom_pattern(self): - the.the_options['patterns'] = [ u'^test\s'] - the.the_options['format'] = the.FORMAT - self.assertEqual(the.func_the('test passed'), 'passed, test') - + ThePlugin().patterns = [ u'^test\s'] + ThePlugin().format = FORMAT + self.assertEqual(ThePlugin().the_template_func('test passed'), + 'passed, test') + def test_custom_format(self): - the.the_options['patterns'] = [the.PATTERN_THE, the.PATTERN_A] - the.the_options['format'] = '{1} ({0})' - self.assertEqual(the.func_the('The A'), 'The (A)') - - + ThePlugin().patterns = [PATTERN_THE, PATTERN_A] + ThePlugin().format = '{1} ({0})' + self.assertEqual(ThePlugin().the_template_func('The A'), 'The (A)') + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 3eb11355cf6789609240ca9090b5901b31d3d484 Mon Sep 17 00:00:00 2001 From: Blemjhoo Tezoulbr Date: Fri, 5 Oct 2012 02:37:21 +0300 Subject: [PATCH 07/85] ihate plugin: ver 1.0 - initial import --- beetsplug/ihate.py | 142 +++++++++++++++++++++++++++++++++++++++++ docs/plugins/ihate.rst | 35 ++++++++++ docs/plugins/index.rst | 2 + test/test_ihate.py | 52 +++++++++++++++ 4 files changed, 231 insertions(+) create mode 100644 beetsplug/ihate.py create mode 100644 docs/plugins/ihate.rst create mode 100644 test/test_ihate.py diff --git a/beetsplug/ihate.py b/beetsplug/ihate.py new file mode 100644 index 000000000..e589765a8 --- /dev/null +++ b/beetsplug/ihate.py @@ -0,0 +1,142 @@ +# This file is part of beets. +# Copyright 2012, Blemjhoo Tezoulbr . +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Warns you about things you hate (or even blocks import).""" + +import re +import logging +from beets.plugins import BeetsPlugin +from beets import ui +from beets.importer import action + + +__author__ = 'baobab@heresiarch.info' +__version__ = '1.0' + + +class IHatePlugin(BeetsPlugin): + + _instance = None + _log = logging.getLogger('beets') + + warn_genre = [] + warn_artist = [] + warn_album = [] + warn_whitelist = [] + skip_genre = [] + skip_artist = [] + skip_album = [] + skip_whitelist = [] + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(IHatePlugin, + cls).__new__(cls, *args, **kwargs) + return cls._instance + + def __str__(self): + return ('(\n warn_genre = {0}\n' + ' warn_artist = {1}\n' + ' warn_album = {2}\n' + ' warn_whitelist = {3}\n' + ' skip_genre = {4}\n' + ' skip_artist = {5}\n' + ' skip_album = {6}\n' + ' skip_whitelist = {7} )\n' + .format(self.warn_genre, self.warn_artist, self.warn_album, + self.warn_whitelist, self.skip_genre, self.skip_artist, + self.skip_album, self.skip_whitelist)) + + def configure(self, config): + if not config.has_section('ihate'): + self._log.warn('[ihate] plugin is not configured') + return + self.warn_genre = ui.config_val(config, 'ihate', 'warn_genre', + '').split() + self.warn_artist = ui.config_val(config, 'ihate', 'warn_artist', + '').split() + self.warn_album = ui.config_val(config, 'ihate', 'warn_album', + '').split() + self.warn_whitelist = ui.config_val(config, 'ihate', 'warn_whitelist', + '').split() + self.skip_genre = ui.config_val(config, 'ihate', 'skip_genre', + '').split() + self.skip_artist = ui.config_val(config, 'ihate', 'skip_artist', + '').split() + self.skip_album = ui.config_val(config, 'ihate', 'skip_album', + '').split() + self.skip_whitelist = ui.config_val(config, 'ihate', 'skip_whitelist', + '').split() + + @classmethod + def match_patterns(cls, s, patterns): + """Check if string is matching any of the patterns in the list.""" + for p in patterns: + if re.findall(p, s, flags=re.IGNORECASE): + return True + return False + + @classmethod + def do_i_hate_this(cls, task, genre_patterns, artist_patterns, + album_patterns, whitelist_patterns): + """Process group of patterns (warn or skip) and returns True if + task is hated and not whitelisted. + """ + hate = False + try: + genre = task.items[0].genre + except: + genre = u'' + if genre and genre_patterns: + if IHatePlugin.match_patterns(genre, genre_patterns): + hate = True + if not hate and task.cur_album and album_patterns: + if IHatePlugin.match_patterns(task.cur_album, album_patterns): + hate = True + if not hate and task.cur_artist and artist_patterns: + if IHatePlugin.match_patterns(task.cur_artist, artist_patterns): + hate = True + if hate and whitelist_patterns: + if IHatePlugin.match_patterns(task.cur_artist, whitelist_patterns): + hate = False + return hate + + def job_to_do(self): + """Return True if at least one pattern is defined.""" + return any([self.warn_genre, self.warn_artist, self.warn_album, + self.skip_genre, self.skip_artist, self.skip_album]) + + def import_task_choice_event(self, task, config): + if task.choice_flag == action.APPLY: + if self.job_to_do: + self._log.debug('[ihate] processing your hate') + if self.do_i_hate_this(task, self.skip_genre, self.skip_artist, + self.skip_album, self.skip_whitelist): + task.choice_flag = action.SKIP + self._log.info(u'[ihate] skipped: {0} - {1}' + .format(task.cur_artist, task.cur_album)) + return + if self.do_i_hate_this(task, self.warn_genre, self.warn_artist, + self.warn_album, self.warn_whitelist): + self._log.info(u'[ihate] you maybe hate this: {0} - {1}' + .format(task.cur_artist, task.cur_album)) + else: + self._log.debug('[ihate] nothing to do') + else: + self._log.debug('[ihate] user make a decision, nothing to do') + + +@IHatePlugin.listen('import_task_choice') +def ihate_import_task_choice(task, config): + IHatePlugin().import_task_choice_event(task, config) diff --git a/docs/plugins/ihate.rst b/docs/plugins/ihate.rst new file mode 100644 index 000000000..24e46fd14 --- /dev/null +++ b/docs/plugins/ihate.rst @@ -0,0 +1,35 @@ +IHate Plugin +============ + +The ``ihate`` plugin allows you to automatically skip things you hate during +import or warn you about them. It supports album, artist and genre patterns. +Also there is whitelist to avoid skipping bands you still like. There are two +groups: warn and skip. Skip group is checked first. Whitelist overrides any +other patterns. + +To use plugin, enable it by including ``ihate`` into ``plugins`` line of +your beets config:: + + [beets] + plugins = ihate + +You need to configure plugin before use, so add following section into config +file and adjust it to your needs:: + + [ihate] + # you will be warned about these suspicious genres/artists (regexps): + warn_genre=rnb soul power\smetal + warn_artist=bad\band another\sbad\sband + warn_album=tribute\sto + # if you don't like genre in general, but accept some band playing it, + # add exceptions here: + warn_whitelist=hate\sexception + # never import any of this: + skip_genre=russian\srock polka + skip_artist=manowar + skip_album=christmas + # but import this: + skip_whitelist= + +Note: plugin will trust you decision in 'as-is' mode. + \ No newline at end of file diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 757d8a3ac..b9a7e64f5 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -53,6 +53,7 @@ disabled by default, but you can turn them on as described above. the fuzzy_search zero + ihate Autotagger Extensions '''''''''''''''''''''' @@ -92,6 +93,7 @@ Miscellaneous * :doc:`rdm`: Randomly choose albums and tracks from your library. * :doc:`fuzzy_search`: Search albums and tracks with fuzzy string matching. * :doc:`mbcollection`: Maintain your MusicBrainz collection list. +* :doc:`ihate`: Skip by defined patterns things you hate during import process. * :doc:`bpd`: A music player for your beets library that emulates `MPD`_ and is compatible with `MPD clients`_. diff --git a/test/test_ihate.py b/test/test_ihate.py new file mode 100644 index 000000000..3ff6dcd8d --- /dev/null +++ b/test/test_ihate.py @@ -0,0 +1,52 @@ +"""Tests for the 'ihate' plugin""" + +from _common import unittest +from beets.importer import ImportTask +from beets.library import Item +from beetsplug.ihate import IHatePlugin + + +class IHatePluginTest(unittest.TestCase): + + def test_hate(self): + genre_p = [] + artist_p = [] + album_p = [] + white_p = [] + task = ImportTask() + task.cur_artist = u'Test Artist' + task.cur_album = u'Test Album' + task.items = [Item({'genre': 'Test Genre'})] + self.assertFalse(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + genre_p = 'some_genre test\sgenre'.split() + self.assertTrue(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + genre_p = [] + artist_p = 'bad_artist test\sartist' + self.assertTrue(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + artist_p = [] + album_p = 'tribute christmas test'.split() + self.assertTrue(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + album_p = [] + white_p = 'goodband test\sartist another_band'.split() + genre_p = 'some_genre test\sgenre'.split() + self.assertFalse(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + genre_p = [] + artist_p = 'bad_artist test\sartist' + self.assertFalse(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + artist_p = [] + album_p = 'tribute christmas test'.split() + self.assertFalse(IHatePlugin.do_i_hate_this(task, genre_p, artist_p, + album_p, white_p)) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From cf32eb953241c7565f204cf1226bacfa5c1b255d Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Fri, 5 Oct 2012 20:56:09 +0200 Subject: [PATCH 08/85] Add --exact argument to stats --- beets/ui/commands.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index b079d0caa..d818b397f 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1004,7 +1004,7 @@ default_commands.append(remove_cmd) # stats: Show library/query statistics. -def show_stats(lib, query): +def show_stats(lib, query, exact): """Shows some statistics about the matched items.""" items = lib.items(query) @@ -1015,30 +1015,32 @@ def show_stats(lib, query): albums = set() for item in items: - #fixme This is approximate, so people might complain that - # this total size doesn't match "du -sh". Could fix this - # by putting total file size in the database. - total_size += int(item.length * item.bitrate / 8) + if exact: + total_size += os.path.getsize(item.path) + else: + total_size += int(item.length * item.bitrate / 8) total_time += item.length total_items += 1 artists.add(item.artist) albums.add(item.album) - print_("""Tracks: %i -Total time: %s -Total size: %s -Artists: %i -Albums: %i""" % ( - total_items, - ui.human_seconds(total_time), - ui.human_bytes(total_size), - len(artists), len(albums) - )) + size_str = '' + ui.human_bytes(total_size) + if exact: + size_str += ' ({0} bytes)'.format(total_size) + + print_("""Tracks: {0} +Total time: {1} ({2:.2f} seconds) +Total size: {3} +Artists: {4} +Albums: {5}""".format(total_items, ui.human_seconds(total_time), total_time, + size_str, len(artists), len(albums))) stats_cmd = ui.Subcommand('stats', help='show statistics about the library or a query') +stats_cmd.parser.add_option('-e', '--exact', action='store_true', + help='get exact file sizes') def stats_func(lib, config, opts, args): - show_stats(lib, decargs(args)) + show_stats(lib, decargs(args), opts.exact) stats_cmd.func = stats_func default_commands.append(stats_cmd) From bbf974e5818959940549cfbf5c6f8de1dc95d03b Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Fri, 5 Oct 2012 20:56:59 +0200 Subject: [PATCH 09/85] First version of convert plugin --- beetsplug/convert.py | 111 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 beetsplug/convert.py diff --git a/beetsplug/convert.py b/beetsplug/convert.py new file mode 100644 index 000000000..703a61321 --- /dev/null +++ b/beetsplug/convert.py @@ -0,0 +1,111 @@ +# Copyright 2012, Jakob Schnitzer. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Converts tracks or albums to external directory +""" +import logging +import os +import subprocess +import os.path + +from beets.plugins import BeetsPlugin +from beets import ui, library, util, mediafile +from beets.util.functemplate import Template + +log = logging.getLogger('beets') + +def _embed(path, items): + """Embed an image file, located at `path`, into each item. + """ + data = open(syspath(path), 'rb').read() + kindstr = imghdr.what(None, data) + if kindstr not in ('jpeg', 'png'): + log.error('A file of type %s is not allowed as coverart.' % kindstr) + return + log.debug('Embedding album art.') + for item in items: + try: + f = mediafile.MediaFile(syspath(item.path)) + except mediafile.UnreadableFileError as exc: + log.warn('Could not embed art in {0}: {1}'.format( + repr(item.path), exc + )) + continue + f.art = data + f.save() + +def convert_track(source, dest): + with open(os.devnull, "w") as fnull: + subprocess.call('flac -cd "{0}" | lame -V2 - "{1}"'.format(source, dest), + stdout=fnull, stderr=fnull, shell=True) + + +def convert_item(lib, item, dest, artpath): + dest_path = os.path.join(dest,lib.destination(item, fragment = True)) + dest_path = os.path.splitext(dest_path)[0] + '.mp3' + if not os.path.exists(dest_path): + util.mkdirall(dest_path) + log.info('Encoding '+ item.path) + convert_track(item.path, dest_path) + converted_item = library.Item.from_path(dest_path) + converted_item.read(item.path) + converted_item.path = dest_path + converted_item.write() + if artpath: + _embed(artpath,[converted_item]) + else: + log.info('Skipping '+item.path) + +def convert_func(lib, config, opts, args): + if not conf['dest']: + log.error('No destination set') + return + if opts.album: + fmt = u'$albumartist - $album' + else: + fmt = u'$artist - $album - $title' + template = Template(fmt) + if opts.album: + objs = lib.albums(ui.decargs(args)) + else: + objs = list(lib.items(ui.decargs(args))) + + for o in objs: + if opts.album: + ui.print_(o.evaluate_template(template)) + else: + ui.print_(o.evaluate_template(template, lib)) + + if not ui.input_yn("Convert? (Y/n)"): + return + + for o in objs: + if opts.album: + for item in o.items(): + convert_item(lib, item, conf['dest'], o.artpath) + else: + album = lib.get_album(o) + convert_item(lib, o, conf['dest'], album.artpath) + +conf = {} + +class ConvertPlugin(BeetsPlugin): + def configure(self, config): + conf['dest'] = ui.config_val(config, 'convert', 'path', None) + + def commands(self): + cmd = ui.Subcommand('convert', help='convert albums to external location') + cmd.parser.add_option('-a', '--album', action='store_true', + help='choose an album instead of track') + cmd.func = convert_func + return [cmd] From 3d580fc933a3ec15a1be5bbfaaa69c954b39b491 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Fri, 5 Oct 2012 23:04:50 +0200 Subject: [PATCH 10/85] Added threads, cleaned up some of the code --- beetsplug/convert.py | 86 +++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 703a61321..1a3afc260 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -17,6 +17,8 @@ import logging import os import subprocess import os.path +import threading +import imghdr from beets.plugins import BeetsPlugin from beets import ui, library, util, mediafile @@ -27,7 +29,7 @@ log = logging.getLogger('beets') def _embed(path, items): """Embed an image file, located at `path`, into each item. """ - data = open(syspath(path), 'rb').read() + data = open(util.syspath(path), 'rb').read() kindstr = imghdr.what(None, data) if kindstr not in ('jpeg', 'png'): log.error('A file of type %s is not allowed as coverart.' % kindstr) @@ -44,58 +46,74 @@ def _embed(path, items): f.art = data f.save() -def convert_track(source, dest): - with open(os.devnull, "w") as fnull: - subprocess.call('flac -cd "{0}" | lame -V2 - "{1}"'.format(source, dest), - stdout=fnull, stderr=fnull, shell=True) +global sema + +class encodeThread(threading.Thread): + def __init__(self, source, dest, artpath): + threading.Thread.__init__(self) + self.source = source + self.dest = dest + self.artpath = artpath + + def run(self): + sema.acquire() + log.info('Started encoding '+ self.source) + temp_dest = self.dest + "~" + + decode = subprocess.Popen(["flac", "-c", "-d", "-s", self.source], stdout=subprocess.PIPE) + encode = subprocess.Popen(['lame', '-V2', '-', temp_dest], stdin=decode.stdout) + decode.stdout.close() + encode.communicate() + + os.rename(temp_dest, self.dest) + converted_item = library.Item.from_path(self.dest) + converted_item.read(self.source) + converted_item.path = self.dest + converted_item.write() + if self.artpath: + _embed(self.artpath,[converted_item]) + log.info('Finished encoding '+ self.source) + sema.release() def convert_item(lib, item, dest, artpath): + if item.format != "FLAC": + log.info('Skipping {0} : not FLAC'.format(item.path)) + return dest_path = os.path.join(dest,lib.destination(item, fragment = True)) dest_path = os.path.splitext(dest_path)[0] + '.mp3' if not os.path.exists(dest_path): util.mkdirall(dest_path) - log.info('Encoding '+ item.path) - convert_track(item.path, dest_path) - converted_item = library.Item.from_path(dest_path) - converted_item.read(item.path) - converted_item.path = dest_path - converted_item.write() - if artpath: - _embed(artpath,[converted_item]) + thread = encodeThread(item.path, dest_path, artpath) + thread.start() else: - log.info('Skipping '+item.path) + log.info('Skipping {0} : target file exists'.format(item.path)) + def convert_func(lib, config, opts, args): + global sema if not conf['dest']: log.error('No destination set') return + sema = threading.BoundedSemaphore(opts.threads) if opts.album: - fmt = u'$albumartist - $album' + fmt = '$albumartist - $album' else: - fmt = u'$artist - $album - $title' - template = Template(fmt) - if opts.album: - objs = lib.albums(ui.decargs(args)) - else: - objs = list(lib.items(ui.decargs(args))) + fmt = '$artist - $album - $title' - for o in objs: - if opts.album: - ui.print_(o.evaluate_template(template)) - else: - ui.print_(o.evaluate_template(template, lib)) + ui.commands.list_items(lib, ui.decargs(args), opts.album, False, fmt) if not ui.input_yn("Convert? (Y/n)"): return - for o in objs: - if opts.album: - for item in o.items(): + if opts.album: + for album in lib.albums(ui.decargs(args)): + for item in album.items(): convert_item(lib, item, conf['dest'], o.artpath) - else: - album = lib.get_album(o) - convert_item(lib, o, conf['dest'], album.artpath) + else: + for item in lib.items(ui.decargs(args)): + album = lib.get_album(item) + convert_item(lib, item, conf['dest'], album.artpath) conf = {} @@ -106,6 +124,8 @@ class ConvertPlugin(BeetsPlugin): def commands(self): cmd = ui.Subcommand('convert', help='convert albums to external location') cmd.parser.add_option('-a', '--album', action='store_true', - help='choose an album instead of track') + help='choose albums instead of tracks') + cmd.parser.add_option('-t', '--threads', action='store', type='int', + help='change the number of threads (default 2)', default=2) cmd.func = convert_func return [cmd] From 114cea0da64f3fa1039f0b719eb85463d1d7ae83 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 6 Oct 2012 18:42:38 -0700 Subject: [PATCH 11/85] changelog: ihate plugin --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c54a9ce47..50a6e3565 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,9 @@ Changelog text for nicely-sorted directory listings. Thanks to Blemjhoo Tezoulbr. * New plugin: :doc:`/plugins/zero` filters out undesirable fields before they are written to your tags. Thanks again to Blemjhoo Tezoulbr. +* New plugin: :doc:`/plugins/ihate` automatically skips (or warns you about) + importing albums that match certain criteria. Thanks once again to Blemjhoo + Tezoulbr. * :doc:`/plugins/scrub`: Scrubbing now removes *all* types of tags from a file rather than just one. For example, if your FLAC file has both ordinary FLAC tags and ID3 tags, the ID3 tags are now also removed. @@ -23,7 +26,7 @@ Changelog * :doc:`/plugins/lastgenre`: Use the albums' existing genre tags if they pass the whitelist (thanks to Fabrice Laporte). * :doc:`/plugins/lastgenre`: Add a ``lastgenre`` command for fetching genres - post facto. + post facto (thanks to yagebu). * :doc:`/plugins/fetchart`: Fix a bug where cover art filenames could lack a ``.jpg`` extension. * :doc:`/plugins/lyrics`: Fix an exception with non-ASCII lyrics. From eebfaafe7039c6b531db1bacdaf9315bdd9cd4a0 Mon Sep 17 00:00:00 2001 From: Matteo Mecucci Date: Sun, 7 Oct 2012 22:00:49 +0200 Subject: [PATCH 12/85] Small optimization in the fuzzy search plugin. --- beetsplug/fuzzy_search.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/beetsplug/fuzzy_search.py b/beetsplug/fuzzy_search.py index dfdb4b296..398a78811 100644 --- a/beetsplug/fuzzy_search.py +++ b/beetsplug/fuzzy_search.py @@ -24,19 +24,18 @@ import difflib # THRESHOLD = 0.7 -def fuzzy_score(query, item): - return difflib.SequenceMatcher(a=query, b=item).quick_ratio() +def fuzzy_score(queryMatcher, item): + queryMatcher.set_seq1(item) + return queryMatcher.quick_ratio() -def is_match(query, item, album=False, verbose=False, threshold=0.7): - query = ' '.join(query) - +def is_match(queryMatcher, item, album=False, verbose=False, threshold=0.7): if album: values = [item.albumartist, item.album] else: values = [item.artist, item.album, item.title] - s = max(fuzzy_score(query.lower(), i.lower()) for i in values) + s = max(fuzzy_score(queryMatcher, i.lower()) for i in values) if verbose: return (s >= threshold, s) else: @@ -45,6 +44,9 @@ def is_match(query, item, album=False, verbose=False, threshold=0.7): def fuzzy_list(lib, config, opts, args): query = decargs(args) + query = ' '.join(query).lower() + queryMatcher = difflib.SequenceMatcher(b=query) + fmt = opts.format if opts.threshold is not None: threshold = float(opts.threshold) @@ -64,7 +66,7 @@ def fuzzy_list(lib, config, opts, args): else: objs = lib.items() - items = filter(lambda i: is_match(query, i, album=opts.album, + items = filter(lambda i: is_match(queryMatcher, i, album=opts.album, threshold=threshold), objs) for i in items: if opts.path: @@ -74,7 +76,7 @@ def fuzzy_list(lib, config, opts, args): else: print_(i.evaluate_template(template, lib)) if opts.verbose: - print(is_match(query, i, album=opts.album, verbose=True)[1]) + print(is_match(queryMatcher, i, album=opts.album, verbose=True)[1]) fuzzy_cmd = Subcommand('fuzzy', From d1ab9267d09ab826a10cc76b6f4bcd974e2f9794 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Sun, 7 Oct 2012 22:19:15 +0200 Subject: [PATCH 13/85] Added lots of options, support MP3 as source --- beetsplug/convert.py | 128 ++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 1a3afc260..869edc7b4 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -1,23 +1,11 @@ -# Copyright 2012, Jakob Schnitzer. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - """Converts tracks or albums to external directory """ import logging import os -import subprocess -import os.path import threading +import shutil +from subprocess import Popen, PIPE + import imghdr from beets.plugins import BeetsPlugin @@ -25,6 +13,8 @@ from beets import ui, library, util, mediafile from beets.util.functemplate import Template log = logging.getLogger('beets') +DEVNULL = open(os.devnull, 'wb') +conf = {} def _embed(path, items): """Embed an image file, located at `path`, into each item. @@ -46,8 +36,6 @@ def _embed(path, items): f.art = data f.save() -global sema - class encodeThread(threading.Thread): def __init__(self, source, dest, artpath): threading.Thread.__init__(self) @@ -57,50 +45,72 @@ class encodeThread(threading.Thread): def run(self): sema.acquire() - log.info('Started encoding '+ self.source) - temp_dest = self.dest + "~" - - decode = subprocess.Popen(["flac", "-c", "-d", "-s", self.source], stdout=subprocess.PIPE) - encode = subprocess.Popen(['lame', '-V2', '-', temp_dest], stdin=decode.stdout) - decode.stdout.close() - encode.communicate() - - os.rename(temp_dest, self.dest) - converted_item = library.Item.from_path(self.dest) - converted_item.read(self.source) - converted_item.path = self.dest - converted_item.write() - if self.artpath: - _embed(self.artpath,[converted_item]) - log.info('Finished encoding '+ self.source) + self.encode() sema.release() + dest_item = library.Item.from_path(self.source) + dest_item.path = self.dest + dest_item.write() + if self.artpath: + _embed(self.artpath,[dest_item]) -def convert_item(lib, item, dest, artpath): - if item.format != "FLAC": - log.info('Skipping {0} : not FLAC'.format(item.path)) + def encode(self): + log.info('Started encoding '+ self.source) + temp_dest = self.dest + '~' + + source_ext = os.path.splitext(self.source)[1].lower() + if source_ext == '.flac': + decode = Popen([conf['flac'], '-c', '-d', '-s', self.source], + stdout=PIPE) + encode = Popen([conf['lame']] + conf['opts'] + ['-', temp_dest], + stdin=decode.stdout, stderr=DEVNULL) + decode.stdout.close() + encode.communicate() + elif source_ext == '.mp3': + encode = Popen([conf['lame']] + conf['opts'] + ['--mp3input'] + + [self.source, temp_dest], close_fds=True, stderr=DEVNULL) + encode.communicate() + else: + log.error('Only converting from FLAC or MP3 implemented') + return + shutil.move(temp_dest, self.dest) + log.info('Finished encoding '+ self.source) + + +def convert_item(lib, item, dest_dir, artpath): + if item.format != 'FLAC' and item.format != 'MP3': + log.info('Skipping {0} : not supported format'.format(item.path)) return - dest_path = os.path.join(dest,lib.destination(item, fragment = True)) - dest_path = os.path.splitext(dest_path)[0] + '.mp3' - if not os.path.exists(dest_path): - util.mkdirall(dest_path) - thread = encodeThread(item.path, dest_path, artpath) - thread.start() + + dest = os.path.join(dest_dir,lib.destination(item, fragment = True)) + dest = os.path.splitext(dest)[0] + '.mp3' + + if not os.path.exists(dest): + util.mkdirall(dest) + if item.format == 'MP3' and item.bitrate < 1000*conf['max_bitrate']: + log.info('Copying {0}'.format(item.path)) + shutil.copy(item.path, dest) + if artpath: + _embed(artpath,[library.Item.from_path(dest)]) + else: + thread = encodeThread(item.path, dest, artpath) + thread.start() else: log.info('Skipping {0} : target file exists'.format(item.path)) def convert_func(lib, config, opts, args): global sema - if not conf['dest']: + + dest = opts.dest if opts.dest is not None else conf['dest'] + if not dest: log.error('No destination set') return - sema = threading.BoundedSemaphore(opts.threads) - if opts.album: - fmt = '$albumartist - $album' - else: - fmt = '$artist - $album - $title' + threads = opts.threads if opts.threads is not None else conf['threads'] + sema = threading.BoundedSemaphore(threads) + fmt = '$albumartist - $album' if opts.album \ + else '$artist - $album - $title' ui.commands.list_items(lib, ui.decargs(args), opts.album, False, fmt) if not ui.input_yn("Convert? (Y/n)"): @@ -109,23 +119,29 @@ def convert_func(lib, config, opts, args): if opts.album: for album in lib.albums(ui.decargs(args)): for item in album.items(): - convert_item(lib, item, conf['dest'], o.artpath) + convert_item(lib, item, dest, album.artpath) else: for item in lib.items(ui.decargs(args)): - album = lib.get_album(item) - convert_item(lib, item, conf['dest'], album.artpath) - -conf = {} + convert_item(lib, item, dest, lib.get_album(item).artpath) class ConvertPlugin(BeetsPlugin): def configure(self, config): - conf['dest'] = ui.config_val(config, 'convert', 'path', None) + conf['dest'] = ui.config_val(config, 'convert', 'dest', None) + conf['threads'] = ui.config_val(config, 'convert', 'threads', 2) + conf['flac'] = ui.config_val(config, 'convert', 'flac', 'flac') + conf['lame'] = ui.config_val(config, 'convert', 'lame', 'lame') + conf['opts'] = ui.config_val(config, 'convert', + 'opts', '-V2').split(' ') + conf['max_bitrate'] = int(ui.config_val(config, 'convert', + 'max_bitrate','500')) def commands(self): - cmd = ui.Subcommand('convert', help='convert albums to external location') + cmd = ui.Subcommand('convert', help='convert to external location') cmd.parser.add_option('-a', '--album', action='store_true', help='choose albums instead of tracks') cmd.parser.add_option('-t', '--threads', action='store', type='int', - help='change the number of threads (default 2)', default=2) + help='change the number of threads (default 2)') + cmd.parser.add_option('-d', '--dest', action='store', + help='set the destination directory') cmd.func = convert_func return [cmd] From cb939008885aa271ee6d9e2b20aab99396cda0cb Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 7 Oct 2012 15:13:49 -0700 Subject: [PATCH 14/85] changelog & docs for "beet stats --exact" --- docs/changelog.rst | 6 ++++-- docs/reference/cli.rst | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 50a6e3565..17f863fe4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,8 +17,10 @@ Changelog * :doc:`/plugins/scrub`: Scrubbing now removes *all* types of tags from a file rather than just one. For example, if your FLAC file has both ordinary FLAC tags and ID3 tags, the ID3 tags are now also removed. -* ``list`` command: Templates given with ``-f`` can now show items' paths (using - ``$path``). +* :ref:`stats-cmd` command: New ``--exact`` switch to make the file size + calculation more accurate (thanks to yagebu). +* :ref:`list-cmd` command: Templates given with ``-f`` can now show items' paths + (using ``$path``). * Fix album queries for ``artpath`` and other non-item fields. * Null values in the database can now be matched with the empty-string regular expression, ``^$``. diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index c1eb446bf..0b92f7cc8 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -119,6 +119,8 @@ right now; this is something we need to work on. Read the or full albums. If you want to retag your whole library, just supply a null query, which matches everything: ``beet import -L`` +.. _list-cmd: + list ```` :: @@ -212,15 +214,20 @@ To perform a "dry run" an update, just use the ``-p`` (for "pretend") flag. This will show you all the proposed changes but won't actually change anything on disk. +.. _stats-cmd: + stats ````` :: - beet stats [QUERY] + beet stats [-e] [QUERY] Show some statistics on your entire library (if you don't provide a :doc:`query `) or the matched items (if you do). +The ``-e`` (``--exact``) option makes the calculation of total file size more +accurate but slower. + fields `````` :: From aa3a66daad661860a0ac352f8168e4179c6b7c42 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Mon, 8 Oct 2012 11:26:33 +0200 Subject: [PATCH 15/85] Add option to disable embedding --- beetsplug/convert.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 869edc7b4..9554e4279 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -51,7 +51,7 @@ class encodeThread(threading.Thread): dest_item = library.Item.from_path(self.source) dest_item.path = self.dest dest_item.write() - if self.artpath: + if self.artpath and conf['embed']: _embed(self.artpath,[dest_item]) def encode(self): @@ -90,7 +90,7 @@ def convert_item(lib, item, dest_dir, artpath): if item.format == 'MP3' and item.bitrate < 1000*conf['max_bitrate']: log.info('Copying {0}'.format(item.path)) shutil.copy(item.path, dest) - if artpath: + if artpath and conf['embed']: _embed(artpath,[library.Item.from_path(dest)]) else: thread = encodeThread(item.path, dest, artpath) @@ -134,6 +134,8 @@ class ConvertPlugin(BeetsPlugin): 'opts', '-V2').split(' ') conf['max_bitrate'] = int(ui.config_val(config, 'convert', 'max_bitrate','500')) + conf['embed'] = ui.config_val(config, 'convert', 'embed', True, + vtype = bool) def commands(self): cmd = ui.Subcommand('convert', help='convert to external location') From ec6bbf53d4b34ee906ebf901cd62818d2c23f54d Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Mon, 8 Oct 2012 12:25:56 +0200 Subject: [PATCH 16/85] convert: Add docs --- docs/plugins/convert.rst | 54 ++++++++++++++++++++++++++++++++++++++++ docs/plugins/index.rst | 2 ++ 2 files changed, 56 insertions(+) create mode 100644 docs/plugins/convert.rst diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst new file mode 100644 index 000000000..4ec52b152 --- /dev/null +++ b/docs/plugins/convert.rst @@ -0,0 +1,54 @@ +Convert Plugin +============== + +The ``convert`` plugin lets you convert parts of your collection to a directory +of your choice. Currently only converting from MP3 or FLAC to MP3 is supported. +It will skip files that are already present in the target directory. It uses +the same directory structure as your library. + +Installation +------------ + +This plugin requires ``flac`` and ``lame``. If thoses executables are in your +path, they will be found automatically by the plugin, otherwise you have to set +their respective config options. Of course you will have to enable the plugin +as well (see :doc:`/plugins/index`):: + + [convert] + flac:/usr/bin/flac + lame:/usr/bin/lame + +Usage +----- + +To convert a part of your collection simply run ``beet convert QUERY``. This +will display all items matching ``QUERY`` and ask you for confirmation before +starting the conversion. The ``-a`` (or ``--album``) option causes the command +to match albums instead of tracks. + +The ``-t`` (``--threads``) and ``-d`` (``--dest``) options allow you to specify +or overwrite the respective configuration options. + +Configuration +------------- + +This plugin offers a couple of configuration options: If you want to disable +that album art is embedded in your converted items (enabled by default), you +will have to set the ``embed`` option to false. If you set ``max_bitrate``, all +MP3 files with a higher bitrate will be converted and thoses with a lower +bitrate will simply be copied. Be aware that this doesn't mean that your +converted files will have a lower bitrate since that depends on the specified +encoding options. By default only FLAC files will be converted (and all MP3s +will be copied). ``opts`` are the encoding options that are passed to ``lame`` +(defaults to "-V2"). Please refer to the ``lame`` docs for possible options. + +The ``dest`` sets the directory the files will be converted (or copied) to. +Finally ``threads`` lets you determine the number of threads to use for +encoding (default: 2). An example configuration:: + + [convert] + embed:false + max_bitrate:200 + opts:-V4 + dest:/home/user/MusicForPhone + threads:4 diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 757d8a3ac..6ed238f0d 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -53,6 +53,7 @@ disabled by default, but you can turn them on as described above. the fuzzy_search zero + convert Autotagger Extensions '''''''''''''''''''''' @@ -94,6 +95,7 @@ Miscellaneous * :doc:`mbcollection`: Maintain your MusicBrainz collection list. * :doc:`bpd`: A music player for your beets library that emulates `MPD`_ and is compatible with `MPD clients`_. +* :doc:`convert`: Converts parts of your collection to an external directory .. _MPD: http://mpd.wikia.com/ .. _MPD clients: http://mpd.wikia.com/wiki/Clients From f4d6826462a50b1e4818f3427365f325bf8edef3 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Mon, 8 Oct 2012 23:02:22 +0200 Subject: [PATCH 17/85] convert: Added note to docs that 'dest' is a required setting --- docs/plugins/convert.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 4ec52b152..f9d2b120c 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -43,8 +43,9 @@ will be copied). ``opts`` are the encoding options that are passed to ``lame`` (defaults to "-V2"). Please refer to the ``lame`` docs for possible options. The ``dest`` sets the directory the files will be converted (or copied) to. -Finally ``threads`` lets you determine the number of threads to use for -encoding (default: 2). An example configuration:: +This is a required setting and has to be set either in ``.beetsconfig`` or on +the commandline. Finally ``threads`` lets you determine the number of threads +to use for encoding (default: 2). An example configuration:: [convert] embed:false From a2ff20979f8ccbf44637ed4f113e896f11b4d57b Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Tue, 9 Oct 2012 12:01:38 +0200 Subject: [PATCH 18/85] convert: Changed threading model to use beets.util.pipeline, fix embed --- beetsplug/convert.py | 120 ++++++++++++++++++++----------------------- 1 file changed, 57 insertions(+), 63 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 9554e4279..03021ec3f 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -10,7 +10,6 @@ import imghdr from beets.plugins import BeetsPlugin from beets import ui, library, util, mediafile -from beets.util.functemplate import Template log = logging.getLogger('beets') DEVNULL = open(os.devnull, 'wb') @@ -23,11 +22,11 @@ def _embed(path, items): kindstr = imghdr.what(None, data) if kindstr not in ('jpeg', 'png'): log.error('A file of type %s is not allowed as coverart.' % kindstr) - return + return log.debug('Embedding album art.') for item in items: try: - f = mediafile.MediaFile(syspath(item.path)) + f = mediafile.MediaFile(util.syspath(item.path)) except mediafile.UnreadableFileError as exc: log.warn('Could not embed art in {0}: {1}'.format( repr(item.path), exc @@ -36,78 +35,70 @@ def _embed(path, items): f.art = data f.save() -class encodeThread(threading.Thread): - def __init__(self, source, dest, artpath): - threading.Thread.__init__(self) - self.source = source - self.dest = dest - self.artpath = artpath +def encode(source, dest): + log.info('Started encoding '+ source) + temp_dest = dest + '~' - def run(self): - sema.acquire() - self.encode() - sema.release() - - dest_item = library.Item.from_path(self.source) - dest_item.path = self.dest - dest_item.write() - if self.artpath and conf['embed']: - _embed(self.artpath,[dest_item]) - - def encode(self): - log.info('Started encoding '+ self.source) - temp_dest = self.dest + '~' - - source_ext = os.path.splitext(self.source)[1].lower() - if source_ext == '.flac': - decode = Popen([conf['flac'], '-c', '-d', '-s', self.source], - stdout=PIPE) - encode = Popen([conf['lame']] + conf['opts'] + ['-', temp_dest], - stdin=decode.stdout, stderr=DEVNULL) - decode.stdout.close() - encode.communicate() - elif source_ext == '.mp3': - encode = Popen([conf['lame']] + conf['opts'] + ['--mp3input'] + - [self.source, temp_dest], close_fds=True, stderr=DEVNULL) - encode.communicate() - else: - log.error('Only converting from FLAC or MP3 implemented') - return - shutil.move(temp_dest, self.dest) - log.info('Finished encoding '+ self.source) - - -def convert_item(lib, item, dest_dir, artpath): - if item.format != 'FLAC' and item.format != 'MP3': - log.info('Skipping {0} : not supported format'.format(item.path)) + source_ext = os.path.splitext(source)[1].lower() + if source_ext == '.flac': + decode = Popen([conf['flac'], '-c', '-d', '-s', source], + stdout=PIPE) + encode = Popen([conf['lame']] + conf['opts'] + ['-', temp_dest], + stdin=decode.stdout, stderr=DEVNULL) + decode.stdout.close() + encode.communicate() + elif source_ext == '.mp3': + encode = Popen([conf['lame']] + conf['opts'] + ['--mp3input'] + + [source, temp_dest], close_fds=True, stderr=DEVNULL) + encode.communicate() + else: + log.error('Only converting from FLAC or MP3 implemented') return + shutil.move(temp_dest, dest) + log.info('Finished encoding '+ source) - dest = os.path.join(dest_dir,lib.destination(item, fragment = True)) - dest = os.path.splitext(dest)[0] + '.mp3' - if not os.path.exists(dest): +def convert_item(lib, dest_dir): + while True: + item = yield + if item.format != 'FLAC' and item.format != 'MP3': + log.info('Skipping {0} : not supported format'.format(item.path)) + continue + + dest = os.path.join(dest_dir,lib.destination(item, fragment = True)) + dest = os.path.splitext(dest)[0] + '.mp3' + + if os.path.exists(dest): + log.info('Skipping {0} : target file exists'.format(item.path)) + continue + util.mkdirall(dest) if item.format == 'MP3' and item.bitrate < 1000*conf['max_bitrate']: log.info('Copying {0}'.format(item.path)) shutil.copy(item.path, dest) - if artpath and conf['embed']: - _embed(artpath,[library.Item.from_path(dest)]) + dest_item = library.Item.from_path(dest) else: - thread = encodeThread(item.path, dest, artpath) - thread.start() - else: - log.info('Skipping {0} : target file exists'.format(item.path)) + encode(item.path, dest) + dest_item = library.Item.from_path(item.path) + dest_item.path = dest + dest_item.write() + + artpath = lib.get_album(item).artpath + if artpath and conf['embed']: + _embed(artpath,[dest_item]) + + +def generator(items): + for item in items: + yield item def convert_func(lib, config, opts, args): - global sema - dest = opts.dest if opts.dest is not None else conf['dest'] if not dest: log.error('No destination set') return threads = opts.threads if opts.threads is not None else conf['threads'] - sema = threading.BoundedSemaphore(threads) fmt = '$albumartist - $album' if opts.album \ else '$artist - $album - $title' @@ -117,12 +108,14 @@ def convert_func(lib, config, opts, args): return if opts.album: - for album in lib.albums(ui.decargs(args)): - for item in album.items(): - convert_item(lib, item, dest, album.artpath) + items = (i for a in lib.albums(ui.decargs(args)) for i in a.items()) else: - for item in lib.items(ui.decargs(args)): - convert_item(lib, item, dest, lib.get_album(item).artpath) + items = lib.items(ui.decargs(args)) + items = generator(items) + convert = [convert_item(lib, dest) for i in range(threads)] + pipe = util.pipeline.Pipeline([items, convert]) + pipe.run_parallel() + class ConvertPlugin(BeetsPlugin): def configure(self, config): @@ -137,6 +130,7 @@ class ConvertPlugin(BeetsPlugin): conf['embed'] = ui.config_val(config, 'convert', 'embed', True, vtype = bool) + def commands(self): cmd = ui.Subcommand('convert', help='convert to external location') cmd.parser.add_option('-a', '--album', action='store_true', From e273f9dfa9b7ac4ec4a96d73baf161f03f3bca7e Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Tue, 9 Oct 2012 19:11:09 +0200 Subject: [PATCH 19/85] Fix a UnicodeDecodeError when using path in templates --- beets/library.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index b5e60d657..dff5e013c 100644 --- a/beets/library.py +++ b/beets/library.py @@ -29,7 +29,8 @@ from unidecode import unidecode from beets.mediafile import MediaFile from beets import plugins from beets import util -from beets.util import bytestring_path, syspath, normpath, samefile +from beets.util import bytestring_path, syspath, normpath, samefile,\ + displayable_path from beets.util.functemplate import Template MAX_FILENAME_LENGTH = 200 @@ -351,7 +352,7 @@ class Item(object): # Additional fields in non-sanitized case. if not sanitize: - mapping['path'] = self.path + mapping['path'] = displayable_path(self.path) # Use the album artist if the track artist is not set and # vice-versa. @@ -1585,6 +1586,8 @@ class Album(BaseAlbum): for key in ALBUM_KEYS: mapping[key] = getattr(self, key) + mapping['artpath'] = displayable_path(mapping['artpath']) + # Get template functions. funcs = DefaultTemplateFunctions().functions() funcs.update(plugins.template_funcs()) From 3de2b7f090840286aac49d6c31cdcca7e37dbc8c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 9 Oct 2012 10:35:48 -0700 Subject: [PATCH 20/85] pipeline: allow non-generator iterators --- beets/util/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/util/pipeline.py b/beets/util/pipeline.py index b81db3c7f..f9d3b27e9 100644 --- a/beets/util/pipeline.py +++ b/beets/util/pipeline.py @@ -304,11 +304,11 @@ class Pipeline(object): raise ValueError('pipeline must have at least two stages') self.stages = [] for stage in stages: - if isinstance(stage, types.GeneratorType): + if isinstance(stage, (list, tuple)): + self.stages.append(stage) + else: # Default to one thread per stage. self.stages.append((stage,)) - else: - self.stages.append(stage) def run_sequential(self): """Run the pipeline sequentially in the current thread. The From a907d629a2e74a7409cd2391ffe340515dc0dcfa Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 9 Oct 2012 10:40:47 -0700 Subject: [PATCH 21/85] fix error when regex-querying path & artpath --- beets/util/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index e311692a9..3afa9b491 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -469,10 +469,12 @@ def str2bool(value): def as_string(value): """Convert a value to a Unicode object for matching with a query. - None becomes the empty string. + None becomes the empty string. Bytestrings are silently decoded. """ if value is None: return u'' + elif isinstance(value, str): + return value.decode('utf8', 'ignore') else: return unicode(value) From 1662f34528bb22d205f3110fe42f44a877590275 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 9 Oct 2012 10:47:19 -0700 Subject: [PATCH 22/85] "plugin is not configured" logged as debug message @tezoulbr: I'm changing these to debug messages partially so they don't print out when running the tests (with nose, for example) but also because it could get a little annoying for someone who *intends* to use the defaults for one of these plugins. Let me know if you disagree. --- beetsplug/ihate.py | 2 +- beetsplug/the.py | 2 +- beetsplug/zero.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/ihate.py b/beetsplug/ihate.py index e589765a8..eb427a825 100644 --- a/beetsplug/ihate.py +++ b/beetsplug/ihate.py @@ -60,7 +60,7 @@ class IHatePlugin(BeetsPlugin): def configure(self, config): if not config.has_section('ihate'): - self._log.warn('[ihate] plugin is not configured') + self._log.debug('[ihate] plugin is not configured') return self.warn_genre = ui.config_val(config, 'ihate', 'warn_genre', '').split() diff --git a/beetsplug/the.py b/beetsplug/the.py index 7c333b566..da9450f7e 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -52,7 +52,7 @@ class ThePlugin(BeetsPlugin): def configure(self, config): if not config.has_section('the'): - self._log.warn(u'[the] plugin is not configured, using defaults') + self._log.debug(u'[the] plugin is not configured, using defaults') return self.the = ui.config_val(config, 'the', 'the', True, bool) self.a = ui.config_val(config, 'the', 'a', True, bool) diff --git a/beetsplug/zero.py b/beetsplug/zero.py index f28ae95ab..9c22b6d2d 100644 --- a/beetsplug/zero.py +++ b/beetsplug/zero.py @@ -47,7 +47,7 @@ class ZeroPlugin(BeetsPlugin): def configure(self, config): if not config.has_section('zero'): - self._log.warn('[zero] plugin is not configured') + self._log.debug('[zero] plugin is not configured') return for f in ui.config_val(config, 'zero', 'fields', '').split(): if f not in ITEM_KEYS: From 4a0513ccd5e85bfa337d9346784a0611898cf703 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 9 Oct 2012 11:04:48 -0700 Subject: [PATCH 23/85] actually fix unicode-path-query exception I mistakenly assumed that the value sent to _regexp from SQLite would be a str object. It's a buffer object, of course. This change explicitly converts to a str before doing the decoding. --- beets/util/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 3afa9b491..313313cf2 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -473,6 +473,8 @@ def as_string(value): """ if value is None: return u'' + elif isinstance(value, buffer): + return str(value).decode('utf8', 'ignore') elif isinstance(value, str): return value.decode('utf8', 'ignore') else: From 8f9b4f0362169b4d2bde81722089fe32225baa83 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Tue, 9 Oct 2012 23:09:58 +0200 Subject: [PATCH 24/85] convert: remove bloat that's not needed after the fix to pipeline --- beetsplug/convert.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 03021ec3f..590f43ef8 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -88,11 +88,6 @@ def convert_item(lib, dest_dir): _embed(artpath,[dest_item]) -def generator(items): - for item in items: - yield item - - def convert_func(lib, config, opts, args): dest = opts.dest if opts.dest is not None else conf['dest'] if not dest: @@ -111,7 +106,6 @@ def convert_func(lib, config, opts, args): items = (i for a in lib.albums(ui.decargs(args)) for i in a.items()) else: items = lib.items(ui.decargs(args)) - items = generator(items) convert = [convert_item(lib, dest) for i in range(threads)] pipe = util.pipeline.Pipeline([items, convert]) pipe.run_parallel() From 115c0e7410bc39aeb70a96adbb3c10f4d8d31e49 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Wed, 10 Oct 2012 10:15:51 +0200 Subject: [PATCH 25/85] coonvert: make sure temporary are deleted if encoding is interrupted --- beetsplug/convert.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 590f43ef8..1f6be5770 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -54,6 +54,12 @@ def encode(source, dest): else: log.error('Only converting from FLAC or MP3 implemented') return + if encode.returncode != 0: + # Something went wrong (probably Ctrl+C), remove temporary files + log.info('Encoding {0} failed. Cleaning up...'.format(source)) + util.remove(temp_dest) + util.prune_dirs(os.path.dirname(temp_dest)) + return shutil.move(temp_dest, dest) log.info('Finished encoding '+ source) From e316d0ea3062a67aab0171dfba93de5f6002b89d Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Thu, 11 Oct 2012 17:10:28 +0200 Subject: [PATCH 26/85] convert: PEP8, changelog note and license --- beetsplug/convert.py | 44 +++++++++++++++++++++++++++++--------------- docs/changelog.rst | 2 ++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 1f6be5770..089962c21 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -1,8 +1,21 @@ +# This file is part of beets. +# Copyright 2012, Jakob Schnitzer. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + """Converts tracks or albums to external directory """ import logging import os -import threading import shutil from subprocess import Popen, PIPE @@ -15,6 +28,7 @@ log = logging.getLogger('beets') DEVNULL = open(os.devnull, 'wb') conf = {} + def _embed(path, items): """Embed an image file, located at `path`, into each item. """ @@ -35,8 +49,9 @@ def _embed(path, items): f.art = data f.save() + def encode(source, dest): - log.info('Started encoding '+ source) + log.info('Started encoding ' + source) temp_dest = dest + '~' source_ext = os.path.splitext(source)[1].lower() @@ -44,12 +59,12 @@ def encode(source, dest): decode = Popen([conf['flac'], '-c', '-d', '-s', source], stdout=PIPE) encode = Popen([conf['lame']] + conf['opts'] + ['-', temp_dest], - stdin=decode.stdout, stderr=DEVNULL) + stdin=decode.stdout, stderr=DEVNULL) decode.stdout.close() encode.communicate() elif source_ext == '.mp3': encode = Popen([conf['lame']] + conf['opts'] + ['--mp3input'] + - [source, temp_dest], close_fds=True, stderr=DEVNULL) + [source, temp_dest], close_fds=True, stderr=DEVNULL) encode.communicate() else: log.error('Only converting from FLAC or MP3 implemented') @@ -61,7 +76,7 @@ def encode(source, dest): util.prune_dirs(os.path.dirname(temp_dest)) return shutil.move(temp_dest, dest) - log.info('Finished encoding '+ source) + log.info('Finished encoding ' + source) def convert_item(lib, dest_dir): @@ -71,7 +86,7 @@ def convert_item(lib, dest_dir): log.info('Skipping {0} : not supported format'.format(item.path)) continue - dest = os.path.join(dest_dir,lib.destination(item, fragment = True)) + dest = os.path.join(dest_dir, lib.destination(item, fragment=True)) dest = os.path.splitext(dest)[0] + '.mp3' if os.path.exists(dest): @@ -79,7 +94,7 @@ def convert_item(lib, dest_dir): continue util.mkdirall(dest) - if item.format == 'MP3' and item.bitrate < 1000*conf['max_bitrate']: + if item.format == 'MP3' and item.bitrate < 1000 * conf['max_bitrate']: log.info('Copying {0}'.format(item.path)) shutil.copy(item.path, dest) dest_item = library.Item.from_path(dest) @@ -91,7 +106,7 @@ def convert_item(lib, dest_dir): artpath = lib.get_album(item).artpath if artpath and conf['embed']: - _embed(artpath,[dest_item]) + _embed(artpath, [dest_item]) def convert_func(lib, config, opts, args): @@ -102,7 +117,7 @@ def convert_func(lib, config, opts, args): threads = opts.threads if opts.threads is not None else conf['threads'] fmt = '$albumartist - $album' if opts.album \ - else '$artist - $album - $title' + else '$artist - $album - $title' ui.commands.list_items(lib, ui.decargs(args), opts.album, False, fmt) if not ui.input_yn("Convert? (Y/n)"): @@ -126,18 +141,17 @@ class ConvertPlugin(BeetsPlugin): conf['opts'] = ui.config_val(config, 'convert', 'opts', '-V2').split(' ') conf['max_bitrate'] = int(ui.config_val(config, 'convert', - 'max_bitrate','500')) + 'max_bitrate', '500')) conf['embed'] = ui.config_val(config, 'convert', 'embed', True, - vtype = bool) - + vtype=bool) def commands(self): cmd = ui.Subcommand('convert', help='convert to external location') cmd.parser.add_option('-a', '--album', action='store_true', - help='choose albums instead of tracks') + help='choose albums instead of tracks') cmd.parser.add_option('-t', '--threads', action='store', type='int', - help='change the number of threads (default 2)') + help='change the number of threads (default 2)') cmd.parser.add_option('-d', '--dest', action='store', - help='set the destination directory') + help='set the destination directory') cmd.func = convert_func return [cmd] diff --git a/docs/changelog.rst b/docs/changelog.rst index 17f863fe4..837fb3bed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog 1.0b16 (in development) ----------------------- +* New plugin: :doc:`/plugins/convert` lets you convert parts of your collection + to an external directory using flac and lame. * New plugin: :doc:`/plugins/fuzzy_search` lets you find albums and tracks using fuzzy string matching so you don't have to type (or even remember) their exact names. Thanks to Philippe Mongeau. From 7265119aadff168c5acd455ed1aea82d56a24ab5 Mon Sep 17 00:00:00 2001 From: kraymer Date: Sat, 30 Jun 2012 14:24:26 +0200 Subject: [PATCH 27/85] change replaygain plugin backend: it now invokes a command line tool (mp3gain or aacgain) --- beetsplug/replaygain.py | 209 ++++++++++++++++++++++++------------ docs/plugins/replaygain.rst | 56 +++++----- 2 files changed, 174 insertions(+), 91 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 195f3edfb..cd4ce6bfc 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1,4 +1,4 @@ -#Copyright (c) 2011, Peter Brunner (Lugoues) +#Copyright (c) 2012, Fabrice Laporte # #Permission is hereby granted, free of charge, to any person obtaining a copy #of this software and associated documentation files (the "Software"), to deal @@ -19,8 +19,10 @@ #THE SOFTWARE. import logging - -from rgain import rgcalc +import subprocess +import tempfile +import os +import errno from beets import ui from beets.plugins import BeetsPlugin @@ -31,98 +33,173 @@ log = logging.getLogger('beets') DEFAULT_REFERENCE_LOUDNESS = 89 +class RgainError(Exception): + """Base for exceptions in this module.""" + +class RgainNoBackendError(RgainError): + """The audio rgain could not be computed because neither mp3gain + nor aacgain command-line tool is installed. + """ class ReplayGainPlugin(BeetsPlugin): '''Provides replay gain analysis for the Beets Music Manager''' ref_level = DEFAULT_REFERENCE_LOUDNESS - overwrite = False def __init__(self): self.register_listener('album_imported', self.album_imported) self.register_listener('item_imported', self.item_imported) def configure(self, config): - self.overwrite = ui.config_val(config, - 'replaygain', - 'overwrite', - False) + self.overwrite = ui.config_val(config,'replaygain', + 'overwrite', False, bool) + self.noclip = ui.config_val(config,'replaygain', + 'noclip', True, bool) + self.albumgain = ui.config_val(config,'replaygain', + 'albumgain', False, bool) + target_level = float(ui.config_val(config,'replaygain', + 'targetlevel', DEFAULT_REFERENCE_LOUDNESS)) + self.gain_offset = int(target_level-DEFAULT_REFERENCE_LOUDNESS) + self.command = ui.config_val(config,'replaygain','command', None) + if not os.path.isfile(self.command): + raise ui.UserError('no valid rgain command filepath given') + if not self.command: + for cmd in ['mp3gain','aacgain']: + proc = subprocess.Popen([cmd,'-v']) + retcode = proc.poll() + if not retcode: + self.command = cmd + if not self.command: + raise ui.UserError('no valid rgain command found') + def album_imported(self, lib, album, config): - self.write_album = True - - log.debug("Calculating ReplayGain for %s - %s" % \ - (album.albumartist, album.album)) - try: media_files = \ [MediaFile(syspath(item.path)) for item in album.items()] - media_files = [mf for mf in media_files if self.requires_gain(mf)] - #calculate gain. - #Return value - track_data: array dictionary indexed by filename - track_data, album_data = rgcalc.calculate( - [syspath(mf.path) for mf in media_files], - True, - self.ref_level) - - for mf in media_files: - self.write_gain(mf, track_data, album_data) + self.apply_replaygain(media_files) except (FileTypeError, UnreadableFileError, TypeError, ValueError) as e: log.error("failed to calculate replaygain: %s ", e) + def item_imported(self, lib, item, config): try: - self.write_album = False - mf = MediaFile(syspath(item.path)) - - if self.requires_gain(mf): - track_data, album_data = rgcalc.calculate([syspath(mf.path)], - True, - self.ref_level) - self.write_gain(mf, track_data, None) + self.apply_replaygain(mf) except (FileTypeError, UnreadableFileError, - TypeError, ValueError) as e: + TypeError, ValueError) as e: log.error("failed to calculate replaygain: %s ", e) - - def write_gain(self, mf, track_data, album_data): - try: - mf.rg_track_gain = track_data[syspath(mf.path)].gain - mf.rg_track_peak = track_data[syspath(mf.path)].peak - - if self.write_album and album_data: - mf.rg_album_gain = album_data.gain - mf.rg_album_peak = album_data.peak - - log.debug('Tagging ReplayGain for: %s - %s \n' - '\tTrack Gain = %f\n' - '\tTrack Peak = %f\n' - '\tAlbum Gain = %f\n' - '\tAlbum Peak = %f' % \ - (mf.artist, - mf.title, - mf.rg_track_gain, - mf.rg_track_peak, - mf.rg_album_gain, - mf.rg_album_peak)) - else: - log.debug('Tagging ReplayGain for: %s - %s \n' - '\tTrack Gain = %f\n' - '\tTrack Peak = %f\n' % \ - (mf.artist, - mf.title, - mf.rg_track_gain, - mf.rg_track_peak)) - - mf.save() - except (FileTypeError, UnreadableFileError, TypeError, ValueError): - log.error("failed to write replaygain: %s" % (mf.title)) + def requires_gain(self, mf): + '''Does the gain need to be computed?''' + return self.overwrite or \ (not mf.rg_track_gain or not mf.rg_track_peak) or \ ((not mf.rg_album_gain or not mf.rg_album_peak) and \ - self.write_album) + self.albumgain) + + + def get_recommended_gains(self, media_paths): + '''Returns recommended track and album gain values''' + + proc = subprocess.Popen([self.command,'-o','-d',str(self.gain_offset)] + + media_paths, + stdout=subprocess.PIPE) + retcode = proc.poll() + if retcode: + raise RgainError("%s exited with status %i" % + (self.command,retcode)) + rgain_out, _ = proc.communicate() + rgain_out = rgain_out.strip('\n').split('\n') + keys = rgain_out[0].split('\t')[1:] + tracks_mp3_gain = [dict(zip(keys, + [float(x) for x in l.split('\t')[1:]])) + for l in rgain_out[1:-1]] + album_mp3_gain = int(rgain_out[-1].split('\t')[1]) + return [tracks_mp3_gain, album_mp3_gain] + + + def extract_rgain_infos(self, text): + '''Extract rgain infos stats from text''' + + return [l.split('\t') for l in text.split('\n') if l.count('\t')>1][1:] + + + def reduce_gain_for_noclip(self, track_gains, albumgain): + '''Reduce albumgain value until no song is clipped. + No command switch give you the max no-clip in album mode. + So we consider the recommended gain and decrease it until no song is + clipped when applying the gain. + Formula used has been found at: + http://www.hydrogenaudio.org/forums//lofiversion/index.php/t10630.html + ''' + + if albumgain > 0: + for (i,mf) in enumerate(track_gains): + maxpcm = track_gains[i]['Max Amplitude'] + while (maxpcm * (2**(albumgain/4.0)) > 32767): + clipped = 1 + albumgain -= 1 + return albumgain + + + def apply_replaygain(self, media_files): + '''Apply replaygain with correct options to given files. + Returns filtered command stdout''' + + cmd_args = [] + media_files = [mf for mf in media_files if self.requires_gain(mf)] + if not media_files: + print 'No gain to compute' + return + + media_paths = [syspath(mf.path) for mf in media_files] + + if self.albumgain: + track_gains, album_gain = self.get_recommended_gains(media_paths) + if self.noclip: + self.gain_offset = self.reduce_gain_for_noclip(track_gains, + album_gain) + + cmd = [self.command, '-o'] + if self.noclip: + cmd = cmd + ['-k'] + cmd = cmd + ['-r','-d', str(self.gain_offset)] + cmd = cmd + media_paths + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + raise RgainError("%s exited with status %i" % (cmd, e.returncode)) + + cmd = [self.command, '-s','c','-o'] + media_paths + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + if proc.poll(): + raise RgainError("%s exited with status %i" % (cmd, retcode)) + + tmp = proc.communicate()[0] + rgain_infos = self.extract_rgain_infos(tmp) + self.write_gain(media_files, rgain_infos) + + + def write_gain(self, media_files, rgain_infos): + '''Write computed gain infos for each media file''' + + for (i,mf) in enumerate(media_files): + + try: + mf.rg_track_gain = float(rgain_infos[i][2]) + mf.rg_track_peak = float(rgain_infos[i][4]) + + print('Tagging ReplayGain for: %s - %s' % (mf.artist, + mf.title)) + print('\tTrack gain = %f\n' % mf.rg_track_gain) + print('\tTrack peak = %f\n' % mf.rg_track_peak) + + mf.save() + except (FileTypeError, UnreadableFileError, TypeError, ValueError): + log.error("failed to write replaygain: %s" % (mf.title)) + diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 393589cd1..387350279 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -4,42 +4,30 @@ ReplayGain Plugin This plugin adds support for `ReplayGain`_, a technique for normalizing audio playback levels. -.. warning:: - - Some users have reported problems with the Gstreamer ReplayGain calculation - plugin. If you experience segmentation faults or random hangs with this - plugin enabled, consider disabling it. (Please `file a bug`_ if you can get - a gdb traceback for such a segfault or hang.) - - .. _file a bug: http://code.google.com/p/beets/issues/entry Installation ------------ -This plugin requires `GStreamer`_ with the `rganalysis`_ plugin (part of -`gst-plugins-good`_), `gst-python`_, and the `rgain`_ Python module. +This plugin use a command line tool to compute the ReplayGain information: -.. _ReplayGain: http://wiki.hydrogenaudio.org/index.php?title=ReplayGain -.. _rganalysis: http://gstreamer.freedesktop.org/data/doc/gstreamer/head/gst-plugins-good-plugins/html/gst-plugins-good-plugins-rganalysis.html -.. _gst-plugins-good: http://gstreamer.freedesktop.org/modules/gst-plugins-good.html -.. _gst-python: http://gstreamer.freedesktop.org/modules/gst-python.html -.. _rgain: https://github.com/cacack/rgain -.. _pip: http://www.pip-installer.org/ -.. _GStreamer: http://gstreamer.freedesktop.org/ +* On Mac OS X, you can use `Homebrew`_. Type ``brew install aacgain``. +* On Windows, install the original `mp3gain`_. -First, install GStreamer, its "good" plugins, and the Python bindings if your -system doesn't have them already. (The :doc:`/plugins/bpd` and -:doc:`/plugins/chroma` pages have hints on getting GStreamer stuff installed.) -Then install `rgain`_ using `pip`_:: +.. _mp3gain: http://mp3gain.sourceforge.net/download.php +.. _Homebrew: http://mxcl.github.com/homebrew/ - $ pip install rgain - -Finally, add ``replaygain`` to your ``plugins`` line in your -:doc:`/reference/config`, like so:: +To enable the plugin, you’ll need to edit your .beetsconfig file and add the +line ``plugins: replaygain``. [beets] plugins = replaygain +In case beets doesn't find the path to the ReplayGain binary, you can write it +explicitely in the plugin options like so : + + [replaygain] + command: /Applications/MacMP3Gain.app/Contents/Resources/aacgain + Usage & Configuration --------------------- @@ -53,3 +41,21 @@ for the plugin in your :doc:`/reference/config`, like so:: [replaygain] overwrite: yes + +The target level can be modified to any target dB with the ``targetlevel``option +(default: 89 dB). + +The use of ReplayGain can cause clipping if the average volume of a song is below +the target level. By default a "prevent clipping" feature named ``noclip`` is +enabled to reduce the amount of ReplayGain adjustment to whatever amount would +keep clipping from occurring. + +ReplayGain allows to make consistent the loudness of a whole album while allowing + the dynamics from song to song on the album to remain intact. This is called + 'Album Gain' (especially important for classical music albums with large loudness + range). +'Track Gain' (each song considered independently) mode is used by default but can +be changed with ``albumgain`` switch:: + + [replaygain] + albumgain: yes From 6208c453c605e1d8113086e71bda6f42be9a3604 Mon Sep 17 00:00:00 2001 From: kraymer Date: Mon, 16 Jul 2012 23:40:00 +0200 Subject: [PATCH 28/85] add apply_gain option (was default) --- beetsplug/replaygain.py | 19 +++++++++++-------- docs/plugins/replaygain.rst | 14 +++++++++----- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index cd4ce6bfc..b99567fe4 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -55,6 +55,8 @@ class ReplayGainPlugin(BeetsPlugin): 'overwrite', False, bool) self.noclip = ui.config_val(config,'replaygain', 'noclip', True, bool) + self.apply_gain = ui.config_val(config,'replaygain', + 'apply_gain', False, bool) self.albumgain = ui.config_val(config,'replaygain', 'albumgain', False, bool) target_level = float(ui.config_val(config,'replaygain', @@ -78,7 +80,7 @@ class ReplayGainPlugin(BeetsPlugin): media_files = \ [MediaFile(syspath(item.path)) for item in album.items()] - self.apply_replaygain(media_files) + self.write_rgain(media_files, self.compute_rgain(media_files)) except (FileTypeError, UnreadableFileError, TypeError, ValueError) as e: @@ -88,7 +90,7 @@ class ReplayGainPlugin(BeetsPlugin): def item_imported(self, lib, item, config): try: mf = MediaFile(syspath(item.path)) - self.apply_replaygain(mf) + self.write_rgain(mf, self.compute_rgain(mf)) except (FileTypeError, UnreadableFileError, TypeError, ValueError) as e: log.error("failed to calculate replaygain: %s ", e) @@ -147,8 +149,8 @@ class ReplayGainPlugin(BeetsPlugin): return albumgain - def apply_replaygain(self, media_files): - '''Apply replaygain with correct options to given files. + def compute_rgain(self, media_files): + '''Compute replaygain taking options into account. Returns filtered command stdout''' cmd_args = [] @@ -168,7 +170,9 @@ class ReplayGainPlugin(BeetsPlugin): cmd = [self.command, '-o'] if self.noclip: cmd = cmd + ['-k'] - cmd = cmd + ['-r','-d', str(self.gain_offset)] + if self.apply_gain: + cmd = cmd + ['-r'] + cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + media_paths try: subprocess.check_call(cmd) @@ -181,11 +185,10 @@ class ReplayGainPlugin(BeetsPlugin): raise RgainError("%s exited with status %i" % (cmd, retcode)) tmp = proc.communicate()[0] - rgain_infos = self.extract_rgain_infos(tmp) - self.write_gain(media_files, rgain_infos) + return self.extract_rgain_infos(tmp) - def write_gain(self, media_files, rgain_infos): + def write_rgain(self, media_files, rgain_infos): '''Write computed gain infos for each media file''' for (i,mf) in enumerate(media_files): diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 387350279..eabd5f0ab 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -4,6 +4,7 @@ ReplayGain Plugin This plugin adds support for `ReplayGain`_, a technique for normalizing audio playback levels. +.. _ReplayGain: http://wiki.hydrogenaudio.org/index.php?title=ReplayGain Installation ------------ @@ -45,11 +46,6 @@ for the plugin in your :doc:`/reference/config`, like so:: The target level can be modified to any target dB with the ``targetlevel``option (default: 89 dB). -The use of ReplayGain can cause clipping if the average volume of a song is below -the target level. By default a "prevent clipping" feature named ``noclip`` is -enabled to reduce the amount of ReplayGain adjustment to whatever amount would -keep clipping from occurring. - ReplayGain allows to make consistent the loudness of a whole album while allowing the dynamics from song to song on the album to remain intact. This is called 'Album Gain' (especially important for classical music albums with large loudness @@ -59,3 +55,11 @@ be changed with ``albumgain`` switch:: [replaygain] albumgain: yes + +If you use a player that does not support ReplayGain specifications, you may want +to force the volume normalization by applying the gain to the file via the ``apply`` +option. This is a lossless and revertable operation with no decoding/re-encoding involved. +The use of ReplayGain can cause clipping if the average volume of a song is below +the target level. By default a "prevent clipping" feature named ``noclip`` is +enabled to reduce the amount of ReplayGain adjustment to whatever amount would +keep clipping from occurring. \ No newline at end of file From 17842b8d0d4468ceaba5e839e47a96392ac2bf09 Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 21 Jul 2012 19:02:27 +0200 Subject: [PATCH 29/85] rgain: fix computation on singleton import --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index b99567fe4..f5880c925 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -90,7 +90,7 @@ class ReplayGainPlugin(BeetsPlugin): def item_imported(self, lib, item, config): try: mf = MediaFile(syspath(item.path)) - self.write_rgain(mf, self.compute_rgain(mf)) + self.write_rgain([mf], self.compute_rgain([mf])) except (FileTypeError, UnreadableFileError, TypeError, ValueError) as e: log.error("failed to calculate replaygain: %s ", e) From ca6fd2ccf535435a12bbf654e54bd7e213b818a5 Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 21 Jul 2012 19:03:09 +0200 Subject: [PATCH 30/85] rgain: unclutter stdout --- beetsplug/replaygain.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index f5880c925..ebc18eed3 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -174,12 +174,15 @@ class ReplayGainPlugin(BeetsPlugin): cmd = cmd + ['-r'] cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + media_paths + try: - subprocess.check_call(cmd) + with open(os.devnull, 'w') as tempf: + subprocess.check_call(cmd, stdout=tempf, stderr=tempf) except subprocess.CalledProcessError as e: raise RgainError("%s exited with status %i" % (cmd, e.returncode)) cmd = [self.command, '-s','c','-o'] + media_paths + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) if proc.poll(): raise RgainError("%s exited with status %i" % (cmd, retcode)) @@ -196,12 +199,6 @@ class ReplayGainPlugin(BeetsPlugin): try: mf.rg_track_gain = float(rgain_infos[i][2]) mf.rg_track_peak = float(rgain_infos[i][4]) - - print('Tagging ReplayGain for: %s - %s' % (mf.artist, - mf.title)) - print('\tTrack gain = %f\n' % mf.rg_track_gain) - print('\tTrack peak = %f\n' % mf.rg_track_peak) - mf.save() except (FileTypeError, UnreadableFileError, TypeError, ValueError): log.error("failed to write replaygain: %s" % (mf.title)) From 365fa4347eca720fdcea84a2c840aec154f4651c Mon Sep 17 00:00:00 2001 From: "Andrew G. Dunn" Date: Fri, 12 Oct 2012 07:48:52 -0400 Subject: [PATCH 31/85] Added processor/thread detection, by default will now use maximum available processor count instead of 2. Idea adapted from soundconverter, credits in function. --- beetsplug/convert.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 089962c21..c5d871e5b 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -16,6 +16,7 @@ """ import logging import os +import sys import shutil from subprocess import Popen, PIPE @@ -29,6 +30,32 @@ DEVNULL = open(os.devnull, 'wb') conf = {} +def _cpu_count(): + """ Returns the number of CPUs in the system. + Code was adapted from observing the soundconverter project: + https://github.com/kassoulet/soundconverter + """ + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif sys.platform == 'darwin': + try: + num = int(os.popen('sysctl -n hw.ncpu').read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + if num >= 1: + return num + else: + return 1 + + def _embed(path, items): """Embed an image file, located at `path`, into each item. """ @@ -135,7 +162,8 @@ def convert_func(lib, config, opts, args): class ConvertPlugin(BeetsPlugin): def configure(self, config): conf['dest'] = ui.config_val(config, 'convert', 'dest', None) - conf['threads'] = ui.config_val(config, 'convert', 'threads', 2) + conf['threads'] = ui.config_val(config, 'convert', 'threads', + _cpu_count) conf['flac'] = ui.config_val(config, 'convert', 'flac', 'flac') conf['lame'] = ui.config_val(config, 'convert', 'lame', 'lame') conf['opts'] = ui.config_val(config, 'convert', @@ -150,7 +178,8 @@ class ConvertPlugin(BeetsPlugin): cmd.parser.add_option('-a', '--album', action='store_true', help='choose albums instead of tracks') cmd.parser.add_option('-t', '--threads', action='store', type='int', - help='change the number of threads (default 2)') + help='change the number of threads, \ + defaults to maximum availble processors ') cmd.parser.add_option('-d', '--dest', action='store', help='set the destination directory') cmd.func = convert_func From 4ee39ed9da3b218266046b601f688e94f59e4b74 Mon Sep 17 00:00:00 2001 From: "Andrew G. Dunn" Date: Fri, 12 Oct 2012 08:25:28 -0400 Subject: [PATCH 32/85] Forgot to actually call the function --- beetsplug/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index c5d871e5b..3f76e70c3 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -163,7 +163,7 @@ class ConvertPlugin(BeetsPlugin): def configure(self, config): conf['dest'] = ui.config_val(config, 'convert', 'dest', None) conf['threads'] = ui.config_val(config, 'convert', 'threads', - _cpu_count) + _cpu_count()) conf['flac'] = ui.config_val(config, 'convert', 'flac', 'flac') conf['lame'] = ui.config_val(config, 'convert', 'lame', 'lame') conf['opts'] = ui.config_val(config, 'convert', From 780cbed80937520ae1be8ae5334320e1b082cb42 Mon Sep 17 00:00:00 2001 From: "Andrew G. Dunn" Date: Fri, 12 Oct 2012 16:57:12 -0400 Subject: [PATCH 33/85] Made a simple update to the documentation --- docs/plugins/convert.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index f9d2b120c..f62cd963c 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -45,11 +45,12 @@ will be copied). ``opts`` are the encoding options that are passed to ``lame`` The ``dest`` sets the directory the files will be converted (or copied) to. This is a required setting and has to be set either in ``.beetsconfig`` or on the commandline. Finally ``threads`` lets you determine the number of threads -to use for encoding (default: 2). An example configuration:: +to use for encoding. By default the convert plugin will detect the maximum +available cores within a system and use them all. An example configuration:: [convert] embed:false max_bitrate:200 opts:-V4 dest:/home/user/MusicForPhone - threads:4 + threads:4 \ No newline at end of file From 71a5a5b02fa3991dbe01bb20ed85bb87de3c8c10 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 12 Oct 2012 21:55:54 -0700 Subject: [PATCH 34/85] only load plugins from specified modules Eliminate the __subclasses__ trick for finding all plugins. Now we explicitly look in each plugin module for a plugin class. This allows us to import plugin modules with unintentionally loading them. This lets us reuse the image embedding machinery without copypasta. --- beets/plugins.py | 16 ++++++++++------ beetsplug/convert.py | 24 ++---------------------- docs/changelog.rst | 2 ++ 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 0752422e4..ca93b9d0a 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -22,7 +22,6 @@ from collections import defaultdict from beets import mediafile PLUGIN_NAMESPACE = 'beetsplug' -DEFAULT_PLUGINS = [] # Plugins using the Last.fm API can share the same API key. LASTFM_KEY = '2dc3914abf35f0d9c92d97d8f8e42b43' @@ -151,24 +150,29 @@ class BeetsPlugin(object): return func return helper +_classes = set() def load_plugins(names=()): """Imports the modules for a sequence of plugin names. Each name must be the name of a Python module under the "beetsplug" namespace package in sys.path; the module indicated should contain the - BeetsPlugin subclasses desired. A default set of plugins is also - loaded. + BeetsPlugin subclasses desired. """ - for name in itertools.chain(names, DEFAULT_PLUGINS): + for name in names: modname = '%s.%s' % (PLUGIN_NAMESPACE, name) try: try: - __import__(modname, None, None) + namespace = __import__(modname, None, None) except ImportError as exc: # Again, this is hacky: if exc.args[0].endswith(' ' + name): log.warn('** plugin %s not found' % name) else: raise + else: + for obj in getattr(namespace, name).__dict__.values(): + if isinstance(obj, type) and issubclass(obj, BeetsPlugin): + _classes.add(obj) + except: log.warn('** error loading plugin %s' % name) log.warn(traceback.format_exc()) @@ -181,7 +185,7 @@ def find_plugins(): """ load_plugins() plugins = [] - for cls in BeetsPlugin.__subclasses__(): + for cls in _classes: # Only instantiate each plugin class once. if cls not in _instances: _instances[cls] = cls() diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 089962c21..c148a2bf3 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -22,34 +22,14 @@ from subprocess import Popen, PIPE import imghdr from beets.plugins import BeetsPlugin -from beets import ui, library, util, mediafile +from beets import ui, library, util +from beetsplug.embedart import _embed log = logging.getLogger('beets') DEVNULL = open(os.devnull, 'wb') conf = {} -def _embed(path, items): - """Embed an image file, located at `path`, into each item. - """ - data = open(util.syspath(path), 'rb').read() - kindstr = imghdr.what(None, data) - if kindstr not in ('jpeg', 'png'): - log.error('A file of type %s is not allowed as coverart.' % kindstr) - return - log.debug('Embedding album art.') - for item in items: - try: - f = mediafile.MediaFile(util.syspath(item.path)) - except mediafile.UnreadableFileError as exc: - log.warn('Could not embed art in {0}: {1}'.format( - repr(item.path), exc - )) - continue - f.art = data - f.save() - - def encode(source, dest): log.info('Started encoding ' + source) temp_dest = dest + '~' diff --git a/docs/changelog.rst b/docs/changelog.rst index 837fb3bed..b7bc7ea6b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -48,6 +48,8 @@ Changelog * Fix a VFS bug leading to a crash in the :doc:`/plugins/bpd` when files had non-ASCII extensions. * Add a human-readable error message when writing files' tags fails. +* Changed plugin loading so that modules can be imported without + unintentionally loading the plugins they contain. .. _Tomahawk resolver: http://beets.radbox.org/blog/tomahawk-resolver.html From d8433f977c9cc4f6d227c117d26794ceead41a3e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 12 Oct 2012 22:11:07 -0700 Subject: [PATCH 35/85] convert: changelog thanks & doc enhancements --- docs/changelog.rst | 8 +++--- docs/plugins/convert.rst | 60 ++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b7bc7ea6b..03c773fe2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,8 +4,8 @@ Changelog 1.0b16 (in development) ----------------------- -* New plugin: :doc:`/plugins/convert` lets you convert parts of your collection - to an external directory using flac and lame. +* New plugin: :doc:`/plugins/convert` transcodes music and embeds album art + while copying to a separate directory. Thanks to Jakob Schnitzer. * New plugin: :doc:`/plugins/fuzzy_search` lets you find albums and tracks using fuzzy string matching so you don't have to type (or even remember) their exact names. Thanks to Philippe Mongeau. @@ -20,7 +20,7 @@ Changelog rather than just one. For example, if your FLAC file has both ordinary FLAC tags and ID3 tags, the ID3 tags are now also removed. * :ref:`stats-cmd` command: New ``--exact`` switch to make the file size - calculation more accurate (thanks to yagebu). + calculation more accurate (thanks to Jakob Schnitzer). * :ref:`list-cmd` command: Templates given with ``-f`` can now show items' paths (using ``$path``). * Fix album queries for ``artpath`` and other non-item fields. @@ -30,7 +30,7 @@ Changelog * :doc:`/plugins/lastgenre`: Use the albums' existing genre tags if they pass the whitelist (thanks to Fabrice Laporte). * :doc:`/plugins/lastgenre`: Add a ``lastgenre`` command for fetching genres - post facto (thanks to yagebu). + post facto (thanks to Jakob Schnitzer). * :doc:`/plugins/fetchart`: Fix a bug where cover art filenames could lack a ``.jpg`` extension. * :doc:`/plugins/lyrics`: Fix an exception with non-ASCII lyrics. diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index f9d2b120c..0e5bb51ac 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -3,25 +3,26 @@ Convert Plugin The ``convert`` plugin lets you convert parts of your collection to a directory of your choice. Currently only converting from MP3 or FLAC to MP3 is supported. -It will skip files that are already present in the target directory. It uses -the same directory structure as your library. +It will skip files that are already present in the target directory. Converted +files follow the same path formats as your library. Installation ------------ -This plugin requires ``flac`` and ``lame``. If thoses executables are in your -path, they will be found automatically by the plugin, otherwise you have to set -their respective config options. Of course you will have to enable the plugin -as well (see :doc:`/plugins/index`):: +First, enable the ``convert`` plugin (see :doc:`/plugins/index`). + +To transcode music, this plugin requires the ``flac`` and ``lame`` command-line +tools. If those executables are in your path, they will be found automatically +by the plugin. Otherwise, configure the plugin to locate the executables:: [convert] - flac:/usr/bin/flac - lame:/usr/bin/lame + flac: /usr/bin/flac + lame: /usr/bin/lame Usage ----- -To convert a part of your collection simply run ``beet convert QUERY``. This +To convert a part of your collection, run ``beet convert QUERY``. This will display all items matching ``QUERY`` and ask you for confirmation before starting the conversion. The ``-a`` (or ``--album``) option causes the command to match albums instead of tracks. @@ -32,24 +33,29 @@ or overwrite the respective configuration options. Configuration ------------- -This plugin offers a couple of configuration options: If you want to disable -that album art is embedded in your converted items (enabled by default), you -will have to set the ``embed`` option to false. If you set ``max_bitrate``, all -MP3 files with a higher bitrate will be converted and thoses with a lower -bitrate will simply be copied. Be aware that this doesn't mean that your -converted files will have a lower bitrate since that depends on the specified -encoding options. By default only FLAC files will be converted (and all MP3s -will be copied). ``opts`` are the encoding options that are passed to ``lame`` -(defaults to "-V2"). Please refer to the ``lame`` docs for possible options. +The plugin offers several configuration options, all of which live under the +``[convert]`` section: -The ``dest`` sets the directory the files will be converted (or copied) to. -This is a required setting and has to be set either in ``.beetsconfig`` or on -the commandline. Finally ``threads`` lets you determine the number of threads -to use for encoding (default: 2). An example configuration:: +* ``dest`` sets the directory the files will be converted (or copied) to. + A destination is required---you either have to provide it in the config file + or on the command line using the ``-d`` flag. +* ``embed`` indicates whether or not to embed album art in converted items. + Default: true. +* If you set ``max_bitrate``, all MP3 files with a higher bitrate will be + transcoded and those with a lower bitrate will simply be copied. Note that + this does not guarantee that all converted files will have a lower + bitrate---that depends on the encoder and its configuration. By default, FLAC + files will be converted and all MP3s will be copied without transcoding. +* ``opts`` are the encoding options that are passed to ``lame``. Default: + "-V2". Please refer to the LAME documentation for possible options. +* Finally, ``threads`` determines the number of threads to use for parallel + encoding. Default: 2. + +Here's an example configuration:: [convert] - embed:false - max_bitrate:200 - opts:-V4 - dest:/home/user/MusicForPhone - threads:4 + embed: false + max_bitrate: 200 + opts: -V4 + dest: /home/user/MusicForPhone + threads: 4 From fcf5ec0b689debae4bca634ae54e652481d46814 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 12 Oct 2012 22:19:27 -0700 Subject: [PATCH 36/85] convert: low-level tweaks Mainly adding some careful handling of paths (pass through displayable_path before logging, etc.). --- beetsplug/convert.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index c148a2bf3..5a466aca7 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -19,8 +19,6 @@ import os import shutil from subprocess import Popen, PIPE -import imghdr - from beets.plugins import BeetsPlugin from beets import ui, library, util from beetsplug.embedart import _embed @@ -63,20 +61,24 @@ def convert_item(lib, dest_dir): while True: item = yield if item.format != 'FLAC' and item.format != 'MP3': - log.info('Skipping {0} : not supported format'.format(item.path)) + log.info('Skipping {0} (unsupported format)'.format( + util.displayable_path(item.path) + )) continue dest = os.path.join(dest_dir, lib.destination(item, fragment=True)) dest = os.path.splitext(dest)[0] + '.mp3' if os.path.exists(dest): - log.info('Skipping {0} : target file exists'.format(item.path)) + log.info('Skipping {0} (target file exists)'.format( + util.displayable_path(item.path) + )) continue util.mkdirall(dest) if item.format == 'MP3' and item.bitrate < 1000 * conf['max_bitrate']: - log.info('Copying {0}'.format(item.path)) - shutil.copy(item.path, dest) + log.info('Copying {0}'.format(util.displayable_path(item.path))) + util.copy(item.path, dest) dest_item = library.Item.from_path(dest) else: encode(item.path, dest) @@ -92,8 +94,7 @@ def convert_item(lib, dest_dir): def convert_func(lib, config, opts, args): dest = opts.dest if opts.dest is not None else conf['dest'] if not dest: - log.error('No destination set') - return + raise ui.UserError('no convert destination set') threads = opts.threads if opts.threads is not None else conf['threads'] fmt = '$albumartist - $album' if opts.album \ From b9e2beddec6e260fdea38d42a28ec5c30b2fbe0b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 12 Oct 2012 22:42:08 -0700 Subject: [PATCH 37/85] replaygain: changelog & doc enhancements for GH-55 --- beetsplug/replaygain.py | 32 ++++++++++--------------- docs/changelog.rst | 5 ++++ docs/plugins/replaygain.rst | 48 ++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ebc18eed3..a1804b8d5 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1,28 +1,20 @@ -#Copyright (c) 2012, Fabrice Laporte +# This file is part of beets. +# Copyright 2012, Fabrice Laporte and Peter Brunner. # -#Permission is hereby granted, free of charge, to any person obtaining a copy -#of this software and associated documentation files (the "Software"), to deal -#in the Software without restriction, including without limitation the rights -#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -#copies of the Software, and to permit persons to whom the Software is -#furnished to do so, subject to the following conditions: +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -#The above copyright notice and this permission notice shall be included in -#all copies or substantial portions of the Software. -# -#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -#THE SOFTWARE. +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. import logging import subprocess -import tempfile import os -import errno from beets import ui from beets.plugins import BeetsPlugin @@ -38,7 +30,7 @@ class RgainError(Exception): class RgainNoBackendError(RgainError): """The audio rgain could not be computed because neither mp3gain - nor aacgain command-line tool is installed. + nor aacgain command-line tool is installed. """ class ReplayGainPlugin(BeetsPlugin): diff --git a/docs/changelog.rst b/docs/changelog.rst index 03c773fe2..ec27a314c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,9 @@ Changelog * New plugin: :doc:`/plugins/ihate` automatically skips (or warns you about) importing albums that match certain criteria. Thanks once again to Blemjhoo Tezoulbr. +* :doc:`/plugins/replaygain`: This plugin has been completely overhauled to use + the `mp3gain`_ or `aacgain`_ command-line tools instead of the failure-prone + Gstreamer ReplayGain implementation. Thanks to Fabrice Laporte. * :doc:`/plugins/scrub`: Scrubbing now removes *all* types of tags from a file rather than just one. For example, if your FLAC file has both ordinary FLAC tags and ID3 tags, the ID3 tags are now also removed. @@ -52,6 +55,8 @@ Changelog unintentionally loading the plugins they contain. .. _Tomahawk resolver: http://beets.radbox.org/blog/tomahawk-resolver.html +.. _mp3gain: http://mp3gain.sourceforge.net/download.php +.. _aacgain: http://aacgain.altosdesign.com 1.0b15 (July 26, 2012) ---------------------- diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index eabd5f0ab..6eead058b 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -9,22 +9,21 @@ playback levels. Installation ------------ -This plugin use a command line tool to compute the ReplayGain information: +This plugin uses the `mp3gain`_ command-line tool or the `aacgain`_ fork +thereof. To get started, install this tool: * On Mac OS X, you can use `Homebrew`_. Type ``brew install aacgain``. -* On Windows, install the original `mp3gain`_. +* On Linux, `mp3gain`_ is probably in your repositories. On Debian or Ubuntu, + for example, you can run ``apt-get install mp3gain``. +* On Windows, download and install the original `mp3gain`_. .. _mp3gain: http://mp3gain.sourceforge.net/download.php +.. _aacgain: http://aacgain.altosdesign.com .. _Homebrew: http://mxcl.github.com/homebrew/ -To enable the plugin, you’ll need to edit your .beetsconfig file and add the -line ``plugins: replaygain``. - - [beets] - plugins = replaygain - -In case beets doesn't find the path to the ReplayGain binary, you can write it -explicitely in the plugin options like so : +Then enable the ``replaygain`` plugin (see :doc:`/reference/config`). If beets +doesn't automatically find the ``mp3gain`` or ``aacgain`` executable, you can +configure the path explicitly like so:: [replaygain] command: /Applications/MacMP3Gain.app/Contents/Resources/aacgain @@ -43,23 +42,22 @@ for the plugin in your :doc:`/reference/config`, like so:: [replaygain] overwrite: yes -The target level can be modified to any target dB with the ``targetlevel``option -(default: 89 dB). +The target level can be modified to any target dB with the ``targetlevel`` +option (default: 89 dB). -ReplayGain allows to make consistent the loudness of a whole album while allowing - the dynamics from song to song on the album to remain intact. This is called - 'Album Gain' (especially important for classical music albums with large loudness - range). -'Track Gain' (each song considered independently) mode is used by default but can -be changed with ``albumgain`` switch:: +ReplayGain can normalize an entire album's loudness while allowing the dynamics +from song to song on the album to remain intact. This is called "album gain" and +is especially important for classical music albums with large loudness ranges. +"Track gain," in which each song is considered independently, is used by +default. To override this, use the ``albumgain`` option:: [replaygain] albumgain: yes -If you use a player that does not support ReplayGain specifications, you may want -to force the volume normalization by applying the gain to the file via the ``apply`` -option. This is a lossless and revertable operation with no decoding/re-encoding involved. -The use of ReplayGain can cause clipping if the average volume of a song is below -the target level. By default a "prevent clipping" feature named ``noclip`` is -enabled to reduce the amount of ReplayGain adjustment to whatever amount would -keep clipping from occurring. \ No newline at end of file +If you use a player that does not support ReplayGain specifications, you can +force the volume normalization by applying the gain to the file via the +``apply`` option. This is a lossless and reversible operation with no +transcoding involved. The use of ReplayGain can cause clipping if the average +volume of a song is below the target level. By default, a "prevent clipping" +option named ``noclip`` is enabled to reduce the amount of ReplayGain adjustment +to whatever amount would keep clipping from occurring. From 375137bc57fc7defd53df05cfe81d0aaab718a5d Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 13 Oct 2012 11:35:24 +0200 Subject: [PATCH 38/85] replaygain: fix aacgain waiting for user input by using -c switch to ignore clipping warnings --- beetsplug/replaygain.py | 49 +++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index a1804b8d5..3092077f6 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1,16 +1,22 @@ -# This file is part of beets. -# Copyright 2012, Fabrice Laporte and Peter Brunner. +#Copyright (c) 2012, Fabrice Laporte # -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: +#Permission is hereby granted, free of charge, to any person obtaining a copy +#of this software and associated documentation files (the "Software"), to deal +#in the Software without restriction, including without limitation the rights +#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +#copies of the Software, and to permit persons to whom the Software is +#furnished to do so, subject to the following conditions: # -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. +#The above copyright notice and this permission notice shall be included in +#all copies or substantial portions of the Software. +# +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +#THE SOFTWARE. import logging import subprocess @@ -30,7 +36,7 @@ class RgainError(Exception): class RgainNoBackendError(RgainError): """The audio rgain could not be computed because neither mp3gain - nor aacgain command-line tool is installed. + nor aacgain command-line tool is installed. """ class ReplayGainPlugin(BeetsPlugin): @@ -128,16 +134,14 @@ class ReplayGainPlugin(BeetsPlugin): No command switch give you the max no-clip in album mode. So we consider the recommended gain and decrease it until no song is clipped when applying the gain. - Formula used has been found at: - http://www.hydrogenaudio.org/forums//lofiversion/index.php/t10630.html + Formula found at: + http://www.hydrogenaudio.org/forums/lofiversion/index.php/t10630.html ''' if albumgain > 0: - for (i,mf) in enumerate(track_gains): - maxpcm = track_gains[i]['Max Amplitude'] - while (maxpcm * (2**(albumgain/4.0)) > 32767): - clipped = 1 - albumgain -= 1 + maxpcm = max([t['Max Amplitude'] for t in track_gains]) + while (maxpcm * (2**(albumgain/4.0)) > 32767): + albumgain -= 1 return albumgain @@ -162,14 +166,16 @@ class ReplayGainPlugin(BeetsPlugin): cmd = [self.command, '-o'] if self.noclip: cmd = cmd + ['-k'] + else: + cmd = cmd + ['-c'] if self.apply_gain: cmd = cmd + ['-r'] cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + media_paths - + try: with open(os.devnull, 'w') as tempf: - subprocess.check_call(cmd, stdout=tempf, stderr=tempf) + subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=tempf) except subprocess.CalledProcessError as e: raise RgainError("%s exited with status %i" % (cmd, e.returncode)) @@ -191,6 +197,7 @@ class ReplayGainPlugin(BeetsPlugin): try: mf.rg_track_gain = float(rgain_infos[i][2]) mf.rg_track_peak = float(rgain_infos[i][4]) + print('Track gains %s %s' % (mf.rg_track_gain, mf.rg_track_peak)) mf.save() except (FileTypeError, UnreadableFileError, TypeError, ValueError): log.error("failed to write replaygain: %s" % (mf.title)) From 526e82feafa8f359d0db4f723cdef211450ab8b7 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 14 Oct 2012 14:09:03 -0700 Subject: [PATCH 39/85] move cpu_count to util module; credit @storrgie --- beets/util/__init__.py | 26 ++++++++++++++++++++++++++ beetsplug/convert.py | 27 +-------------------------- docs/changelog.rst | 3 ++- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 313313cf2..9be05ee18 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -524,3 +524,29 @@ def plurality(objs): res = obj return res, max_freq + +def cpu_count(): + """Return the number of hardware thread contexts (cores or SMT + threads) in the system. + """ + # Adapted from observing the soundconverter project: + # https://github.com/kassoulet/soundconverter + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif sys.platform == 'darwin': + try: + num = int(os.popen('sysctl -n hw.ncpu').read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + if num >= 1: + return num + else: + return 1 diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 54210c417..de2fdf10c 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -29,31 +29,6 @@ DEVNULL = open(os.devnull, 'wb') conf = {} -def _cpu_count(): - """ Returns the number of CPUs in the system. - Code was adapted from observing the soundconverter project: - https://github.com/kassoulet/soundconverter - """ - if sys.platform == 'win32': - try: - num = int(os.environ['NUMBER_OF_PROCESSORS']) - except (ValueError, KeyError): - num = 0 - elif sys.platform == 'darwin': - try: - num = int(os.popen('sysctl -n hw.ncpu').read()) - except ValueError: - num = 0 - else: - try: - num = os.sysconf('SC_NPROCESSORS_ONLN') - except (ValueError, OSError, AttributeError): - num = 0 - if num >= 1: - return num - else: - return 1 - def encode(source, dest): log.info('Started encoding ' + source) @@ -144,7 +119,7 @@ class ConvertPlugin(BeetsPlugin): def configure(self, config): conf['dest'] = ui.config_val(config, 'convert', 'dest', None) conf['threads'] = ui.config_val(config, 'convert', 'threads', - _cpu_count()) + util.cpu_count()) conf['flac'] = ui.config_val(config, 'convert', 'flac', 'flac') conf['lame'] = ui.config_val(config, 'convert', 'lame', 'lame') conf['opts'] = ui.config_val(config, 'convert', diff --git a/docs/changelog.rst b/docs/changelog.rst index ec27a314c..b3929941b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,7 +5,8 @@ Changelog ----------------------- * New plugin: :doc:`/plugins/convert` transcodes music and embeds album art - while copying to a separate directory. Thanks to Jakob Schnitzer. + while copying to a separate directory. Thanks to Jakob Schnitzer and Andrew G. + Dunn. * New plugin: :doc:`/plugins/fuzzy_search` lets you find albums and tracks using fuzzy string matching so you don't have to type (or even remember) their exact names. Thanks to Philippe Mongeau. From 244ffd71e2ea6be44d9b029964c9ad80f14097db Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 14 Oct 2012 20:27:13 -0700 Subject: [PATCH 40/85] fix "beet modify" for date fields (GC-449) This is fixed by allowing MediaFiles to convert strings to integers on assignment. An eventual complete fix will perform these type conversions in the Item interface. --- beets/mediafile.py | 2 +- beets/util/__init__.py | 2 +- docs/changelog.rst | 2 ++ docs/reference/cli.rst | 2 ++ test/test_mediafile.py | 9 +++++++++ 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 48b212b83..b5e0acce2 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -255,7 +255,7 @@ class Packed(object): field_lengths = [4, 2, 2] # YYYY-MM-DD elems = [] for i, item in enumerate(new_items): - elems.append( ('%0' + str(field_lengths[i]) + 'i') % item ) + elems.append('{0:0{1}}'.format(int(item), field_lengths[i])) self.items = '-'.join(elems) elif self.packstyle == packing.TUPLE: self.items = new_items diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 9be05ee18..9c8b7f35e 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -529,7 +529,7 @@ def cpu_count(): """Return the number of hardware thread contexts (cores or SMT threads) in the system. """ - # Adapted from observing the soundconverter project: + # Adapted from the soundconverter project: # https://github.com/kassoulet/soundconverter if sys.platform == 'win32': try: diff --git a/docs/changelog.rst b/docs/changelog.rst index b3929941b..05cf1938c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -51,6 +51,8 @@ Changelog modify files' tags even when they successfully change the database. * Fix a VFS bug leading to a crash in the :doc:`/plugins/bpd` when files had non-ASCII extensions. +* Fix for changing date fields (like "year") with the :ref:`modify-cmd` + command. * Add a human-readable error message when writing files' tags fails. * Changed plugin loading so that modules can be imported without unintentionally loading the plugins they contain. diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 0b92f7cc8..64b760199 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -164,6 +164,8 @@ You'll be shown a list of the files that will be removed and asked to confirm. By default, this just removes entries from the library database; it doesn't touch the files on disk. To actually delete the files, use ``beet remove -d``. +.. _modify-cmd: + modify `````` :: diff --git a/test/test_mediafile.py b/test/test_mediafile.py index 3c1fc1104..5ac8b0d02 100644 --- a/test/test_mediafile.py +++ b/test/test_mediafile.py @@ -196,6 +196,15 @@ class MissingAudioDataTest(unittest.TestCase): del self.mf.mgfile.info.bitrate # Not available directly. self.assertEqual(self.mf.bitrate, 0) +class TypeTest(unittest.TestCase): + def setUp(self): + path = os.path.join(_common.RSRC, 'full.mp3') + self.mf = beets.mediafile.MediaFile(path) + + def test_year_integer_in_string(self): + self.mf.year = '2009' + self.assertEqual(self.mf.year, 2009) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From f2ab26d6a4c4692ca16076aa5ef3ed469ea0d4f7 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 14 Oct 2012 20:35:03 -0700 Subject: [PATCH 41/85] mbcollection: change chunk size to 200 releases --- beetsplug/mbcollection.py | 2 +- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/mbcollection.py b/beetsplug/mbcollection.py index 9e524f780..a79e0242a 100644 --- a/beetsplug/mbcollection.py +++ b/beetsplug/mbcollection.py @@ -20,7 +20,7 @@ from beets import ui import musicbrainzngs from musicbrainzngs import musicbrainz -SUBMISSION_CHUNK_SIZE = 350 +SUBMISSION_CHUNK_SIZE = 200 def submit_albums(collection_id, release_ids): """Add all of the release IDs to the indicated collection. Multiple diff --git a/docs/changelog.rst b/docs/changelog.rst index 05cf1938c..68922815e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,9 @@ Changelog than just being called "file" (thanks to Zach Denton). * :doc:`/plugins/importfeeds`: Fix error in symlink mode with non-ASCII filenames. +* :doc:`/plugins/mbcollection`: Fix an error when submitting a large number of + releases (we now submit only 200 releases at a time instead of 350). Thanks + to Jonathan Towne. * Add the track mapping dictionary to the ``album_distance`` plugin function. * Fix an assertion failure when the MusicBrainz main database and search server disagree. From 4ebc5237d09bb20f37b83515f0d8ac6a8b81e668 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Mon, 15 Oct 2012 16:01:01 +0200 Subject: [PATCH 42/85] replaygain: Fix TypeError if command option is not set --- beetsplug/replaygain.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3092077f6..35fa7853e 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -61,14 +61,16 @@ class ReplayGainPlugin(BeetsPlugin): 'targetlevel', DEFAULT_REFERENCE_LOUDNESS)) self.gain_offset = int(target_level-DEFAULT_REFERENCE_LOUDNESS) self.command = ui.config_val(config,'replaygain','command', None) - if not os.path.isfile(self.command): - raise ui.UserError('no valid rgain command filepath given') - if not self.command: + if self.command: + if not os.path.isfile(self.command): + raise ui.UserError('no valid rgain command filepath given') + else: for cmd in ['mp3gain','aacgain']: - proc = subprocess.Popen([cmd,'-v']) - retcode = proc.poll() - if not retcode: + try: + subprocess.call([cmd,'-v'], stderr=subprocess.PIPE) self.command = cmd + except OSError: + pass if not self.command: raise ui.UserError('no valid rgain command found') From df6c2443812afe82641d44013b54ec7c778ac065 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 09:57:44 -0700 Subject: [PATCH 43/85] replaygain: fix some spacing and error messages --- beetsplug/replaygain.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 35fa7853e..d37a22bad 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -58,13 +58,20 @@ class ReplayGainPlugin(BeetsPlugin): self.albumgain = ui.config_val(config,'replaygain', 'albumgain', False, bool) target_level = float(ui.config_val(config,'replaygain', - 'targetlevel', DEFAULT_REFERENCE_LOUDNESS)) - self.gain_offset = int(target_level-DEFAULT_REFERENCE_LOUDNESS) + 'targetlevel', DEFAULT_REFERENCE_LOUDNESS)) + self.gain_offset = int(target_level - DEFAULT_REFERENCE_LOUDNESS) + self.command = ui.config_val(config,'replaygain','command', None) if self.command: - if not os.path.isfile(self.command): - raise ui.UserError('no valid rgain command filepath given') + # Explicit executable path. + if not os.path.isfile(self.command): + raise ui.UserError( + 'replaygain command does not exist: {0}'.format( + self.command + ) + ) else: + # Check whether the program is in $PATH. for cmd in ['mp3gain','aacgain']: try: subprocess.call([cmd,'-v'], stderr=subprocess.PIPE) @@ -72,7 +79,9 @@ class ReplayGainPlugin(BeetsPlugin): except OSError: pass if not self.command: - raise ui.UserError('no valid rgain command found') + raise ui.UserError( + 'no replaygain command found: install mp3gain or aacgain' + ) def album_imported(self, lib, album, config): From 4adc896a86bf91a05e024a2ceb0ae29daa88fe59 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 12:14:35 -0700 Subject: [PATCH 44/85] replaygain: consolidate command invocation code Invocations of the mp3gain/aacgain commands are now wrapped in a centralized function that takes care of output capture and error handling. This avoids code duplication for the various sites at which the tool needs to be invoked. This change also avoids unintentionally modifying tags via the command-line tool. The "-s s" option makes the tool *just* calculate RG values rather than toying with tags at all. --- beetsplug/replaygain.py | 66 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index d37a22bad..92e586f77 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -31,14 +31,23 @@ log = logging.getLogger('beets') DEFAULT_REFERENCE_LOUDNESS = 89 -class RgainError(Exception): - """Base for exceptions in this module.""" - -class RgainNoBackendError(RgainError): - """The audio rgain could not be computed because neither mp3gain - nor aacgain command-line tool is installed. +class ReplayGainError(Exception): + """Raised when an error occurs during mp3gain/aacgain execution. """ +def call(args): + """Execute the command indicated by `args` (an array of strings) and + return the command's output. The stderr stream is ignored. If the command + exits abnormally, a ReplayGainError is raised. + """ + try: + with open(os.devnull, 'w') as devnull: + return subprocess.check_output(args, stderr=devnull) + except subprocess.CalledProcessError as e: + raise ReplayGainError( + "{0} exited with status {1}".format(args[0], e.returncode) + ) + class ReplayGainPlugin(BeetsPlugin): '''Provides replay gain analysis for the Beets Music Manager''' @@ -72,11 +81,11 @@ class ReplayGainPlugin(BeetsPlugin): ) else: # Check whether the program is in $PATH. - for cmd in ['mp3gain','aacgain']: + for cmd in ('mp3gain', 'aacgain'): try: - subprocess.call([cmd,'-v'], stderr=subprocess.PIPE) + call([cmd, '-v']) self.command = cmd - except OSError: + except OSError as exc: pass if not self.command: raise ui.UserError( @@ -116,15 +125,8 @@ class ReplayGainPlugin(BeetsPlugin): def get_recommended_gains(self, media_paths): '''Returns recommended track and album gain values''' - - proc = subprocess.Popen([self.command,'-o','-d',str(self.gain_offset)] + - media_paths, - stdout=subprocess.PIPE) - retcode = proc.poll() - if retcode: - raise RgainError("%s exited with status %i" % - (self.command,retcode)) - rgain_out, _ = proc.communicate() + rgain_out = call([self.command, '-o', '-d', str(self.gain_offset)] + + media_paths) rgain_out = rgain_out.strip('\n').split('\n') keys = rgain_out[0].split('\t')[1:] tracks_mp3_gain = [dict(zip(keys, @@ -160,7 +162,6 @@ class ReplayGainPlugin(BeetsPlugin): '''Compute replaygain taking options into account. Returns filtered command stdout''' - cmd_args = [] media_files = [mf for mf in media_files if self.requires_gain(mf)] if not media_files: print 'No gain to compute' @@ -174,30 +175,27 @@ class ReplayGainPlugin(BeetsPlugin): self.gain_offset = self.reduce_gain_for_noclip(track_gains, album_gain) - cmd = [self.command, '-o'] + # Construct shell command. The "-o" option makes the output + # easily parseable (tab-delimited). "-s s" forces gain + # recalculation even if tags are already present and disables + # tag-writing; this turns the mp3gain/aacgain tool into a gain + # calculator rather than a tag manipulator because we take care + # of changing tags ourselves. + cmd = [self.command, '-o', '-s', 's'] if self.noclip: + # Adjust to avoid clipping. cmd = cmd + ['-k'] else: + # Disable clipping warning. cmd = cmd + ['-c'] if self.apply_gain: + # Lossless audio adjustment. cmd = cmd + ['-r'] cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + media_paths - try: - with open(os.devnull, 'w') as tempf: - subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=tempf) - except subprocess.CalledProcessError as e: - raise RgainError("%s exited with status %i" % (cmd, e.returncode)) - - cmd = [self.command, '-s','c','-o'] + media_paths - - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) - if proc.poll(): - raise RgainError("%s exited with status %i" % (cmd, retcode)) - - tmp = proc.communicate()[0] - return self.extract_rgain_infos(tmp) + output = call(cmd) + return self.extract_rgain_infos(output) def write_rgain(self, media_files, rgain_infos): From 8de8777b7e1b28d13a8106470d2af1fcac474255 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 12:19:14 -0700 Subject: [PATCH 45/85] replaygain: use log messages instead of prints --- beetsplug/replaygain.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 92e586f77..97f40c81a 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -49,10 +49,8 @@ def call(args): ) class ReplayGainPlugin(BeetsPlugin): - '''Provides replay gain analysis for the Beets Music Manager''' - - ref_level = DEFAULT_REFERENCE_LOUDNESS - + """Provides ReplayGain analysis. + """ def __init__(self): self.register_listener('album_imported', self.album_imported) self.register_listener('item_imported', self.item_imported) @@ -164,7 +162,7 @@ class ReplayGainPlugin(BeetsPlugin): media_files = [mf for mf in media_files if self.requires_gain(mf)] if not media_files: - print 'No gain to compute' + log.debug('replaygain: no gain to compute') return media_paths = [syspath(mf.path) for mf in media_files] @@ -206,7 +204,9 @@ class ReplayGainPlugin(BeetsPlugin): try: mf.rg_track_gain = float(rgain_infos[i][2]) mf.rg_track_peak = float(rgain_infos[i][4]) - print('Track gains %s %s' % (mf.rg_track_gain, mf.rg_track_peak)) + log.debug('replaygain: wrote track gain {0}, peak {1}'.format( + mf.rg_track_gain, mf.rg_track_peak + )) mf.save() except (FileTypeError, UnreadableFileError, TypeError, ValueError): log.error("failed to write replaygain: %s" % (mf.title)) From 9afaed534cde94ff8aa77d06ab613538ee3c3e5a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 14:31:31 -0700 Subject: [PATCH 46/85] refactor output parsing code to use a dictionary --- beetsplug/replaygain.py | 49 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 97f40c81a..6aaf2fbb4 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -134,10 +134,25 @@ class ReplayGainPlugin(BeetsPlugin): return [tracks_mp3_gain, album_mp3_gain] - def extract_rgain_infos(self, text): - '''Extract rgain infos stats from text''' - - return [l.split('\t') for l in text.split('\n') if l.count('\t')>1][1:] + def parse_tool_output(self, text): + """Given the tab-delimited output from an invocation of mp3gain + or aacgain, parse the text and return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + for line in text.split('\n'): + parts = line.split('\t') + if len(parts) != 6 or parts[0] == 'File': + continue + out.append({ + 'file': parts[0], + 'mp3gain': int(parts[1]), + 'gain': float(parts[2]), + 'peak': float(parts[3]), + 'maxgain': int(parts[4]), + 'mingain': int(parts[5]), + }) + return out def reduce_gain_for_noclip(self, track_gains, albumgain): @@ -157,9 +172,9 @@ class ReplayGainPlugin(BeetsPlugin): def compute_rgain(self, media_files): - '''Compute replaygain taking options into account. - Returns filtered command stdout''' - + """Compute ReplayGain values and return a list of results + dictionaries as given by `parse_tool_output`. + """ media_files = [mf for mf in media_files if self.requires_gain(mf)] if not media_files: log.debug('replaygain: no gain to compute') @@ -192,22 +207,22 @@ class ReplayGainPlugin(BeetsPlugin): cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + media_paths + log.debug('replaygain: analyzing {0} files'.format(len(media_files))) output = call(cmd) - return self.extract_rgain_infos(output) + log.debug('replaygain: analysis finished') + return self.parse_tool_output(output) def write_rgain(self, media_files, rgain_infos): - '''Write computed gain infos for each media file''' - - for (i,mf) in enumerate(media_files): - + """Write computed gain values for each media file. + """ + for mf, info in zip(media_files, rgain_infos): try: - mf.rg_track_gain = float(rgain_infos[i][2]) - mf.rg_track_peak = float(rgain_infos[i][4]) + mf.rg_track_gain = float(info['gain']) + mf.rg_track_peak = float(info['peak']) log.debug('replaygain: wrote track gain {0}, peak {1}'.format( mf.rg_track_gain, mf.rg_track_peak )) mf.save() - except (FileTypeError, UnreadableFileError, TypeError, ValueError): - log.error("failed to write replaygain: %s" % (mf.title)) - + except (UnreadableFileError, TypeError, ValueError): + log.error("replaygain: write failed for %s" % (mf.title)) From 672ac78e76431c0341496d17aafa9215afcfa6f9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 14:42:28 -0700 Subject: [PATCH 47/85] replaygain: write album-level tags --- beetsplug/replaygain.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 6aaf2fbb4..9db317cd1 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -96,7 +96,9 @@ class ReplayGainPlugin(BeetsPlugin): media_files = \ [MediaFile(syspath(item.path)) for item in album.items()] - self.write_rgain(media_files, self.compute_rgain(media_files)) + self.write_rgain(media_files, + self.compute_rgain(media_files), + True) except (FileTypeError, UnreadableFileError, TypeError, ValueError) as e: @@ -213,16 +215,28 @@ class ReplayGainPlugin(BeetsPlugin): return self.parse_tool_output(output) - def write_rgain(self, media_files, rgain_infos): + def write_rgain(self, media_files, rgain_infos, album=False): """Write computed gain values for each media file. """ + if album: + assert len(rgain_infos) == len(media_files) + 1 + album_info = rgain_infos[-1] + for mf, info in zip(media_files, rgain_infos): try: - mf.rg_track_gain = float(info['gain']) - mf.rg_track_peak = float(info['peak']) - log.debug('replaygain: wrote track gain {0}, peak {1}'.format( - mf.rg_track_gain, mf.rg_track_peak + mf.rg_track_gain = info['gain'] + mf.rg_track_peak = info['peak'] + + if album: + mf.rg_album_gain = album_info['gain'] + mf.rg_album_peak = album_info['peak'] + + log.debug('replaygain: writing track gain {0}, peak {1}; ' + 'album gain {2}, peak {3}'.format( + mf.rg_track_gain, mf.rg_track_peak, + mf.rg_album_gain, mf.rg_album_peak )) mf.save() - except (UnreadableFileError, TypeError, ValueError): + + except UnreadableFileError: log.error("replaygain: write failed for %s" % (mf.title)) From 6115fba76514b562a255b0660be20206c1da9e42 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 14:54:06 -0700 Subject: [PATCH 48/85] replaygain: calculate when any file needs calculation This ensures accurate album-level data. It also fixes a problem with the old way of doing things where the MediaFiles and tool results would become misaligned if a subset of the tracks needed recalculation. --- beetsplug/replaygain.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 9db317cd1..c4401b0cc 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -177,8 +177,11 @@ class ReplayGainPlugin(BeetsPlugin): """Compute ReplayGain values and return a list of results dictionaries as given by `parse_tool_output`. """ - media_files = [mf for mf in media_files if self.requires_gain(mf)] - if not media_files: + # Skip calculating gain only when *all* files don't need + # recalculation. This way, if any file among an album's tracks + # needs recalculation, we still get an accurate album gain + # value. + if all([not self.requires_gain(mf) for mf in media_files]): log.debug('replaygain: no gain to compute') return From b81ac1d6e05ef73a3613134ea799332990706bc3 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 14:58:59 -0700 Subject: [PATCH 49/85] fix crash when stdin comes from pipe w/o encoding This allows, for example, "yes | beet convert". --- beets/ui/__init__.py | 2 +- docs/changelog.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index d6a29a2e1..c3fb162e3 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -132,7 +132,7 @@ def input_(prompt=None): except EOFError: raise UserError('stdin stream ended while input required') - return resp.decode(sys.stdin.encoding, 'ignore') + return resp.decode(sys.stdin.encoding or 'utf8', 'ignore') def input_options(options, require=False, prompt=None, fallback_prompt=None, numrange=None, default=None, color=False, max_width=72): diff --git a/docs/changelog.rst b/docs/changelog.rst index 68922815e..19d95b86d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -56,6 +56,7 @@ Changelog non-ASCII extensions. * Fix for changing date fields (like "year") with the :ref:`modify-cmd` command. +* Fix a crash when input is read from a pipe without a specified encoding. * Add a human-readable error message when writing files' tags fails. * Changed plugin loading so that modules can be imported without unintentionally loading the plugins they contain. From cc8ead7e342d896cf90c4b8bf8633b608f1ca83f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Oct 2012 19:53:17 -0700 Subject: [PATCH 50/85] convert: atomic mkdirall() call --- beetsplug/convert.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index de2fdf10c..1a415f254 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -16,8 +16,8 @@ """ import logging import os -import sys import shutil +import threading from subprocess import Popen, PIPE from beets.plugins import BeetsPlugin @@ -27,7 +27,7 @@ from beetsplug.embedart import _embed log = logging.getLogger('beets') DEVNULL = open(os.devnull, 'wb') conf = {} - +_fs_lock = threading.Lock() def encode(source, dest): @@ -77,7 +77,12 @@ def convert_item(lib, dest_dir): )) continue - util.mkdirall(dest) + # Ensure that only one thread tries to create directories at a + # time. (The existence check is not atomic with the directory + # creation inside this function.) + with _fs_lock: + util.mkdirall(dest) + if item.format == 'MP3' and item.bitrate < 1000 * conf['max_bitrate']: log.info('Copying {0}'.format(util.displayable_path(item.path))) util.copy(item.path, dest) From 26dfe38bb0e4008d13f7b7232da1162fadbf530d Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Wed, 17 Oct 2012 21:12:31 +0200 Subject: [PATCH 51/85] convert: Write tags from library instead of copying them --- beetsplug/convert.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index de2fdf10c..f7c57c225 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -84,9 +84,8 @@ def convert_item(lib, dest_dir): dest_item = library.Item.from_path(dest) else: encode(item.path, dest) - dest_item = library.Item.from_path(item.path) - dest_item.path = dest - dest_item.write() + item.path = dest + item.write() artpath = lib.get_album(item).artpath if artpath and conf['embed']: From 3d68cf5debefa3e82baba2e6c00b8f1949790674 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Oct 2012 20:06:10 -0700 Subject: [PATCH 52/85] replaygain: fix album nopeak adjustment Removed the second tool invocation. We now adjust the album-level gain based on track peaks in one fell swoop. Based on help from Fabrice via email. --- beetsplug/replaygain.py | 49 ++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index c4401b0cc..e2b15908f 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -83,7 +83,7 @@ class ReplayGainPlugin(BeetsPlugin): try: call([cmd, '-v']) self.command = cmd - except OSError as exc: + except OSError: pass if not self.command: raise ui.UserError( @@ -97,7 +97,7 @@ class ReplayGainPlugin(BeetsPlugin): [MediaFile(syspath(item.path)) for item in album.items()] self.write_rgain(media_files, - self.compute_rgain(media_files), + self.compute_rgain(media_files, True), True) except (FileTypeError, UnreadableFileError, @@ -123,19 +123,6 @@ class ReplayGainPlugin(BeetsPlugin): self.albumgain) - def get_recommended_gains(self, media_paths): - '''Returns recommended track and album gain values''' - rgain_out = call([self.command, '-o', '-d', str(self.gain_offset)] + - media_paths) - rgain_out = rgain_out.strip('\n').split('\n') - keys = rgain_out[0].split('\t')[1:] - tracks_mp3_gain = [dict(zip(keys, - [float(x) for x in l.split('\t')[1:]])) - for l in rgain_out[1:-1]] - album_mp3_gain = int(rgain_out[-1].split('\t')[1]) - return [tracks_mp3_gain, album_mp3_gain] - - def parse_tool_output(self, text): """Given the tab-delimited output from an invocation of mp3gain or aacgain, parse the text and return a list of dictionaries @@ -157,7 +144,7 @@ class ReplayGainPlugin(BeetsPlugin): return out - def reduce_gain_for_noclip(self, track_gains, albumgain): + def reduce_gain_for_noclip(self, track_peaks, album_gain): '''Reduce albumgain value until no song is clipped. No command switch give you the max no-clip in album mode. So we consider the recommended gain and decrease it until no song is @@ -165,15 +152,14 @@ class ReplayGainPlugin(BeetsPlugin): Formula found at: http://www.hydrogenaudio.org/forums/lofiversion/index.php/t10630.html ''' - - if albumgain > 0: - maxpcm = max([t['Max Amplitude'] for t in track_gains]) - while (maxpcm * (2**(albumgain/4.0)) > 32767): - albumgain -= 1 - return albumgain + if album_gain > 0: + maxpcm = max(track_peaks) + while (maxpcm * (2 ** (album_gain / 4.0)) > 32767): + album_gain -= 1 + return album_gain - def compute_rgain(self, media_files): + def compute_rgain(self, media_files, album=False): """Compute ReplayGain values and return a list of results dictionaries as given by `parse_tool_output`. """ @@ -187,12 +173,6 @@ class ReplayGainPlugin(BeetsPlugin): media_paths = [syspath(mf.path) for mf in media_files] - if self.albumgain: - track_gains, album_gain = self.get_recommended_gains(media_paths) - if self.noclip: - self.gain_offset = self.reduce_gain_for_noclip(track_gains, - album_gain) - # Construct shell command. The "-o" option makes the output # easily parseable (tab-delimited). "-s s" forces gain # recalculation even if tags are already present and disables @@ -215,7 +195,16 @@ class ReplayGainPlugin(BeetsPlugin): log.debug('replaygain: analyzing {0} files'.format(len(media_files))) output = call(cmd) log.debug('replaygain: analysis finished') - return self.parse_tool_output(output) + results = self.parse_tool_output(output) + + # Adjust for noclip mode. + if album and self.noclip: + album_gain = results[-1]['gain'] + track_peaks = [r['peak'] for r in results[:-1]] + album_gain = self.reduce_gain_for_noclip(track_peaks, album_gain) + results[-1]['gain'] = album_gain + + return results def write_rgain(self, media_files, rgain_infos, album=False): From 83f3069d57ce6de3e4f562f8655022dbf552bdf5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Oct 2012 20:11:20 -0700 Subject: [PATCH 53/85] replaygain: remove albumgain setting We now always calculate album gain when importing an album. This is "free" (no performance cost) now and players are free to ignore the setting if they so choose. --- beetsplug/replaygain.py | 16 +++++++--------- docs/plugins/replaygain.rst | 13 +++++-------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index e2b15908f..e12208ed8 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -36,9 +36,9 @@ class ReplayGainError(Exception): """ def call(args): - """Execute the command indicated by `args` (an array of strings) and - return the command's output. The stderr stream is ignored. If the command - exits abnormally, a ReplayGainError is raised. + """Execute the command indicated by `args` (a list of strings) and + return the command's output. The stderr stream is ignored. If the + command exits abnormally, a ReplayGainError is raised. """ try: with open(os.devnull, 'w') as devnull: @@ -62,8 +62,6 @@ class ReplayGainPlugin(BeetsPlugin): 'noclip', True, bool) self.apply_gain = ui.config_val(config,'replaygain', 'apply_gain', False, bool) - self.albumgain = ui.config_val(config,'replaygain', - 'albumgain', False, bool) target_level = float(ui.config_val(config,'replaygain', 'targetlevel', DEFAULT_REFERENCE_LOUDNESS)) self.gain_offset = int(target_level - DEFAULT_REFERENCE_LOUDNESS) @@ -114,13 +112,13 @@ class ReplayGainPlugin(BeetsPlugin): log.error("failed to calculate replaygain: %s ", e) - def requires_gain(self, mf): + def requires_gain(self, mf, album=False): '''Does the gain need to be computed?''' return self.overwrite or \ (not mf.rg_track_gain or not mf.rg_track_peak) or \ ((not mf.rg_album_gain or not mf.rg_album_peak) and \ - self.albumgain) + album) def parse_tool_output(self, text): @@ -145,7 +143,7 @@ class ReplayGainPlugin(BeetsPlugin): def reduce_gain_for_noclip(self, track_peaks, album_gain): - '''Reduce albumgain value until no song is clipped. + '''Reduce album gain value until no song is clipped. No command switch give you the max no-clip in album mode. So we consider the recommended gain and decrease it until no song is clipped when applying the gain. @@ -167,7 +165,7 @@ class ReplayGainPlugin(BeetsPlugin): # recalculation. This way, if any file among an album's tracks # needs recalculation, we still get an accurate album gain # value. - if all([not self.requires_gain(mf) for mf in media_files]): + if all([not self.requires_gain(mf, album) for mf in media_files]): log.debug('replaygain: no gain to compute') return diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 6eead058b..7811dcc24 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -45,14 +45,11 @@ for the plugin in your :doc:`/reference/config`, like so:: The target level can be modified to any target dB with the ``targetlevel`` option (default: 89 dB). -ReplayGain can normalize an entire album's loudness while allowing the dynamics -from song to song on the album to remain intact. This is called "album gain" and -is especially important for classical music albums with large loudness ranges. -"Track gain," in which each song is considered independently, is used by -default. To override this, use the ``albumgain`` option:: - - [replaygain] - albumgain: yes +When analyzing albums, this plugin calculates both an "album gain" alongside +individual track gains. Album gain normalizes an entire album's loudness while +allowing the dynamics from song to song on the album to remain intact. This is +especially important for classical music albums with large loudness ranges. +Players can choose which gain (track or album) to honor. If you use a player that does not support ReplayGain specifications, you can force the volume normalization by applying the gain to the file via the From 3a4e1ca4f7e7bf2cd27c162c31065ca8a6957dd5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Oct 2012 20:28:24 -0700 Subject: [PATCH 54/85] replaygain: add fields to Item This has been a long time coming, but we now finally keep track of ReplayGain values in the database. This is an intermediate step toward a refactoring of the RG plugin; at the moment, these values are not actually saved! --- beets/library.py | 6 ++++ beetsplug/replaygain.py | 73 ++++++++++++++-------------------------- test/rsrc/test.blb | Bin 7168 -> 7168 bytes 3 files changed, 32 insertions(+), 47 deletions(-) diff --git a/beets/library.py b/beets/library.py index dff5e013c..b634d3951 100644 --- a/beets/library.py +++ b/beets/library.py @@ -89,6 +89,10 @@ ITEM_FIELDS = [ ('albumdisambig', 'text', True, True), ('disctitle', 'text', True, True), ('encoder', 'text', True, True), + ('rg_track_gain', 'real', True, True), + ('rg_track_peak', 'real', True, True), + ('rg_album_gain', 'real', True, True), + ('rg_album_peak', 'real', True, True), ('length', 'real', False, True), ('bitrate', 'int', False, True), @@ -133,6 +137,8 @@ ALBUM_FIELDS = [ ('albumstatus', 'text', True), ('media', 'text', True), ('albumdisambig', 'text', True), + ('rg_album_gain', 'real', True), + ('rg_album_peak', 'real', True), ] ALBUM_KEYS = [f[0] for f in ALBUM_FIELDS] ALBUM_KEYS_ITEM = [f[0] for f in ALBUM_FIELDS if f[2]] diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index e12208ed8..196fb9220 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -24,7 +24,6 @@ import os from beets import ui from beets.plugins import BeetsPlugin -from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError from beets.util import syspath log = logging.getLogger('beets') @@ -90,34 +89,21 @@ class ReplayGainPlugin(BeetsPlugin): def album_imported(self, lib, album, config): - try: - media_files = \ - [MediaFile(syspath(item.path)) for item in album.items()] - - self.write_rgain(media_files, - self.compute_rgain(media_files, True), - True) - - except (FileTypeError, UnreadableFileError, - TypeError, ValueError) as e: - log.error("failed to calculate replaygain: %s ", e) + items = list(album.items()) + self.store_gain(items, + self.compute_rgain(items, True), + True) def item_imported(self, lib, item, config): - try: - mf = MediaFile(syspath(item.path)) - self.write_rgain([mf], self.compute_rgain([mf])) - except (FileTypeError, UnreadableFileError, - TypeError, ValueError) as e: - log.error("failed to calculate replaygain: %s ", e) + self.store_gain([item], self.compute_rgain([item])) - def requires_gain(self, mf, album=False): - '''Does the gain need to be computed?''' - + def requires_gain(self, item, album=False): + """Does the gain need to be computed?""" return self.overwrite or \ - (not mf.rg_track_gain or not mf.rg_track_peak) or \ - ((not mf.rg_album_gain or not mf.rg_album_peak) and \ + (not item.rg_track_gain or not item.rg_track_peak) or \ + ((not item.rg_album_gain or not item.rg_album_peak) and \ album) @@ -157,7 +143,7 @@ class ReplayGainPlugin(BeetsPlugin): return album_gain - def compute_rgain(self, media_files, album=False): + def compute_rgain(self, items, album=False): """Compute ReplayGain values and return a list of results dictionaries as given by `parse_tool_output`. """ @@ -165,12 +151,10 @@ class ReplayGainPlugin(BeetsPlugin): # recalculation. This way, if any file among an album's tracks # needs recalculation, we still get an accurate album gain # value. - if all([not self.requires_gain(mf, album) for mf in media_files]): + if all([not self.requires_gain(i, album) for i in items]): log.debug('replaygain: no gain to compute') return - media_paths = [syspath(mf.path) for mf in media_files] - # Construct shell command. The "-o" option makes the output # easily parseable (tab-delimited). "-s s" forces gain # recalculation even if tags are already present and disables @@ -188,9 +172,9 @@ class ReplayGainPlugin(BeetsPlugin): # Lossless audio adjustment. cmd = cmd + ['-r'] cmd = cmd + ['-d', str(self.gain_offset)] - cmd = cmd + media_paths + cmd = cmd + [syspath(i.path) for i in items] - log.debug('replaygain: analyzing {0} files'.format(len(media_files))) + log.debug('replaygain: analyzing {0} files'.format(len(items))) output = call(cmd) log.debug('replaygain: analysis finished') results = self.parse_tool_output(output) @@ -205,28 +189,23 @@ class ReplayGainPlugin(BeetsPlugin): return results - def write_rgain(self, media_files, rgain_infos, album=False): + def store_gain(self, items, rgain_infos, album=False): """Write computed gain values for each media file. """ if album: - assert len(rgain_infos) == len(media_files) + 1 + assert len(rgain_infos) == len(items) + 1 album_info = rgain_infos[-1] - for mf, info in zip(media_files, rgain_infos): - try: - mf.rg_track_gain = info['gain'] - mf.rg_track_peak = info['peak'] + for item, info in zip(items, rgain_infos): + item.rg_track_gain = info['gain'] + item.rg_track_peak = info['peak'] - if album: - mf.rg_album_gain = album_info['gain'] - mf.rg_album_peak = album_info['peak'] + if album: + item.rg_album_gain = album_info['gain'] + item.rg_album_peak = album_info['peak'] - log.debug('replaygain: writing track gain {0}, peak {1}; ' - 'album gain {2}, peak {3}'.format( - mf.rg_track_gain, mf.rg_track_peak, - mf.rg_album_gain, mf.rg_album_peak - )) - mf.save() - - except UnreadableFileError: - log.error("replaygain: write failed for %s" % (mf.title)) + log.debug('replaygain: applying track gain {0}, peak {1}; ' + 'album gain {2}, peak {3}'.format( + item.rg_track_gain, item.rg_track_peak, + item.rg_album_gain, item.rg_album_peak + )) diff --git a/test/rsrc/test.blb b/test/rsrc/test.blb index 3fe41c8ae885de67124b0b08de219233766b50da..5daf4a2f49489e5a3c9866b590f23deb082b0bc1 100644 GIT binary patch delta 230 zcmZp$Xt0ySjGAD>i7{w1qX-+zW;Tgm%m4$mLNfpW delta 126 zcmZp$Xt00h_GDrgmy~2|%ia8(NrZ879~1lJ Y>wGSgONEmdH8(Sgu(51rlla990LL{Qc>n+a From 95910a366b42ec5e5c11c41b4846cca103134f58 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Oct 2012 21:43:27 -0700 Subject: [PATCH 55/85] replaygain: run in import stage This is the "new way" to post-process files on import (superseding the imported events). --- beetsplug/replaygain.py | 45 +++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 196fb9220..f82928693 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -51,8 +51,8 @@ class ReplayGainPlugin(BeetsPlugin): """Provides ReplayGain analysis. """ def __init__(self): - self.register_listener('album_imported', self.album_imported) - self.register_listener('item_imported', self.item_imported) + super(ReplayGainPlugin, self).__init__() + self.import_stages = [self.imported] def configure(self, config): self.overwrite = ui.config_val(config,'replaygain', @@ -87,16 +87,17 @@ class ReplayGainPlugin(BeetsPlugin): 'no replaygain command found: install mp3gain or aacgain' ) + def imported(self, config, task): + """Our import stage function.""" + if task.is_album: + album = config.lib.get_album(task.album_id) + items = list(album.items()) + else: + items = [task.item] - def album_imported(self, lib, album, config): - items = list(album.items()) - self.store_gain(items, - self.compute_rgain(items, True), - True) - - - def item_imported(self, lib, item, config): - self.store_gain([item], self.compute_rgain([item])) + results = self.compute_rgain(items, task.is_album) + self.store_gain(config.lib, items, results, + album if task.is_album else None) def requires_gain(self, item, album=False): @@ -189,23 +190,23 @@ class ReplayGainPlugin(BeetsPlugin): return results - def store_gain(self, items, rgain_infos, album=False): - """Write computed gain values for each media file. + def store_gain(self, lib, items, rgain_infos, album=None): + """Store computed ReplayGain values to the Items and the Album + (if it is provided). """ - if album: - assert len(rgain_infos) == len(items) + 1 - album_info = rgain_infos[-1] - for item, info in zip(items, rgain_infos): item.rg_track_gain = info['gain'] item.rg_track_peak = info['peak'] + lib.store(item) - if album: - item.rg_album_gain = album_info['gain'] - item.rg_album_peak = album_info['peak'] - - log.debug('replaygain: applying track gain {0}, peak {1}; ' + log.debug('replaygain: applied track gain {0}, peak {1}; ' 'album gain {2}, peak {3}'.format( item.rg_track_gain, item.rg_track_peak, item.rg_album_gain, item.rg_album_peak )) + + if album: + assert len(rgain_infos) == len(items) + 1 + album_info = rgain_infos[-1] + album.rg_album_gain = album_info['gain'] + album.rg_album_peak = album_info['peak'] From 1a261db918379249877221cbfce618be399f9e50 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Oct 2012 21:47:17 -0700 Subject: [PATCH 56/85] replaygain: style and light refactoring --- beetsplug/replaygain.py | 105 ++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 58 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index f82928693..49dfb40be 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1,22 +1,16 @@ -#Copyright (c) 2012, Fabrice Laporte +# This file is part of beets. +# Copyright 2012, Fabrice Laporte. # -#Permission is hereby granted, free of charge, to any person obtaining a copy -#of this software and associated documentation files (the "Software"), to deal -#in the Software without restriction, including without limitation the rights -#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -#copies of the Software, and to permit persons to whom the Software is -#furnished to do so, subject to the following conditions: +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -#The above copyright notice and this permission notice shall be included in -#all copies or substantial portions of the Software. -# -#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -#THE SOFTWARE. +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. import logging import subprocess @@ -47,6 +41,40 @@ def call(args): "{0} exited with status {1}".format(args[0], e.returncode) ) +def reduce_gain_for_noclip(track_peaks, album_gain): + """Reduce album gain value until no song is clipped. + No command switch give you the max no-clip in album mode. + So we consider the recommended gain and decrease it until no song is + clipped when applying the gain. + Formula found at: + http://www.hydrogenaudio.org/forums/lofiversion/index.php/t10630.html + """ + if album_gain > 0: + maxpcm = max(track_peaks) + while (maxpcm * (2 ** (album_gain / 4.0)) > 32767): + album_gain -= 1 + return album_gain + +def parse_tool_output(text): + """Given the tab-delimited output from an invocation of mp3gain + or aacgain, parse the text and return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + for line in text.split('\n'): + parts = line.split('\t') + if len(parts) != 6 or parts[0] == 'File': + continue + out.append({ + 'file': parts[0], + 'mp3gain': int(parts[1]), + 'gain': float(parts[2]), + 'peak': float(parts[3]), + 'maxgain': int(parts[4]), + 'mingain': int(parts[5]), + }) + return out + class ReplayGainPlugin(BeetsPlugin): """Provides ReplayGain analysis. """ @@ -98,7 +126,6 @@ class ReplayGainPlugin(BeetsPlugin): results = self.compute_rgain(items, task.is_album) self.store_gain(config.lib, items, results, album if task.is_album else None) - def requires_gain(self, item, album=False): """Does the gain need to be computed?""" @@ -107,43 +134,6 @@ class ReplayGainPlugin(BeetsPlugin): ((not item.rg_album_gain or not item.rg_album_peak) and \ album) - - def parse_tool_output(self, text): - """Given the tab-delimited output from an invocation of mp3gain - or aacgain, parse the text and return a list of dictionaries - containing information about each analyzed file. - """ - out = [] - for line in text.split('\n'): - parts = line.split('\t') - if len(parts) != 6 or parts[0] == 'File': - continue - out.append({ - 'file': parts[0], - 'mp3gain': int(parts[1]), - 'gain': float(parts[2]), - 'peak': float(parts[3]), - 'maxgain': int(parts[4]), - 'mingain': int(parts[5]), - }) - return out - - - def reduce_gain_for_noclip(self, track_peaks, album_gain): - '''Reduce album gain value until no song is clipped. - No command switch give you the max no-clip in album mode. - So we consider the recommended gain and decrease it until no song is - clipped when applying the gain. - Formula found at: - http://www.hydrogenaudio.org/forums/lofiversion/index.php/t10630.html - ''' - if album_gain > 0: - maxpcm = max(track_peaks) - while (maxpcm * (2 ** (album_gain / 4.0)) > 32767): - album_gain -= 1 - return album_gain - - def compute_rgain(self, items, album=False): """Compute ReplayGain values and return a list of results dictionaries as given by `parse_tool_output`. @@ -178,18 +168,17 @@ class ReplayGainPlugin(BeetsPlugin): log.debug('replaygain: analyzing {0} files'.format(len(items))) output = call(cmd) log.debug('replaygain: analysis finished') - results = self.parse_tool_output(output) + results = parse_tool_output(output) # Adjust for noclip mode. if album and self.noclip: album_gain = results[-1]['gain'] track_peaks = [r['peak'] for r in results[:-1]] - album_gain = self.reduce_gain_for_noclip(track_peaks, album_gain) + album_gain = reduce_gain_for_noclip(track_peaks, album_gain) results[-1]['gain'] = album_gain return results - def store_gain(self, lib, items, rgain_infos, album=None): """Store computed ReplayGain values to the Items and the Album (if it is provided). From 0ab3426bd92c4c6994b7b8e327b8e7d68bde0181 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Oct 2012 22:12:38 -0700 Subject: [PATCH 57/85] replaygain: add command You can now disable automatic analysis and instead use a command to manually analyze albums or tracks. --- beetsplug/replaygain.py | 64 +++++++++++++++++++++++++++++++------ docs/plugins/replaygain.rst | 19 +++++++++++ 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 49dfb40be..fc88d80b0 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -19,6 +19,7 @@ import os from beets import ui from beets.plugins import BeetsPlugin from beets.util import syspath +from beets.ui import commands log = logging.getLogger('beets') @@ -83,15 +84,18 @@ class ReplayGainPlugin(BeetsPlugin): self.import_stages = [self.imported] def configure(self, config): - self.overwrite = ui.config_val(config,'replaygain', + self.overwrite = ui.config_val(config, 'replaygain', 'overwrite', False, bool) - self.noclip = ui.config_val(config,'replaygain', - 'noclip', True, bool) - self.apply_gain = ui.config_val(config,'replaygain', - 'apply_gain', False, bool) - target_level = float(ui.config_val(config,'replaygain', - 'targetlevel', DEFAULT_REFERENCE_LOUDNESS)) + self.noclip = ui.config_val(config, 'replaygain', + 'noclip', True, bool) + self.apply_gain = ui.config_val(config, 'replaygain', + 'apply_gain', False, bool) + target_level = float(ui.config_val(config, 'replaygain', + 'targetlevel', + DEFAULT_REFERENCE_LOUDNESS)) self.gain_offset = int(target_level - DEFAULT_REFERENCE_LOUDNESS) + self.automatic = ui.config_val(config, 'replaygain', + 'automatic', True, bool) self.command = ui.config_val(config,'replaygain','command', None) if self.command: @@ -117,6 +121,9 @@ class ReplayGainPlugin(BeetsPlugin): def imported(self, config, task): """Our import stage function.""" + if not self.automatic: + return + if task.is_album: album = config.lib.get_album(task.album_id) items = list(album.items()) @@ -124,8 +131,47 @@ class ReplayGainPlugin(BeetsPlugin): items = [task.item] results = self.compute_rgain(items, task.is_album) - self.store_gain(config.lib, items, results, - album if task.is_album else None) + if results: + self.store_gain(config.lib, items, results, + album if task.is_album else None) + + def commands(self): + """Provide a ReplayGain command.""" + def func(lib, config, opts, args): + write = ui.config_val(config, 'beets', 'import_write', + commands.DEFAULT_IMPORT_WRITE, bool) + + if opts.album: + # Analyze albums. + for album in lib.albums(ui.decargs(args)): + log.info('analyzing {0} - {1}'.format(album.albumartist, + album.album)) + items = list(album.items()) + results = self.compute_rgain(items, True) + if results: + self.store_gain(lib, items, results, album) + + if write: + for item in items: + item.write() + + else: + # Analyze individual tracks. + for item in lib.items(ui.decargs(args)): + log.info('analyzing {0} - {1}'.format(item.artist, + item.title)) + results = self.compute_rgain([item], False) + if results: + self.store_gain(lib, [item], results, None) + + if write: + item.write() + + cmd = ui.Subcommand('replaygain', help='analyze for ReplayGain') + cmd.parser.add_option('-a', '--album', action='store_true', + help='analyze albums instead of tracks') + cmd.func = func + return [cmd] def requires_gain(self, item, album=False): """Does the gain need to be computed?""" diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 7811dcc24..a60062843 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -58,3 +58,22 @@ transcoding involved. The use of ReplayGain can cause clipping if the average volume of a song is below the target level. By default, a "prevent clipping" option named ``noclip`` is enabled to reduce the amount of ReplayGain adjustment to whatever amount would keep clipping from occurring. + +Manual Analysis +--------------- + +By default, the plugin will analyze all items an albums as they are implemented. +However, you can also manually analyze files that are already in your library. +Use the ``beet replaygain`` command:: + + $ beet replaygain [-a] [QUERY] + +The ``-a`` flag analyzes whole albums instead of individual tracks. Provide a +query (see :doc:`/reference/query`) to indicate which items or albums to +analyze. + +ReplayGain analysis is not fast, so you may want to disable it during import. +Use the ``automatic`` config option to control this:: + + [replaygain] + automatic: no From 9d4d95fa272d71111f085d7d6a30fb58498722d2 Mon Sep 17 00:00:00 2001 From: Mike Kazantsev Date: Thu, 18 Oct 2012 21:43:50 +0600 Subject: [PATCH 58/85] readme: fix readthedocs.org links --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 9c336e737..8780e95a8 100644 --- a/README.rst +++ b/README.rst @@ -31,11 +31,11 @@ imagine for your music collection. Via `plugins`_, beets becomes a panacea: If beets doesn't do what you want yet, `writing your own plugin`_ is shockingly simple if you know a little Python. -.. _plugins: http://readthedocs.org/docs/beets/-/plugins/ +.. _plugins: http://beets.readthedocs.org/en/latest/plugins/ .. _MPD: http://mpd.wikia.com/ .. _MusicBrainz music collection: http://musicbrainz.org/show/collection/ .. _writing your own plugin: - http://readthedocs.org/docs/beets/-/plugins/#writing-plugins + http://beets.readthedocs.org/en/latest/plugins/#writing-plugins .. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html @@ -50,7 +50,7 @@ cutting edge, type ``pip install beets==dev`` for the `latest source`_.) Check out the `Getting Started`_ guide to learn more about installing and using beets. .. _its Web site: http://beets.radbox.org/ -.. _Getting Started: http://readthedocs.org/docs/beets/-/guides/main.html +.. _Getting Started: http://beets.readthedocs.org/en/latest/guides/main.html .. _@b33ts: http://twitter.com/b33ts/ .. _latest source: https://github.com/sampsyo/beets/tarball/master#egg=beets-dev From 58ba4b3d75ce1716a05f2ad2ae67501e6eb6d79c Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Thu, 18 Oct 2012 18:35:25 +0200 Subject: [PATCH 59/85] convert: fix album art embedding --- beetsplug/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 97584993a..270ec8df4 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -94,7 +94,7 @@ def convert_item(lib, dest_dir): artpath = lib.get_album(item).artpath if artpath and conf['embed']: - _embed(artpath, [dest_item]) + _embed(artpath, [item]) def convert_func(lib, config, opts, args): From 2c38c15fb845e1f99155d8d779da29dd0efcdb1b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 18 Oct 2012 11:33:13 -0700 Subject: [PATCH 60/85] replaygain: apply album gain in album mode --- beetsplug/replaygain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index fc88d80b0..6932afbed 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -201,13 +201,13 @@ class ReplayGainPlugin(BeetsPlugin): cmd = [self.command, '-o', '-s', 's'] if self.noclip: # Adjust to avoid clipping. - cmd = cmd + ['-k'] + cmd = cmd + ['-k'] else: # Disable clipping warning. cmd = cmd + ['-c'] if self.apply_gain: # Lossless audio adjustment. - cmd = cmd + ['-r'] + cmd = cmd + ['-a' if album else '-r'] cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] From 4f164fb83ed9bd92f9b3bc473d7e6b521b03d10c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 19 Oct 2012 10:05:06 -0700 Subject: [PATCH 61/85] windows: use UTF-8 in displayable_path --- beets/util/__init__.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 9c8b7f35e..5e7cb8e38 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -242,6 +242,20 @@ def components(path, pathmod=None): return comps +def _fsencoding(): + """Get the system's filesystem encoding. On Windows, this is always + UTF-8 (not MBCS). + """ + encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + if encoding == 'mbcs': + # On Windows, a broken encoding known to Python as "MBCS" is + # used for the filesystem. However, we only use the Unicode API + # for Windows paths, so the encoding is actually immaterial so + # we can avoid dealing with this nastiness. We arbitrarily + # choose UTF-8. + encoding = 'utf8' + return encoding + def bytestring_path(path): """Given a path, which is either a str or a unicode, returns a str path (ensuring that we never deal with Unicode pathnames). @@ -251,16 +265,8 @@ def bytestring_path(path): return path # Try to encode with default encodings, but fall back to UTF8. - encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() - if encoding == 'mbcs': - # On Windows, a broken encoding known to Python as "MBCS" is - # used for the filesystem. However, we only use the Unicode API - # for Windows paths, so the encoding is actually immaterial so - # we can avoid dealing with this nastiness. We arbitrarily - # choose UTF-8. - encoding = 'utf8' try: - return path.encode(encoding) + return path.encode(_fsencoding()) except (UnicodeError, LookupError): return path.encode('utf8') @@ -274,9 +280,8 @@ def displayable_path(path): # A non-string object: just get its unicode representation. return unicode(path) - encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() try: - return path.decode(encoding, 'ignore') + return path.decode(_fsencoding(), 'ignore') except (UnicodeError, LookupError): return path.decode('utf8', 'ignore') From 492f168124ec29db715770fbe3496ef61d6845cc Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Sat, 20 Oct 2012 13:25:25 +0200 Subject: [PATCH 62/85] convert: Fix problem with "threads" config option --- beetsplug/convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 270ec8df4..fecaa9fa0 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -122,8 +122,8 @@ def convert_func(lib, config, opts, args): class ConvertPlugin(BeetsPlugin): def configure(self, config): conf['dest'] = ui.config_val(config, 'convert', 'dest', None) - conf['threads'] = ui.config_val(config, 'convert', 'threads', - util.cpu_count()) + conf['threads'] = int(ui.config_val(config, 'convert', 'threads', + util.cpu_count())) conf['flac'] = ui.config_val(config, 'convert', 'flac', 'flac') conf['lame'] = ui.config_val(config, 'convert', 'lame', 'lame') conf['opts'] = ui.config_val(config, 'convert', From 12cae9ee93a35f51dbd26bcbf435cd0a9ff231d6 Mon Sep 17 00:00:00 2001 From: Mike Kazantsev Date: Sat, 20 Oct 2012 20:07:27 +0600 Subject: [PATCH 63/85] library: log path, reading of which has raised an exception --- beets/library.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index b634d3951..3131c7b79 100644 --- a/beets/library.py +++ b/beets/library.py @@ -273,7 +273,10 @@ class Item(object): read_path = self.path else: read_path = normpath(read_path) - f = MediaFile(syspath(read_path)) + try: f = MediaFile(syspath(read_path)) + except Exception as err: + log.error('Failed processing file: {!r}'.format(read_path)) + raise for key in ITEM_KEYS_META: setattr(self, key, getattr(f, key)) From 609e57f0a0234173e75542a78beab5859cfc4b9e Mon Sep 17 00:00:00 2001 From: Mike Kazantsev Date: Sat, 20 Oct 2012 20:10:29 +0600 Subject: [PATCH 64/85] library: don't set/update item mtime if read-path is passed to read() --- beets/library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index b634d3951..24ac1df0c 100644 --- a/beets/library.py +++ b/beets/library.py @@ -277,12 +277,13 @@ class Item(object): for key in ITEM_KEYS_META: setattr(self, key, getattr(f, key)) - self.path = read_path # Database's mtime should now reflect the on-disk value. if read_path == self.path: self.mtime = self.current_mtime() + self.path = read_path + def write(self): """Writes the item's metadata to the associated file. """ From 8b07ea157d6ee978002d4a6f2d720e04a0433fbd Mon Sep 17 00:00:00 2001 From: Mike Kazantsev Date: Sat, 20 Oct 2012 21:45:14 +0600 Subject: [PATCH 65/85] Fix inconsistent three-space indentation --- beets/ui/commands.py | 2 +- beets/util/pipeline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index d818b397f..2534e6f76 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -489,7 +489,7 @@ def manual_id(singleton): # Find the first thing that looks like a UUID/MBID. match = re.search('[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', entry) if match: - return match.group() + return match.group() else: log.error('Invalid MBID.') return None diff --git a/beets/util/pipeline.py b/beets/util/pipeline.py index f9d3b27e9..8d2bb0dd1 100644 --- a/beets/util/pipeline.py +++ b/beets/util/pipeline.py @@ -432,7 +432,7 @@ if __name__ == '__main__': print('processing %i' % num) time.sleep(3) if num == 3: - raise Exception() + raise Exception() num = yield num * 2 def exc_consume(): while True: From f8cf3817fc264d1fa8afec2b07b9d72bd7d9e1ab Mon Sep 17 00:00:00 2001 From: Mike Kazantsev Date: Sat, 20 Oct 2012 22:30:12 +0600 Subject: [PATCH 66/85] ui: use configured format when printing album/item from all commands --- beets/ui/commands.py | 68 +++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index d818b397f..f6d1b0a28 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -84,6 +84,36 @@ def _showdiff(field, oldval, newval, color): print_(u' %s: %s -> %s' % (field, oldval, newval)) +DEFAULT_LIST_FORMAT_ITEM = '$artist - $album - $title' +DEFAULT_LIST_FORMAT_ALBUM = '$albumartist - $album' + +def _pick_format(config=None, album=False, fmt=None): + """Pick album / item printing format from passed arguments, + falling back to config options and defaults.""" + if not fmt and not config: + fmt = DEFAULT_LIST_FORMAT_ALBUM \ + if album else DEFAULT_LIST_FORMAT_ITEM + elif not fmt: + if album: + fmt = ui.config_val(config, 'beets', 'list_format_album', + DEFAULT_LIST_FORMAT_ALBUM) + else: + fmt = ui.config_val(config, 'beets', 'list_format_item', + DEFAULT_LIST_FORMAT_ITEM) + return fmt + +def _format_and_print(obj, lib, album=False, fmt=None): + """Print object according to specified format.""" + if not fmt: + fmt = _pick_format(album=album) + template = Template(fmt) + if album: + print_(obj.evaluate_template(template)) + else: + print_(obj.evaluate_template(template, lib=lib)) + + + # fields: Shows a list of available fields for queries and format strings. fields_cmd = ui.Subcommand('fields', help='show fields available for queries and format strings') @@ -808,9 +838,6 @@ default_commands.append(import_cmd) # list: Query and show library contents. -DEFAULT_LIST_FORMAT_ITEM = '$artist - $album - $title' -DEFAULT_LIST_FORMAT_ALBUM = '$albumartist - $album' - def list_items(lib, query, album, path, fmt): """Print out items in lib matching query. If album, then search for albums instead of single items. If path, print the matched objects' @@ -839,15 +866,7 @@ list_cmd.parser.add_option('-p', '--path', action='store_true', list_cmd.parser.add_option('-f', '--format', action='store', help='print with custom format', default=None) def list_func(lib, config, opts, args): - fmt = opts.format - if not fmt: - # If no format is specified, fall back to a default. - if opts.album: - fmt = ui.config_val(config, 'beets', 'list_format_album', - DEFAULT_LIST_FORMAT_ALBUM) - else: - fmt = ui.config_val(config, 'beets', 'list_format_item', - DEFAULT_LIST_FORMAT_ITEM) + fmt = _pick_format(config, opts.album, opts.format) list_items(lib, decargs(args), opts.album, opts.path, fmt) list_cmd.func = list_func default_commands.append(list_cmd) @@ -855,7 +874,7 @@ default_commands.append(list_cmd) # update: Update library contents according to on-disk tags. -def update_items(lib, query, album, move, color, pretend): +def update_items(lib, query, album, move, color, pretend, fmt=None): """For all the items matched by the query, update the library to reflect the item's embedded tags. """ @@ -867,7 +886,7 @@ def update_items(lib, query, album, move, color, pretend): for item in items: # Item deleted? if not os.path.exists(syspath(item.path)): - print_(u'X %s - %s' % (item.artist, item.title)) + _format_and_print(item, lib, fmt=fmt) if not pretend: lib.remove(item, True) affected_albums.add(item.album_id) @@ -899,7 +918,7 @@ def update_items(lib, query, album, move, color, pretend): changes[key] = old_data[key], getattr(item, key) if changes: # Something changed. - print_(u'* %s - %s' % (item.artist, item.title)) + _format_and_print(item, lib, fmt=fmt) for key, (oldval, newval) in changes.iteritems(): _showdiff(key, oldval, newval, color) @@ -953,14 +972,14 @@ update_cmd.parser.add_option('-p', '--pretend', action='store_true', help="show all changes but do nothing") def update_func(lib, config, opts, args): color = ui.config_val(config, 'beets', 'color', DEFAULT_COLOR, bool) - update_items(lib, decargs(args), opts.album, opts.move, color, opts.pretend) + update_items(lib, config, decargs(args), opts.album, opts.move, color, opts.pretend) update_cmd.func = update_func default_commands.append(update_cmd) # remove: Remove items from library, delete files. -def remove_items(lib, query, album, delete=False): +def remove_items(lib, query, album, delete=False, fmt=None): """Remove items matching query from lib. If album, then match and remove whole albums. If delete, also remove files from disk. """ @@ -969,7 +988,7 @@ def remove_items(lib, query, album, delete=False): # Show all the items. for item in items: - print_(item.artist + ' - ' + item.album + ' - ' + item.title) + _format_and_print(item, lib, fmt=fmt) # Confirm with user. print_() @@ -997,7 +1016,7 @@ remove_cmd.parser.add_option("-d", "--delete", action="store_true", remove_cmd.parser.add_option('-a', '--album', action='store_true', help='match albums instead of tracks') def remove_func(lib, config, opts, args): - remove_items(lib, decargs(args), opts.album, opts.delete) + remove_items(lib, config, decargs(args), opts.album, opts.delete) remove_cmd.func = remove_func default_commands.append(remove_cmd) @@ -1066,7 +1085,7 @@ default_commands.append(version_cmd) # modify: Declaratively change metadata. -def modify_items(lib, mods, query, write, move, album, color, confirm): +def modify_items(lib, mods, query, write, move, album, color, confirm, fmt=None): """Modifies matching items according to key=value assignments.""" # Parse key=value specifications into a dictionary. allowed_keys = library.ALBUM_KEYS if album else library.ITEM_KEYS_WRITABLE @@ -1085,10 +1104,7 @@ def modify_items(lib, mods, query, write, move, album, color, confirm): print_('Modifying %i %ss.' % (len(objs), 'album' if album else 'item')) for obj in objs: # Identify the changed object. - if album: - print_(u'* %s - %s' % (obj.albumartist, obj.album)) - else: - print_(u'* %s - %s' % (obj.artist, obj.title)) + _format_and_print(obj, lib, album=album, fmt=fmt) # Show each change. for field, value in fsets.iteritems(): @@ -1149,8 +1165,8 @@ def modify_func(lib, config, opts, args): ui.config_val(config, 'beets', 'import_write', DEFAULT_IMPORT_WRITE, bool) color = ui.config_val(config, 'beets', 'color', DEFAULT_COLOR, bool) - modify_items(lib, mods, query, write, opts.move, opts.album, color, - not opts.yes) + modify_items(lib, config, mods, query, write, opts.move, opts.album, + color, not opts.yes) modify_cmd.func = modify_func default_commands.append(modify_cmd) From 037f2907762f1db94a9c4327ffd06b45f93c5631 Mon Sep 17 00:00:00 2001 From: Mike Kazantsev Date: Sat, 20 Oct 2012 23:04:17 +0600 Subject: [PATCH 67/85] ui: add --format option to all commands that can use it --- beets/ui/commands.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index f6d1b0a28..00468143e 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -970,9 +970,12 @@ update_cmd.parser.add_option('-M', '--nomove', action='store_false', default=True, dest='move', help="don't move files in library") update_cmd.parser.add_option('-p', '--pretend', action='store_true', help="show all changes but do nothing") +update_cmd.parser.add_option('-f', '--format', action='store', + help='print with custom format', default=None) def update_func(lib, config, opts, args): color = ui.config_val(config, 'beets', 'color', DEFAULT_COLOR, bool) - update_items(lib, config, decargs(args), opts.album, opts.move, color, opts.pretend) + fmt = _pick_format(config, opts.album, opts.format) + update_items(lib, decargs(args), opts.album, opts.move, color, opts.pretend, fmt) update_cmd.func = update_func default_commands.append(update_cmd) @@ -1015,8 +1018,11 @@ remove_cmd.parser.add_option("-d", "--delete", action="store_true", help="also remove files from disk") remove_cmd.parser.add_option('-a', '--album', action='store_true', help='match albums instead of tracks') +remove_cmd.parser.add_option('-f', '--format', action='store', + help='print with custom format', default=None) def remove_func(lib, config, opts, args): - remove_items(lib, config, decargs(args), opts.album, opts.delete) + fmt = _pick_format(config, opts.album, opts.format) + remove_items(lib, decargs(args), opts.album, opts.delete, fmt) remove_cmd.func = remove_func default_commands.append(remove_cmd) @@ -1155,6 +1161,8 @@ modify_cmd.parser.add_option('-a', '--album', action='store_true', help='modify whole albums instead of tracks') modify_cmd.parser.add_option('-y', '--yes', action='store_true', help='skip confirmation') +modify_cmd.parser.add_option('-f', '--format', action='store', + help='print with custom format', default=None) def modify_func(lib, config, opts, args): args = decargs(args) mods = [a for a in args if '=' in a] @@ -1165,8 +1173,9 @@ def modify_func(lib, config, opts, args): ui.config_val(config, 'beets', 'import_write', DEFAULT_IMPORT_WRITE, bool) color = ui.config_val(config, 'beets', 'color', DEFAULT_COLOR, bool) - modify_items(lib, config, mods, query, write, opts.move, opts.album, - color, not opts.yes) + fmt = _pick_format(config, opts.album, opts.format) + modify_items(lib, mods, query, write, opts.move, opts.album, color, + not opts.yes, fmt) modify_cmd.func = modify_func default_commands.append(modify_cmd) From 848b56e54ca6ec843cc53cde836af84fd2fbe4e5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 16:41:31 -0700 Subject: [PATCH 68/85] replaygain: remove album noclip gain adjustment as suggested by @kraYmer --- beetsplug/replaygain.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 6932afbed..ed04751b9 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -42,20 +42,6 @@ def call(args): "{0} exited with status {1}".format(args[0], e.returncode) ) -def reduce_gain_for_noclip(track_peaks, album_gain): - """Reduce album gain value until no song is clipped. - No command switch give you the max no-clip in album mode. - So we consider the recommended gain and decrease it until no song is - clipped when applying the gain. - Formula found at: - http://www.hydrogenaudio.org/forums/lofiversion/index.php/t10630.html - """ - if album_gain > 0: - maxpcm = max(track_peaks) - while (maxpcm * (2 ** (album_gain / 4.0)) > 32767): - album_gain -= 1 - return album_gain - def parse_tool_output(text): """Given the tab-delimited output from an invocation of mp3gain or aacgain, parse the text and return a list of dictionaries @@ -216,13 +202,6 @@ class ReplayGainPlugin(BeetsPlugin): log.debug('replaygain: analysis finished') results = parse_tool_output(output) - # Adjust for noclip mode. - if album and self.noclip: - album_gain = results[-1]['gain'] - track_peaks = [r['peak'] for r in results[:-1]] - album_gain = reduce_gain_for_noclip(track_peaks, album_gain) - results[-1]['gain'] = album_gain - return results def store_gain(self, lib, items, rgain_infos, album=None): From 9368075756958c608361a0d1ad50422bcbbf1528 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 16:49:52 -0700 Subject: [PATCH 69/85] replaygain: reinstate albumgain config option --- beetsplug/replaygain.py | 17 +++++++++++------ docs/plugins/replaygain.rst | 6 ++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ed04751b9..0459f3557 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -72,6 +72,8 @@ class ReplayGainPlugin(BeetsPlugin): def configure(self, config): self.overwrite = ui.config_val(config, 'replaygain', 'overwrite', False, bool) + self.albumgain = ui.config_val(config, 'replaygain', + 'albumgain', False, bool) self.noclip = ui.config_val(config, 'replaygain', 'noclip', True, bool) self.apply_gain = ui.config_val(config, 'replaygain', @@ -193,7 +195,7 @@ class ReplayGainPlugin(BeetsPlugin): cmd = cmd + ['-c'] if self.apply_gain: # Lossless audio adjustment. - cmd = cmd + ['-a' if album else '-r'] + cmd = cmd + ['-a' if album and self.albumgain else '-r'] cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] @@ -213,14 +215,17 @@ class ReplayGainPlugin(BeetsPlugin): item.rg_track_peak = info['peak'] lib.store(item) - log.debug('replaygain: applied track gain {0}, peak {1}; ' - 'album gain {2}, peak {3}'.format( - item.rg_track_gain, item.rg_track_peak, - item.rg_album_gain, item.rg_album_peak + log.debug('replaygain: applied track gain {0}, peak {1}'.format( + item.rg_track_gain, + item.rg_track_peak )) - if album: + if album and self.albumgain: assert len(rgain_infos) == len(items) + 1 album_info = rgain_infos[-1] album.rg_album_gain = album_info['gain'] album.rg_album_peak = album_info['peak'] + log.debug('replaygain: applied album gain {0}, peak {1}'.format( + album.rg_album_gain, + album.rg_album_peak + )) diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index a60062843..9b83765f6 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -45,11 +45,13 @@ for the plugin in your :doc:`/reference/config`, like so:: The target level can be modified to any target dB with the ``targetlevel`` option (default: 89 dB). -When analyzing albums, this plugin calculates both an "album gain" alongside +When analyzing albums, this plugin can calculates an "album gain" alongside individual track gains. Album gain normalizes an entire album's loudness while allowing the dynamics from song to song on the album to remain intact. This is especially important for classical music albums with large loudness ranges. -Players can choose which gain (track or album) to honor. +Players can choose which gain (track or album) to honor. By default, only +per-track gains are used; to calculate album gain also, set the ``albumgain`` +option to ``yes``. If you use a player that does not support ReplayGain specifications, you can force the volume normalization by applying the gain to the file via the From b9cc206093e039a88b1ec3091d81b196ac38cdd4 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 16:57:56 -0700 Subject: [PATCH 70/85] changelog & cleanup for pull request #59 --- beets/library.py | 9 ++++++--- docs/changelog.rst | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/beets/library.py b/beets/library.py index 3131c7b79..0fa14811e 100644 --- a/beets/library.py +++ b/beets/library.py @@ -273,9 +273,12 @@ class Item(object): read_path = self.path else: read_path = normpath(read_path) - try: f = MediaFile(syspath(read_path)) - except Exception as err: - log.error('Failed processing file: {!r}'.format(read_path)) + try: + f = MediaFile(syspath(read_path)) + except Exception: + log.error('failed reading file: {0}'.format( + displayable_path(read_path)) + ) raise for key in ITEM_KEYS_META: diff --git a/docs/changelog.rst b/docs/changelog.rst index 19d95b86d..51655b854 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -48,6 +48,8 @@ Changelog releases (we now submit only 200 releases at a time instead of 350). Thanks to Jonathan Towne. * Add the track mapping dictionary to the ``album_distance`` plugin function. +* When an exception is raised while reading a file, the path of the file in + question is now logged (thanks to Mike Kazantsev). * Fix an assertion failure when the MusicBrainz main database and search server disagree. * Fix a bug that caused the :doc:`/plugins/lastgenre` and other plugins not to From 3952fbec62f9f3f2515e98b002ecb9360b86e15a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 20:29:49 -0700 Subject: [PATCH 71/85] docs & changelog for pervasive format config (#62) This version of the (renamed) _print_obj function uses introspection to determine whether we're printing an Album or an Item. It's like function overloading for Python! :grin: --- beets/ui/commands.py | 32 ++++++++++++++++++-------------- docs/changelog.rst | 3 +++ docs/reference/cli.rst | 4 ++++ docs/reference/config.rst | 13 +++++++------ 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index ed03555b4..5d08b0bc9 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -40,7 +40,11 @@ log = logging.getLogger('beets') # objects that can be fed to a SubcommandsOptionParser. default_commands = [] -# Utility. +DEFAULT_LIST_FORMAT_ITEM = '$artist - $album - $title' +DEFAULT_LIST_FORMAT_ALBUM = '$albumartist - $album' + + +# Utilities. def _do_query(lib, query, album, also_items=True): """For commands that operate on matched items, performs a query @@ -83,13 +87,10 @@ def _showdiff(field, oldval, newval, color): oldval, newval = unicode(oldval), unicode(newval) print_(u' %s: %s -> %s' % (field, oldval, newval)) - -DEFAULT_LIST_FORMAT_ITEM = '$artist - $album - $title' -DEFAULT_LIST_FORMAT_ALBUM = '$albumartist - $album' - def _pick_format(config=None, album=False, fmt=None): - """Pick album / item printing format from passed arguments, - falling back to config options and defaults.""" + """Pick a format string for printing Album or Item objects, + falling back to config options and defaults. + """ if not fmt and not config: fmt = DEFAULT_LIST_FORMAT_ALBUM \ if album else DEFAULT_LIST_FORMAT_ITEM @@ -102,8 +103,11 @@ def _pick_format(config=None, album=False, fmt=None): DEFAULT_LIST_FORMAT_ITEM) return fmt -def _format_and_print(obj, lib, album=False, fmt=None): - """Print object according to specified format.""" +def _print_obj(obj, lib, fmt=None): + """Print an Album or Item object. If `fmt` is specified, use that + format string. Otherwise, use the configured (or default) template. + """ + album = isinstance(obj, library.Album) if not fmt: fmt = _pick_format(album=album) template = Template(fmt) @@ -113,8 +117,8 @@ def _format_and_print(obj, lib, album=False, fmt=None): print_(obj.evaluate_template(template, lib=lib)) - # fields: Shows a list of available fields for queries and format strings. + fields_cmd = ui.Subcommand('fields', help='show fields available for queries and format strings') def fields_func(lib, config, opts, args): @@ -886,7 +890,7 @@ def update_items(lib, query, album, move, color, pretend, fmt=None): for item in items: # Item deleted? if not os.path.exists(syspath(item.path)): - _format_and_print(item, lib, fmt=fmt) + _print_obj(item, lib, fmt=fmt) if not pretend: lib.remove(item, True) affected_albums.add(item.album_id) @@ -918,7 +922,7 @@ def update_items(lib, query, album, move, color, pretend, fmt=None): changes[key] = old_data[key], getattr(item, key) if changes: # Something changed. - _format_and_print(item, lib, fmt=fmt) + _print_obj(item, lib, fmt=fmt) for key, (oldval, newval) in changes.iteritems(): _showdiff(key, oldval, newval, color) @@ -991,7 +995,7 @@ def remove_items(lib, query, album, delete=False, fmt=None): # Show all the items. for item in items: - _format_and_print(item, lib, fmt=fmt) + _print_obj(item, lib, fmt=fmt) # Confirm with user. print_() @@ -1110,7 +1114,7 @@ def modify_items(lib, mods, query, write, move, album, color, confirm, fmt=None) print_('Modifying %i %ss.' % (len(objs), 'album' if album else 'item')) for obj in objs: # Identify the changed object. - _format_and_print(obj, lib, album=album, fmt=fmt) + _print_obj(obj, lib, fmt=fmt) # Show each change. for field, value in fsets.iteritems(): diff --git a/docs/changelog.rst b/docs/changelog.rst index 51655b854..9c976b01e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,9 @@ Changelog calculation more accurate (thanks to Jakob Schnitzer). * :ref:`list-cmd` command: Templates given with ``-f`` can now show items' paths (using ``$path``). +* The output of the :ref:`update-cmd`, :ref:`remove-cmd`, and :ref:`modify-cmd` + commands now respects the :ref:`list_format_album` and + :ref:`list_format_item` config options. Thanks to Mike Kazantsev. * Fix album queries for ``artpath`` and other non-item fields. * Null values in the database can now be matched with the empty-string regular expression, ``^$``. diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 64b760199..cd588ea38 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -151,6 +151,8 @@ variable expansion. .. _xargs: http://en.wikipedia.org/wiki/Xargs +.. _remove-cmd: + remove `````` :: @@ -199,6 +201,8 @@ destination directory with ``-d`` manually, you can move items matching a query anywhere in your filesystem. The ``-c`` option copies files instead of moving them. As with other commands, the ``-a`` option matches albums instead of items. +.. _update-cmd: + update `````` :: diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 7936b4551..eb1954a2d 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -200,18 +200,19 @@ to be changed except on very slow systems. Defaults to 5.0 (5 seconds). list_format_item ~~~~~~~~~~~~~~~~ -Format to use when listing *individual items* with the ``beet list`` -command. Defaults to ``$artist - $album - $title``. The ``-f`` command-line -option overrides this setting. +Format to use when listing *individual items* with the :ref:`list-cmd` +command and other commands that need to print out items. Defaults to +``$artist - $album - $title``. The ``-f`` command-line option overrides +this setting. .. _list_format_album: list_format_album ~~~~~~~~~~~~~~~~~ -Format to use when listing *albums* with the ``beet list`` command. -Defaults to ``$albumartist - $album``. The ``-f`` command-line option -overrides this setting. +Format to use when listing *albums* with :ref:`list-cmd` and other +commands. Defaults to ``$albumartist - $album``. The ``-f`` command-line +option overrides this setting. .. _per_disc_numbering: From 93a7251b576f0d492d4b4afd7333a097af374e7c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 20:50:03 -0700 Subject: [PATCH 72/85] #62: use list_format_* in non-list commands The list_format_album and list_format_item options now *actually* affect the display in commands other than "beet list". This replaces the -f/--format flags -- if any users want to control this on the command line, we can reconsider this decision. Note that this involved passing around a "config" object, which we previously haven't done. This seems a little bit messy, but configuration is about to change entirely to be more like this style -- so this isn't a huge liability. --- beets/ui/commands.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 5d08b0bc9..5d957d2aa 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -103,13 +103,13 @@ def _pick_format(config=None, album=False, fmt=None): DEFAULT_LIST_FORMAT_ITEM) return fmt -def _print_obj(obj, lib, fmt=None): +def _print_obj(obj, lib, config, fmt=None): """Print an Album or Item object. If `fmt` is specified, use that format string. Otherwise, use the configured (or default) template. """ album = isinstance(obj, library.Album) if not fmt: - fmt = _pick_format(album=album) + fmt = _pick_format(config, album=album) template = Template(fmt) if album: print_(obj.evaluate_template(template)) @@ -878,7 +878,7 @@ default_commands.append(list_cmd) # update: Update library contents according to on-disk tags. -def update_items(lib, query, album, move, color, pretend, fmt=None): +def update_items(lib, query, album, move, color, pretend, config): """For all the items matched by the query, update the library to reflect the item's embedded tags. """ @@ -890,7 +890,7 @@ def update_items(lib, query, album, move, color, pretend, fmt=None): for item in items: # Item deleted? if not os.path.exists(syspath(item.path)): - _print_obj(item, lib, fmt=fmt) + _print_obj(item, lib, config) if not pretend: lib.remove(item, True) affected_albums.add(item.album_id) @@ -922,7 +922,7 @@ def update_items(lib, query, album, move, color, pretend, fmt=None): changes[key] = old_data[key], getattr(item, key) if changes: # Something changed. - _print_obj(item, lib, fmt=fmt) + _print_obj(item, lib, config) for key, (oldval, newval) in changes.iteritems(): _showdiff(key, oldval, newval, color) @@ -978,15 +978,15 @@ update_cmd.parser.add_option('-f', '--format', action='store', help='print with custom format', default=None) def update_func(lib, config, opts, args): color = ui.config_val(config, 'beets', 'color', DEFAULT_COLOR, bool) - fmt = _pick_format(config, opts.album, opts.format) - update_items(lib, decargs(args), opts.album, opts.move, color, opts.pretend, fmt) + update_items(lib, decargs(args), opts.album, opts.move, + color, opts.pretend, config) update_cmd.func = update_func default_commands.append(update_cmd) # remove: Remove items from library, delete files. -def remove_items(lib, query, album, delete=False, fmt=None): +def remove_items(lib, query, album, delete, config): """Remove items matching query from lib. If album, then match and remove whole albums. If delete, also remove files from disk. """ @@ -995,7 +995,7 @@ def remove_items(lib, query, album, delete=False, fmt=None): # Show all the items. for item in items: - _print_obj(item, lib, fmt=fmt) + _print_obj(item, lib, config) # Confirm with user. print_() @@ -1022,11 +1022,8 @@ remove_cmd.parser.add_option("-d", "--delete", action="store_true", help="also remove files from disk") remove_cmd.parser.add_option('-a', '--album', action='store_true', help='match albums instead of tracks') -remove_cmd.parser.add_option('-f', '--format', action='store', - help='print with custom format', default=None) def remove_func(lib, config, opts, args): - fmt = _pick_format(config, opts.album, opts.format) - remove_items(lib, decargs(args), opts.album, opts.delete, fmt) + remove_items(lib, decargs(args), opts.album, opts.delete, config) remove_cmd.func = remove_func default_commands.append(remove_cmd) @@ -1095,7 +1092,8 @@ default_commands.append(version_cmd) # modify: Declaratively change metadata. -def modify_items(lib, mods, query, write, move, album, color, confirm, fmt=None): +def modify_items(lib, mods, query, write, move, album, color, confirm, + config): """Modifies matching items according to key=value assignments.""" # Parse key=value specifications into a dictionary. allowed_keys = library.ALBUM_KEYS if album else library.ITEM_KEYS_WRITABLE @@ -1114,7 +1112,7 @@ def modify_items(lib, mods, query, write, move, album, color, confirm, fmt=None) print_('Modifying %i %ss.' % (len(objs), 'album' if album else 'item')) for obj in objs: # Identify the changed object. - _print_obj(obj, lib, fmt=fmt) + _print_obj(obj, lib, config) # Show each change. for field, value in fsets.iteritems(): @@ -1177,9 +1175,8 @@ def modify_func(lib, config, opts, args): ui.config_val(config, 'beets', 'import_write', DEFAULT_IMPORT_WRITE, bool) color = ui.config_val(config, 'beets', 'color', DEFAULT_COLOR, bool) - fmt = _pick_format(config, opts.album, opts.format) modify_items(lib, mods, query, write, opts.move, opts.album, color, - not opts.yes, fmt) + not opts.yes, config) modify_cmd.func = modify_func default_commands.append(modify_cmd) From 2770b7d6fc652e2fb5912571a53aa06f260b00d6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 20:55:14 -0700 Subject: [PATCH 73/85] #62: move format/print utilities to beets.ui Now that these functions are generally useful, this is the right home for them. Plugins should also use print_obj. --- beets/ui/__init__.py | 34 ++++++++++++++++++++++++++++++++++ beets/ui/commands.py | 42 +++++------------------------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index c3fb162e3..a4afa0579 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -38,6 +38,7 @@ from beets.util.functemplate import Template # On Windows platforms, use colorama to support "ANSI" terminal colors. + if sys.platform == 'win32': try: import colorama @@ -48,6 +49,7 @@ if sys.platform == 'win32': # Constants. + CONFIG_PATH_VAR = 'BEETSCONFIG' DEFAULT_CONFIG_FILENAME_UNIX = '.beetsconfig' DEFAULT_CONFIG_FILENAME_WINDOWS = 'beetsconfig.ini' @@ -70,6 +72,9 @@ DEFAULT_PATH_FORMATS = [ DEFAULT_ART_FILENAME = 'cover' DEFAULT_TIMEOUT = 5.0 NULL_REPLACE = '' +DEFAULT_LIST_FORMAT_ITEM = '$artist - $album - $title' +DEFAULT_LIST_FORMAT_ALBUM = '$albumartist - $album' + # UI exception. Commands should throw this in order to display # nonrecoverable errors to the user. @@ -511,6 +516,35 @@ def _get_path_formats(config): return path_formats +def _pick_format(config=None, album=False, fmt=None): + """Pick a format string for printing Album or Item objects, + falling back to config options and defaults. + """ + if not fmt and not config: + fmt = DEFAULT_LIST_FORMAT_ALBUM \ + if album else DEFAULT_LIST_FORMAT_ITEM + elif not fmt: + if album: + fmt = config_val(config, 'beets', 'list_format_album', + DEFAULT_LIST_FORMAT_ALBUM) + else: + fmt = config_val(config, 'beets', 'list_format_item', + DEFAULT_LIST_FORMAT_ITEM) + return fmt + +def print_obj(obj, lib, config, fmt=None): + """Print an Album or Item object. If `fmt` is specified, use that + format string. Otherwise, use the configured (or default) template. + """ + album = isinstance(obj, library.Album) + if not fmt: + fmt = _pick_format(config, album=album) + template = Template(fmt) + if album: + print_(obj.evaluate_template(template)) + else: + print_(obj.evaluate_template(template, lib=lib)) + # Subcommand parsing infrastructure. diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 5d957d2aa..864c54e62 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -40,9 +40,6 @@ log = logging.getLogger('beets') # objects that can be fed to a SubcommandsOptionParser. default_commands = [] -DEFAULT_LIST_FORMAT_ITEM = '$artist - $album - $title' -DEFAULT_LIST_FORMAT_ALBUM = '$albumartist - $album' - # Utilities. @@ -87,35 +84,6 @@ def _showdiff(field, oldval, newval, color): oldval, newval = unicode(oldval), unicode(newval) print_(u' %s: %s -> %s' % (field, oldval, newval)) -def _pick_format(config=None, album=False, fmt=None): - """Pick a format string for printing Album or Item objects, - falling back to config options and defaults. - """ - if not fmt and not config: - fmt = DEFAULT_LIST_FORMAT_ALBUM \ - if album else DEFAULT_LIST_FORMAT_ITEM - elif not fmt: - if album: - fmt = ui.config_val(config, 'beets', 'list_format_album', - DEFAULT_LIST_FORMAT_ALBUM) - else: - fmt = ui.config_val(config, 'beets', 'list_format_item', - DEFAULT_LIST_FORMAT_ITEM) - return fmt - -def _print_obj(obj, lib, config, fmt=None): - """Print an Album or Item object. If `fmt` is specified, use that - format string. Otherwise, use the configured (or default) template. - """ - album = isinstance(obj, library.Album) - if not fmt: - fmt = _pick_format(config, album=album) - template = Template(fmt) - if album: - print_(obj.evaluate_template(template)) - else: - print_(obj.evaluate_template(template, lib=lib)) - # fields: Shows a list of available fields for queries and format strings. @@ -870,7 +838,7 @@ list_cmd.parser.add_option('-p', '--path', action='store_true', list_cmd.parser.add_option('-f', '--format', action='store', help='print with custom format', default=None) def list_func(lib, config, opts, args): - fmt = _pick_format(config, opts.album, opts.format) + fmt = ui._pick_format(config, opts.album, opts.format) list_items(lib, decargs(args), opts.album, opts.path, fmt) list_cmd.func = list_func default_commands.append(list_cmd) @@ -890,7 +858,7 @@ def update_items(lib, query, album, move, color, pretend, config): for item in items: # Item deleted? if not os.path.exists(syspath(item.path)): - _print_obj(item, lib, config) + ui.print_obj(item, lib, config) if not pretend: lib.remove(item, True) affected_albums.add(item.album_id) @@ -922,7 +890,7 @@ def update_items(lib, query, album, move, color, pretend, config): changes[key] = old_data[key], getattr(item, key) if changes: # Something changed. - _print_obj(item, lib, config) + ui.print_obj(item, lib, config) for key, (oldval, newval) in changes.iteritems(): _showdiff(key, oldval, newval, color) @@ -995,7 +963,7 @@ def remove_items(lib, query, album, delete, config): # Show all the items. for item in items: - _print_obj(item, lib, config) + ui.print_obj(item, lib, config) # Confirm with user. print_() @@ -1112,7 +1080,7 @@ def modify_items(lib, mods, query, write, move, album, color, confirm, print_('Modifying %i %ss.' % (len(objs), 'album' if album else 'item')) for obj in objs: # Identify the changed object. - _print_obj(obj, lib, config) + ui.print_obj(obj, lib, config) # Show each change. for field, value in fsets.iteritems(): From 91ad913399a98bea9018c3a33901d41b2da73a11 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 21:10:39 -0700 Subject: [PATCH 74/85] #62: simplify list (and random) code With the new centralized print_obj function, we can greatly simplify the code for the list command. This necessitated a couple of additional tweaks: - For performance reasons, print_obj can now take a compiled template. (There's still an issue with using the default/configured template, but we can cross that bridge later). - When listing albums, $path now expands to the album's item dir. So the format string '$path' now exactly corresponds to passing the -p switch. As an added bonus, we can now also reduce copypasta in the random plugin (which behaves almost exactly the same as list). --- beets/library.py | 1 + beets/ui/__init__.py | 5 ++++- beets/ui/commands.py | 25 ++++++++++--------------- beetsplug/rdm.py | 29 +++++++++-------------------- docs/changelog.rst | 4 ++-- 5 files changed, 26 insertions(+), 38 deletions(-) diff --git a/beets/library.py b/beets/library.py index 135bd2be0..801ce5d3a 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1600,6 +1600,7 @@ class Album(BaseAlbum): mapping[key] = getattr(self, key) mapping['artpath'] = displayable_path(mapping['artpath']) + mapping['path'] = displayable_path(self.item_dir()) # Get template functions. funcs = DefaultTemplateFunctions().functions() diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index a4afa0579..a1dc8bd04 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -539,7 +539,10 @@ def print_obj(obj, lib, config, fmt=None): album = isinstance(obj, library.Album) if not fmt: fmt = _pick_format(config, album=album) - template = Template(fmt) + if isinstance(fmt, Template): + template = fmt + else: + template = Template(fmt) if album: print_(obj.evaluate_template(template)) else: diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 864c54e62..81e1dc553 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -810,25 +810,17 @@ default_commands.append(import_cmd) # list: Query and show library contents. -def list_items(lib, query, album, path, fmt): +def list_items(lib, query, album, fmt, config): """Print out items in lib matching query. If album, then search for - albums instead of single items. If path, print the matched objects' - paths instead of human-readable information about them. + albums instead of single items. """ - template = Template(fmt) - + tmpl = Template(fmt) if fmt else None if album: for album in lib.albums(query): - if path: - print_(album.item_dir()) - elif fmt is not None: - print_(album.evaluate_template(template)) + ui.print_obj(album, lib, config, tmpl) else: for item in lib.items(query): - if path: - print_(item.path) - elif fmt is not None: - print_(item.evaluate_template(template, lib)) + ui.print_obj(item, lib, config, tmpl) list_cmd = ui.Subcommand('list', help='query the library', aliases=('ls',)) list_cmd.parser.add_option('-a', '--album', action='store_true', @@ -838,8 +830,11 @@ list_cmd.parser.add_option('-p', '--path', action='store_true', list_cmd.parser.add_option('-f', '--format', action='store', help='print with custom format', default=None) def list_func(lib, config, opts, args): - fmt = ui._pick_format(config, opts.album, opts.format) - list_items(lib, decargs(args), opts.album, opts.path, fmt) + if opts.path: + fmt = '$path' + else: + fmt = opts.format + list_items(lib, decargs(args), opts.album, fmt, config) list_cmd.func = list_func default_commands.append(list_cmd) diff --git a/beetsplug/rdm.py b/beetsplug/rdm.py index 63779e8d2..72b338138 100644 --- a/beetsplug/rdm.py +++ b/beetsplug/rdm.py @@ -15,22 +15,17 @@ """Get a random song or album from the library. """ from beets.plugins import BeetsPlugin -from beets.ui import Subcommand, decargs, print_ +from beets.ui import Subcommand, decargs, print_obj from beets.util.functemplate import Template import random def random_item(lib, config, opts, args): query = decargs(args) - path = opts.path - fmt = opts.format - - if fmt is None: - # If no specific template is supplied, use a default - if opts.album: - fmt = u'$albumartist - $album' - else: - fmt = u'$artist - $album - $title' - template = Template(fmt) + if opts.path: + fmt = '$path' + else: + fmt = opts.format + template = Template(fmt) if fmt else None if opts.album: objs = list(lib.albums(query=query)) @@ -41,16 +36,10 @@ def random_item(lib, config, opts, args): if opts.album: for album in objs: - if path: - print_(album.item_dir()) - else: - print_(album.evaluate_template(template)) + print_obj(album, lib, config, template) else: - for item in objs: - if path: - print_(item.path) - else: - print_(item.evaluate_template(template, lib)) + for item in objs: + print_obj(item, lib, config, template) random_cmd = Subcommand('random', help='chose a random track or album') diff --git a/docs/changelog.rst b/docs/changelog.rst index 9c976b01e..718fb7c8f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,8 +25,8 @@ Changelog tags and ID3 tags, the ID3 tags are now also removed. * :ref:`stats-cmd` command: New ``--exact`` switch to make the file size calculation more accurate (thanks to Jakob Schnitzer). -* :ref:`list-cmd` command: Templates given with ``-f`` can now show items' paths - (using ``$path``). +* :ref:`list-cmd` command: Templates given with ``-f`` can now show items' and + albums' paths (using ``$path``). * The output of the :ref:`update-cmd`, :ref:`remove-cmd`, and :ref:`modify-cmd` commands now respects the :ref:`list_format_album` and :ref:`list_format_item` config options. Thanks to Mike Kazantsev. From 1f8fff7445cff2504e6a168dd1230f6d630e7a54 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 21:19:50 -0700 Subject: [PATCH 75/85] fix tests for recent API changes in commands --- test/test_ui.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/test/test_ui.py b/test/test_ui.py index 98204771b..eea86adf8 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -47,12 +47,7 @@ class ListTest(unittest.TestCase): self.io.restore() def _run_list(self, query='', album=False, path=False, fmt=None): - if not fmt: - if album: - fmt = commands.DEFAULT_LIST_FORMAT_ALBUM - else: - fmt = commands.DEFAULT_LIST_FORMAT_ITEM - commands.list_items(self.lib, query, album, path, fmt) + commands.list_items(self.lib, query, album, fmt, None) def test_list_outputs_item(self): self._run_list() @@ -69,7 +64,7 @@ class ListTest(unittest.TestCase): self.assertTrue(u'na\xefve' in out.decode(self.io.stdout.encoding)) def test_list_item_path(self): - self._run_list(path=True) + self._run_list(fmt='$path') out = self.io.getoutput() self.assertEqual(out.strip(), u'xxx/yyy') @@ -79,7 +74,7 @@ class ListTest(unittest.TestCase): self.assertGreater(len(out), 0) def test_list_album_path(self): - self._run_list(album=True, path=True) + self._run_list(album=True, fmt='$path') out = self.io.getoutput() self.assertEqual(out.strip(), u'xxx') @@ -119,11 +114,6 @@ class ListTest(unittest.TestCase): self.assertTrue(u'the genre' in out) self.assertTrue(u'the album' not in out) - def test_list_item_path_ignores_format(self): - self._run_list(path=True, fmt='$year - $artist') - out = self.io.getoutput() - self.assertEqual(out.strip(), u'xxx/yyy') - class RemoveTest(unittest.TestCase): def setUp(self): self.io = _common.DummyIO() @@ -143,14 +133,14 @@ class RemoveTest(unittest.TestCase): def test_remove_items_no_delete(self): self.io.addinput('y') - commands.remove_items(self.lib, '', False, False) + commands.remove_items(self.lib, '', False, False, None) items = self.lib.items() self.assertEqual(len(list(items)), 0) self.assertTrue(os.path.exists(self.i.path)) def test_remove_items_with_delete(self): self.io.addinput('y') - commands.remove_items(self.lib, '', False, True) + commands.remove_items(self.lib, '', False, True, None) items = self.lib.items() self.assertEqual(len(list(items)), 0) self.assertFalse(os.path.exists(self.i.path)) @@ -176,7 +166,7 @@ class ModifyTest(unittest.TestCase): def _modify(self, mods, query=(), write=False, move=False, album=False): self.io.addinput('y') commands.modify_items(self.lib, mods, query, - write, move, album, True, True) + write, move, album, True, True, None) def test_modify_item_dbdata(self): self._modify(["title=newTitle"]) @@ -334,7 +324,7 @@ class UpdateTest(unittest.TestCase, _common.ExtraAsserts): if reset_mtime: self.i.mtime = 0 self.lib.store(self.i) - commands.update_items(self.lib, query, album, move, True, False) + commands.update_items(self.lib, query, album, move, True, False, None) def test_delete_removes_item(self): self.assertTrue(list(self.lib.items())) From 16f207e927094bc68647d05ddb5f1c609b132041 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Oct 2012 23:49:43 -0700 Subject: [PATCH 76/85] make syspath/bytestring_path roundtrip on Windows This is an alternative to #58 that makes bytestring_path perform more like the inverse of syspath on Windows. This way, we can convert to syspath, operate on the path, and then bring back to internal representation without data loss. This involves looking for the magic prefix on the Unicode string and removing it before encoding to the internal (UTF-8) representation. --- beets/util/__init__.py | 20 ++++++++++++++++---- test/test_db.py | 41 ++++++++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 5e7cb8e38..5a36c31bf 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -24,6 +24,7 @@ from collections import defaultdict import traceback MAX_FILENAME_LENGTH = 200 +WINDOWS_MAGIC_PREFIX = u'\\\\?\\' class HumanReadableException(Exception): """An Exception that can include a human-readable error message to @@ -108,7 +109,9 @@ def normpath(path): """Provide the canonical form of the path suitable for storing in the database. """ - return os.path.normpath(os.path.abspath(os.path.expanduser(path))) + path = syspath(path) + path = os.path.normpath(os.path.abspath(os.path.expanduser(path))) + return bytestring_path(path) def ancestry(path, pathmod=None): """Return a list consisting of path's parent directory, its @@ -256,14 +259,23 @@ def _fsencoding(): encoding = 'utf8' return encoding -def bytestring_path(path): +def bytestring_path(path, pathmod=None): """Given a path, which is either a str or a unicode, returns a str path (ensuring that we never deal with Unicode pathnames). """ + pathmod = pathmod or os.path + windows = pathmod.__name__ == 'ntpath' + # Pass through bytestrings. if isinstance(path, str): return path + # On Windows, remove the magic prefix added by `syspath`. This makes + # ``bytestring_path(syspath(X)) == X``, i.e., we can safely + # round-trip through `syspath`. + if windows and path.startswith(WINDOWS_MAGIC_PREFIX): + path = path[len(WINDOWS_MAGIC_PREFIX):] + # Try to encode with default encodings, but fall back to UTF8. try: return path.encode(_fsencoding()) @@ -310,8 +322,8 @@ def syspath(path, pathmod=None): path = path.decode(encoding, 'replace') # Add the magic prefix if it isn't already there - if not path.startswith(u'\\\\?\\'): - path = u'\\\\?\\' + path + if not path.startswith(WINDOWS_MAGIC_PREFIX): + path = WINDOWS_MAGIC_PREFIX + path return path diff --git a/test/test_db.py b/test/test_db.py index fc597b6cd..5df34f45e 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -341,17 +341,6 @@ class DestinationTest(unittest.TestCase): ] self.assertEqual(self.lib.destination(self.i), np('one/three')) - def test_syspath_windows_format(self): - path = ntpath.join('a', 'b', 'c') - outpath = util.syspath(path, ntpath) - self.assertTrue(isinstance(outpath, unicode)) - self.assertTrue(outpath.startswith(u'\\\\?\\')) - - def test_syspath_posix_unchanged(self): - path = posixpath.join('a', 'b', 'c') - outpath = util.syspath(path, posixpath) - self.assertEqual(path, outpath) - def test_sanitize_windows_replaces_trailing_space(self): p = util.sanitize_path(u'one/two /three', ntpath) self.assertFalse(' ' in p) @@ -563,6 +552,36 @@ class DisambiguationTest(unittest.TestCase, PathFormattingMixin): self._setf(u'foo%aunique{albumartist album,albumtype}/$title') self._assert_dest('/base/foo [foo_bar]/the title', self.i1) +class PathConversionTest(unittest.TestCase): + def test_syspath_windows_format(self): + path = ntpath.join('a', 'b', 'c') + outpath = util.syspath(path, ntpath) + self.assertTrue(isinstance(outpath, unicode)) + self.assertTrue(outpath.startswith(u'\\\\?\\')) + + def test_syspath_posix_unchanged(self): + path = posixpath.join('a', 'b', 'c') + outpath = util.syspath(path, posixpath) + self.assertEqual(path, outpath) + + def _windows_bytestring_path(self, path): + old_gfse = sys.getfilesystemencoding + sys.getfilesystemencoding = lambda: 'mbcs' + try: + return util.bytestring_path(path, ntpath) + finally: + sys.getfilesystemencoding = old_gfse + + def test_bytestring_path_windows_encodes_utf8(self): + path = u'caf\xe9' + outpath = self._windows_bytestring_path(path) + self.assertEqual(path, outpath.decode('utf8')) + + def test_bytesting_path_windows_removes_magic_prefix(self): + path = u'\\\\?\\C:\\caf\xe9' + outpath = self._windows_bytestring_path(path) + self.assertEqual(outpath, u'C:\\caf\xe9'.encode('utf8')) + class PluginDestinationTest(unittest.TestCase): # Mock the plugins.template_values(item) function. def _template_values(self, item): From 54f29812cf321aef2934ba5fd352d6770fcf5d09 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Sun, 21 Oct 2012 13:35:41 +0200 Subject: [PATCH 77/85] convert: fix breakage due to recent API changes --- beetsplug/convert.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index fecaa9fa0..0c4ee7ead 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -103,9 +103,7 @@ def convert_func(lib, config, opts, args): raise ui.UserError('no convert destination set') threads = opts.threads if opts.threads is not None else conf['threads'] - fmt = '$albumartist - $album' if opts.album \ - else '$artist - $album - $title' - ui.commands.list_items(lib, ui.decargs(args), opts.album, False, fmt) + ui.commands.list_items(lib, ui.decargs(args), opts.album, None, config) if not ui.input_yn("Convert? (Y/n)"): return From d6f20e91bd1667e2eced38ae825ce9bad0bf1d44 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Sun, 21 Oct 2012 13:54:24 +0200 Subject: [PATCH 78/85] Speedup 'beet ls' if no format is specified --- beets/ui/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 81e1dc553..24d7280e7 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -814,7 +814,7 @@ def list_items(lib, query, album, fmt, config): """Print out items in lib matching query. If album, then search for albums instead of single items. """ - tmpl = Template(fmt) if fmt else None + tmpl = Template(fmt) if fmt else Template(ui._pick_format(config, album)) if album: for album in lib.albums(query): ui.print_obj(album, lib, config, tmpl) From e80dce6930c5ca2a75005fa434c0352523508ef3 Mon Sep 17 00:00:00 2001 From: Philippe Mongeau Date: Sun, 21 Oct 2012 11:29:21 -0400 Subject: [PATCH 79/85] fuzzy: use the new print_obj function --- beetsplug/fuzzy_search.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/beetsplug/fuzzy_search.py b/beetsplug/fuzzy_search.py index 398a78811..37a664ac7 100644 --- a/beetsplug/fuzzy_search.py +++ b/beetsplug/fuzzy_search.py @@ -16,7 +16,7 @@ """ import beets from beets.plugins import BeetsPlugin -from beets.ui import Subcommand, decargs, print_ +from beets.ui import Subcommand, decargs, print_obj from beets.util.functemplate import Template import difflib @@ -47,19 +47,16 @@ def fuzzy_list(lib, config, opts, args): query = ' '.join(query).lower() queryMatcher = difflib.SequenceMatcher(b=query) - fmt = opts.format if opts.threshold is not None: threshold = float(opts.threshold) else: threshold = float(conf['threshold']) - if fmt is None: - # If no specific template is supplied, use a default - if opts.album: - fmt = u'$albumartist - $album' - else: - fmt = u'$artist - $album - $title' - template = Template(fmt) + if opts.path: + fmt = '$path' + else: + fmt = opts.format + template = Template(fmt) if fmt else None if opts.album: objs = lib.albums() @@ -68,13 +65,9 @@ def fuzzy_list(lib, config, opts, args): items = filter(lambda i: is_match(queryMatcher, i, album=opts.album, threshold=threshold), objs) - for i in items: - if opts.path: - print_(i.item_dir() if opts.album else i.path) - elif opts.album: - print_(i.evaluate_template(template)) - else: - print_(i.evaluate_template(template, lib)) + + for item in items: + print_obj(item, lib, config, template) if opts.verbose: print(is_match(queryMatcher, i, album=opts.album, verbose=True)[1]) From 78f2003eb015b2ec1b45b695339a0ae28bf08edc Mon Sep 17 00:00:00 2001 From: Philippe Mongeau Date: Sun, 21 Oct 2012 11:39:23 -0400 Subject: [PATCH 80/85] simplify the random print code We don't need the 'if opts.album' since we print with the same function and argsuments. --- beetsplug/rdm.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/beetsplug/rdm.py b/beetsplug/rdm.py index 72b338138..6eebba0bd 100644 --- a/beetsplug/rdm.py +++ b/beetsplug/rdm.py @@ -34,12 +34,8 @@ def random_item(lib, config, opts, args): number = min(len(objs), opts.number) objs = random.sample(objs, number) - if opts.album: - for album in objs: - print_obj(album, lib, config, template) - else: - for item in objs: - print_obj(item, lib, config, template) + for item in objs: + print_obj(item, lib, config, template) random_cmd = Subcommand('random', help='chose a random track or album') From 1a94d9e4b7ed40ddcfe56be5406e31e568161437 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 21 Oct 2012 14:27:40 -0700 Subject: [PATCH 81/85] warnings about using syspath with some utils Also pertaining to #58: for most utility functions, paths should *not* be `syspath`-ified. (This only occurs right before a path is sent to the OS.) In fact, as @Wessie discovered, using the result of `syspath` with `ancestry` leads to incorrect behavior. I checked and this should not currently happen anywhere, but these docstring changes make that requirement explicit. --- beets/util/__init__.py | 6 ++++++ docs/changelog.rst | 2 ++ 2 files changed, 8 insertions(+) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 5a36c31bf..6074c557b 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -116,8 +116,11 @@ def normpath(path): def ancestry(path, pathmod=None): """Return a list consisting of path's parent directory, its grandparent, and so on. For instance: + >>> ancestry('/a/b/c') ['/', '/a', '/a/b'] + + The argument should *not* be the result of a call to `syspath`. """ pathmod = pathmod or os.path out = [] @@ -226,8 +229,11 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')): def components(path, pathmod=None): """Return a list of the path components in path. For instance: + >>> components('/a/b/c') ['a', 'b', 'c'] + + The argument should *not* be the result of a call to `syspath`. """ pathmod = pathmod or os.path comps = [] diff --git a/docs/changelog.rst b/docs/changelog.rst index 718fb7c8f..31b2dcd01 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -62,6 +62,8 @@ Changelog * Fix for changing date fields (like "year") with the :ref:`modify-cmd` command. * Fix a crash when input is read from a pipe without a specified encoding. +* Fix some problem with identifying files on Windows with Unicode directory + names in their path. * Add a human-readable error message when writing files' tags fails. * Changed plugin loading so that modules can be imported without unintentionally loading the plugins they contain. From dcb9ad7373d3cca6f49edd17fe6b1c2110d6ab90 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 24 Oct 2012 15:14:33 -0700 Subject: [PATCH 82/85] fix several non-unicode logging statements A user reported a problem with one of the logging statements where .format() tried to convert a Unicode string to bytes because the log message was '', not u''. As a rule, we should ensure that all logging statements use Unicode literals. --- beets/autotag/__init__.py | 6 ++++-- beets/library.py | 4 ++-- beetsplug/convert.py | 14 +++++++------- beetsplug/lyrics.py | 4 ++-- beetsplug/replaygain.py | 18 +++++++++--------- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 0ceb1dfab..1addca7a3 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -19,7 +19,7 @@ import logging import re from beets import library, mediafile -from beets.util import sorted_walk, ancestry +from beets.util import sorted_walk, ancestry, displayable_path # Parts of external interface. from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch @@ -57,7 +57,9 @@ def albums_in_dir(path, ignore=()): except mediafile.FileTypeError: pass except mediafile.UnreadableFileError: - log.warn('unreadable file: ' + filename) + log.warn(u'unreadable file: {0}'.format( + displayable_path(filename)) + ) else: items.append(i) diff --git a/beets/library.py b/beets/library.py index 801ce5d3a..d767331fb 100644 --- a/beets/library.py +++ b/beets/library.py @@ -276,7 +276,7 @@ class Item(object): try: f = MediaFile(syspath(read_path)) except Exception: - log.error('failed reading file: {0}'.format( + log.error(u'failed reading file: {0}'.format( displayable_path(read_path)) ) raise @@ -610,7 +610,7 @@ class CollectionQuery(Query): # Unrecognized field. else: - log.warn('no such field in query: {0}'.format(key)) + log.warn(u'no such field in query: {0}'.format(key)) if not subqueries: # No terms in query. subqueries = [TrueQuery()] diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 0c4ee7ead..42bc9e041 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -31,7 +31,7 @@ _fs_lock = threading.Lock() def encode(source, dest): - log.info('Started encoding ' + source) + log.info(u'Started encoding {0}'.format(util.displayable_path(source))) temp_dest = dest + '~' source_ext = os.path.splitext(source)[1].lower() @@ -47,23 +47,23 @@ def encode(source, dest): [source, temp_dest], close_fds=True, stderr=DEVNULL) encode.communicate() else: - log.error('Only converting from FLAC or MP3 implemented') + log.error(u'Only converting from FLAC or MP3 implemented') return if encode.returncode != 0: # Something went wrong (probably Ctrl+C), remove temporary files - log.info('Encoding {0} failed. Cleaning up...'.format(source)) + log.info(u'Encoding {0} failed. Cleaning up...'.format(source)) util.remove(temp_dest) util.prune_dirs(os.path.dirname(temp_dest)) return shutil.move(temp_dest, dest) - log.info('Finished encoding ' + source) + log.info(u'Finished encoding {0}'.format(util.displayable_path(source))) def convert_item(lib, dest_dir): while True: item = yield if item.format != 'FLAC' and item.format != 'MP3': - log.info('Skipping {0} (unsupported format)'.format( + log.info(u'Skipping {0} (unsupported format)'.format( util.displayable_path(item.path) )) continue @@ -72,7 +72,7 @@ def convert_item(lib, dest_dir): dest = os.path.splitext(dest)[0] + '.mp3' if os.path.exists(dest): - log.info('Skipping {0} (target file exists)'.format( + log.info(u'Skipping {0} (target file exists)'.format( util.displayable_path(item.path) )) continue @@ -84,7 +84,7 @@ def convert_item(lib, dest_dir): util.mkdirall(dest) if item.format == 'MP3' and item.bitrate < 1000 * conf['max_bitrate']: - log.info('Copying {0}'.format(util.displayable_path(item.path))) + log.info(u'Copying {0}'.format(util.displayable_path(item.path))) util.copy(item.path, dest) dest_item = library.Item.from_path(dest) else: diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 6269e38a1..2a801436e 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -44,7 +44,7 @@ def fetch_url(url): try: return urllib.urlopen(url).read() except IOError as exc: - log.debug('failed to fetch: {0} ({1})'.format(url, str(exc))) + log.debug(u'failed to fetch: {0} ({1})'.format(url, unicode(exc))) return None def unescape(text): @@ -160,7 +160,7 @@ def get_lyrics(artist, title): if lyrics: if isinstance(lyrics, str): lyrics = lyrics.decode('utf8', 'ignore') - log.debug('got lyrics from backend: {0}'.format(backend.__name__)) + log.debug(u'got lyrics from backend: {0}'.format(backend.__name__)) return lyrics diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 0459f3557..1f03a4ed8 100755 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -132,8 +132,8 @@ class ReplayGainPlugin(BeetsPlugin): if opts.album: # Analyze albums. for album in lib.albums(ui.decargs(args)): - log.info('analyzing {0} - {1}'.format(album.albumartist, - album.album)) + log.info(u'analyzing {0} - {1}'.format(album.albumartist, + album.album)) items = list(album.items()) results = self.compute_rgain(items, True) if results: @@ -146,8 +146,8 @@ class ReplayGainPlugin(BeetsPlugin): else: # Analyze individual tracks. for item in lib.items(ui.decargs(args)): - log.info('analyzing {0} - {1}'.format(item.artist, - item.title)) + log.info(u'analyzing {0} - {1}'.format(item.artist, + item.title)) results = self.compute_rgain([item], False) if results: self.store_gain(lib, [item], results, None) @@ -177,7 +177,7 @@ class ReplayGainPlugin(BeetsPlugin): # needs recalculation, we still get an accurate album gain # value. if all([not self.requires_gain(i, album) for i in items]): - log.debug('replaygain: no gain to compute') + log.debug(u'replaygain: no gain to compute') return # Construct shell command. The "-o" option makes the output @@ -199,9 +199,9 @@ class ReplayGainPlugin(BeetsPlugin): cmd = cmd + ['-d', str(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] - log.debug('replaygain: analyzing {0} files'.format(len(items))) + log.debug(u'replaygain: analyzing {0} files'.format(len(items))) output = call(cmd) - log.debug('replaygain: analysis finished') + log.debug(u'replaygain: analysis finished') results = parse_tool_output(output) return results @@ -215,7 +215,7 @@ class ReplayGainPlugin(BeetsPlugin): item.rg_track_peak = info['peak'] lib.store(item) - log.debug('replaygain: applied track gain {0}, peak {1}'.format( + log.debug(u'replaygain: applied track gain {0}, peak {1}'.format( item.rg_track_gain, item.rg_track_peak )) @@ -225,7 +225,7 @@ class ReplayGainPlugin(BeetsPlugin): album_info = rgain_infos[-1] album.rg_album_gain = album_info['gain'] album.rg_album_peak = album_info['peak'] - log.debug('replaygain: applied album gain {0}, peak {1}'.format( + log.debug(u'replaygain: applied album gain {0}, peak {1}'.format( album.rg_album_gain, album.rg_album_peak )) From 02fd9bf809b3d94f91e1526d69fd1099312436b2 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 24 Oct 2012 15:17:00 -0700 Subject: [PATCH 83/85] convert: embed into destination file, not source file Paging @yagebu: I think the old version of the code would embed album art into the wrong file. Please correct me (and accept my apologies) if I'm wrong though. --- beetsplug/convert.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 42bc9e041..79f487b2e 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -86,7 +86,6 @@ def convert_item(lib, dest_dir): if item.format == 'MP3' and item.bitrate < 1000 * conf['max_bitrate']: log.info(u'Copying {0}'.format(util.displayable_path(item.path))) util.copy(item.path, dest) - dest_item = library.Item.from_path(dest) else: encode(item.path, dest) item.path = dest @@ -94,7 +93,7 @@ def convert_item(lib, dest_dir): artpath = lib.get_album(item).artpath if artpath and conf['embed']: - _embed(artpath, [item]) + _embed(artpath, [library.Item.from_path(dest)]) def convert_func(lib, config, opts, args): From cf98bfcbe6ef2caf5befe3b8891058d4aa041fca Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Thu, 25 Oct 2012 01:05:06 +0200 Subject: [PATCH 84/85] convert: write tags from library to copied files --- beetsplug/convert.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 79f487b2e..4ea3b60e3 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -88,12 +88,13 @@ def convert_item(lib, dest_dir): util.copy(item.path, dest) else: encode(item.path, dest) - item.path = dest - item.write() + + item.path = dest + item.write() artpath = lib.get_album(item).artpath if artpath and conf['embed']: - _embed(artpath, [library.Item.from_path(dest)]) + _embed(artpath, [item]) def convert_func(lib, config, opts, args): From 57e66d7b1ac07eff23cae59e90558046664168aa Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 25 Oct 2012 18:02:22 -0700 Subject: [PATCH 85/85] fetchart: sort image filenames (GC-452) --- beetsplug/fetchart.py | 5 +++-- docs/changelog.rst | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 68906c1c4..19bf787e7 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -117,10 +117,11 @@ def art_in_path(path): for ext in IMAGE_EXTENSIONS: if fn.lower().endswith('.' + ext): images.append(fn) + images.sort() # Look for "preferred" filenames. - for fn in images: - for name in COVER_NAMES: + for name in COVER_NAMES: + for fn in images: if fn.lower().startswith(name): log.debug('Using well-named art file %s' % fn) return os.path.join(path, fn) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31b2dcd01..6f21b72c2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -38,6 +38,8 @@ Changelog the whitelist (thanks to Fabrice Laporte). * :doc:`/plugins/lastgenre`: Add a ``lastgenre`` command for fetching genres post facto (thanks to Jakob Schnitzer). +* :doc:`/plugins/fetchart`: Local image filenames are now used in alphabetical + order. * :doc:`/plugins/fetchart`: Fix a bug where cover art filenames could lack a ``.jpg`` extension. * :doc:`/plugins/lyrics`: Fix an exception with non-ASCII lyrics.