From 6572e1bf5abd134547f241b74ccb969d2b0415f8 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Mon, 9 Feb 2015 14:59:56 +0100 Subject: [PATCH 001/129] Fetchart: add empty album check to iTunes art Was causing some tests to fail in test_art.py:CombinedTest --- beetsplug/fetchart.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 1896552bf..7db05583b 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -142,6 +142,8 @@ class ITunesStore(ArtSource): def get(self, album): """Return art URL from iTunes Store given an album title. """ + if not (album.albumartist and album.album): + return search_string = (album.albumartist + ' ' + album.album).encode('utf-8') try: # Isolate bugs in the iTunes library while searching. From 7476d6be463399ce32359302dec4b275339461e5 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 9 Feb 2015 19:25:23 +0100 Subject: [PATCH 002/129] InvalidQuery*Error extend ParsingError And InvalidQueryArgumentTypeError does not extend TypeError anymore. --- beets/dbcore/query.py | 12 +++++++++--- test/test_library.py | 6 ++++-- test/test_query.py | 5 +++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 3727f6d7f..cd891148e 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -23,8 +23,14 @@ from beets import util from datetime import datetime, timedelta -class InvalidQueryError(ValueError): - """Represent any kind of invalid query +class ParsingError(ValueError): + """Abstract class for any unparseable user-requested album/query + specification. + """ + + +class InvalidQueryError(ParsingError): + """Represent any kind of invalid query. The query should be a unicode string or a list, which will be space-joined. """ @@ -35,7 +41,7 @@ class InvalidQueryError(ValueError): super(InvalidQueryError, self).__init__(message) -class InvalidQueryArgumentTypeError(TypeError): +class InvalidQueryArgumentTypeError(ParsingError): """Represent a query argument that could not be converted as expected. It exists to be caught in upper stack levels so a meaningful (i.e. with the diff --git a/test/test_library.py b/test/test_library.py index d3bfe1373..0bac0f173 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -31,7 +31,7 @@ from test._common import unittest from test._common import item import beets.library import beets.mediafile -import beets.dbcore +import beets.dbcore.query from beets import util from beets import plugins from beets import config @@ -1174,8 +1174,10 @@ class ItemReadTest(unittest.TestCase): class ParseQueryTest(unittest.TestCase): def test_parse_invalid_query_string(self): - with self.assertRaises(beets.dbcore.InvalidQueryError): + with self.assertRaises(beets.dbcore.InvalidQueryError) as raised: beets.library.parse_query_string('foo"', None) + self.assertIsInstance(raised.exception, + beets.dbcore.query.ParsingError) def suite(): diff --git a/test/test_query.py b/test/test_query.py index a32d8d60d..a9b1058bd 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -24,7 +24,8 @@ from test import helper import beets.library from beets import dbcore from beets.dbcore import types -from beets.dbcore.query import NoneQuery, InvalidQueryArgumentTypeError +from beets.dbcore.query import (NoneQuery, ParsingError, + InvalidQueryArgumentTypeError) from beets.library import Library, Item @@ -290,7 +291,7 @@ class GetTest(DummyDataTestCase): dbcore.query.RegexpQuery('year', '199(') self.assertIn('not a regular expression', unicode(raised.exception)) self.assertIn('unbalanced parenthesis', unicode(raised.exception)) - self.assertIsInstance(raised.exception, TypeError) + self.assertIsInstance(raised.exception, ParsingError) class MatchTest(_common.TestCase): From 60b1819db0de6d601a150e7ffd135d7e73667be4 Mon Sep 17 00:00:00 2001 From: Arturo R Date: Mon, 9 Feb 2015 09:55:32 -0800 Subject: [PATCH 003/129] Stop applying mp3gain directly to files. Fixes #1316 Update docs to remove non-existent `apply` option. --- beetsplug/replaygain.py | 1 - docs/changelog.rst | 2 ++ docs/plugins/replaygain.rst | 5 ----- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 4ec407cfb..ce41cad57 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -179,7 +179,6 @@ class CommandBackend(Backend): else: # Disable clipping warning. cmd = cmd + ['-c'] - cmd = cmd + ['-a' if is_album else '-r'] cmd = cmd + ['-d', bytes(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] diff --git a/docs/changelog.rst b/docs/changelog.rst index 81493428c..00a225b01 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -63,6 +63,8 @@ Core changes: Fixes: +* :doc:`/plugins/replaygain`: Stop applying replaygain directly to source files + when using the mp3gain backend. :bug:`1316` * :doc:`/plugins/lyrics`: Silence a warning about insecure requests in the new MusixMatch backend. :bug:`1204` * Fix a crash when ``beet`` is invoked without arguments. :bug:`1205` diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index d572902dd..d2584a648 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -92,11 +92,6 @@ configuration file. The available options are: These options only work with the "command" backend: -- **apply**: If you use a player that does not support ReplayGain - specifications, you can force the volume normalization by applying the gain - to the file via the ``apply`` option. This is a lossless and reversible - operation with no transcoding involved. - Default: ``no``. - **command**: The path to the ``mp3gain`` or ``aacgain`` executable (if beets cannot find it by itself). For example: ``/Applications/MacMP3Gain.app/Contents/Resources/aacgain``. From 4578c4f0e1659daab8d25c79d1f466ae9838d8c7 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 10 Feb 2015 15:30:15 +0100 Subject: [PATCH 004/129] Improve logging management for plugins Listeners get logging level modulations, like import_stages already did. Add extensive tests: - explicit plugin commands are logged like core beets (INFO or DEBUG) - import stages are more silent - event listeners are like import stages Delete @BeetsPlugin.listen decorator since listeners need plugin instance context to set its logging level. Improve #1244. Next is multiple verbosity levels. --- beets/plugins.py | 66 ++++++++++++--------------------- test/test_logging.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 43 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index bd285da2d..b7d4c9445 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -84,10 +84,8 @@ class BeetsPlugin(object): self._log = log.getChild(self.name) self._log.setLevel(logging.NOTSET) # Use `beets` logger level. - if beets.config['verbose']: - if not any(isinstance(f, PluginLogFilter) - for f in self._log.filters): - self._log.addFilter(PluginLogFilter(self)) + if not any(isinstance(f, PluginLogFilter) for f in self._log.filters): + self._log.addFilter(PluginLogFilter(self)) def commands(self): """Should return a list of beets.ui.Subcommand objects for @@ -106,23 +104,24 @@ class BeetsPlugin(object): return [self._set_log_level(logging.WARNING, import_stage) for import_stage in self.import_stages] - def _set_log_level(self, log_level, func): + def _set_log_level(self, base_log_level, func): """Wrap `func` to temporarily set this plugin's logger level to - `log_level` (and restore it after the function returns). - - The level is *not* adjusted when beets is in verbose - mode---i.e., the plugin logger continues to delegate to the base - beets logger. + `base_log_level` + config options (and restore it to NOTSET after the + function returns). """ @wraps(func) def wrapper(*args, **kwargs): - if not beets.config['verbose']: - old_log_level = self._log.level - self._log.setLevel(log_level) - result = func(*args, **kwargs) - if not beets.config['verbose']: - self._log.setLevel(old_log_level) - return result + assert self._log.level == logging.NOTSET + + log_level = base_log_level + if beets.config['verbose'].get(bool): + log_level -= 10 + self._log.setLevel(log_level) + + try: + return func(*args, **kwargs) + finally: + self._log.setLevel(logging.NOTSET) return wrapper def queries(self): @@ -183,34 +182,15 @@ class BeetsPlugin(object): listeners = None - @classmethod - def register_listener(cls, event, func): - """Add a function as a listener for the specified event. (An - imperative alternative to the @listen decorator.) + def register_listener(self, event, func): + """Add a function as a listener for the specified event. """ - if cls.listeners is None: - cls.listeners = defaultdict(list) - if func not in cls.listeners[event]: - cls.listeners[event].append(func) + func = self._set_log_level(logging.WARNING, func) - @classmethod - def listen(cls, event): - """Decorator that adds a function as an event handler for the - specified event (as a string). The parameters passed to function - will vary depending on what event occurred. - - The function should respond to named parameters. - function(**kwargs) will trap all arguments in a dictionary. - Example: - - >>> @MyPlugin.listen("imported") - >>> def importListener(**kwargs): - ... pass - """ - def helper(func): - cls.register_listener(event, func) - return func - return helper + if self.listeners is None: + self.listeners = defaultdict(list) + if func not in self.listeners[event]: + self.listeners[event].append(func) template_funcs = None template_fields = None diff --git a/test/test_logging.py b/test/test_logging.py index 864fd021a..bc8ade0aa 100644 --- a/test/test_logging.py +++ b/test/test_logging.py @@ -2,11 +2,15 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +import sys import logging as log from StringIO import StringIO import beets.logging as blog +from beets import plugins, ui +import beetsplug from test._common import unittest, TestCase +from test import helper class LoggingTest(TestCase): @@ -37,6 +41,90 @@ class LoggingTest(TestCase): self.assertTrue(stream.getvalue(), "foo oof baz") +class LoggingLevelTest(unittest.TestCase, helper.TestHelper): + class DummyModule(object): + class DummyPlugin(plugins.BeetsPlugin): + def __init__(self): + plugins.BeetsPlugin.__init__(self, 'dummy') + self.import_stages = [self.import_stage] + self.register_listener('dummy_event', self.listener) + + def log_all(self, name): + self._log.debug('debug ' + name) + self._log.info('info ' + name) + self._log.warning('warning ' + name) + + def commands(self): + cmd = ui.Subcommand('dummy') + cmd.func = lambda _, __, ___: self.log_all('cmd') + return (cmd,) + + def import_stage(self, session, task): + self.log_all('import_stage') + + def listener(self): + self.log_all('listener') + + def setUp(self): + sys.modules['beetsplug.dummy'] = self.DummyModule + beetsplug.dummy = self.DummyModule + self.setup_beets() + self.load_plugins('dummy') + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() + del beetsplug.dummy + sys.modules.pop('beetsplug.dummy') + + def test_command_logging(self): + self.config['verbose'] = False + with helper.capture_log() as logs: + self.run_command('dummy') + self.assertIn('dummy: warning cmd', logs) + self.assertIn('dummy: info cmd', logs) + self.assertNotIn('dummy: debug cmd', logs) + + self.config['verbose'] = True + with helper.capture_log() as logs: + self.run_command('dummy') + self.assertIn('dummy: warning cmd', logs) + self.assertIn('dummy: info cmd', logs) + self.assertIn('dummy: debug cmd', logs) + + def test_listener_logging(self): + self.config['verbose'] = False + with helper.capture_log() as logs: + plugins.send('dummy_event') + self.assertIn('dummy: warning listener', logs) + self.assertNotIn('dummy: info listener', logs) + self.assertNotIn('dummy: debug listener', logs) + + self.config['verbose'] = True + with helper.capture_log() as logs: + plugins.send('dummy_event') + self.assertIn('dummy: warning listener', logs) + self.assertIn('dummy: info listener', logs) + self.assertNotIn('dummy: debug listener', logs) + + def test_import_stage_logging(self): + self.config['verbose'] = False + with helper.capture_log() as logs: + importer = self.create_importer() + importer.run() + self.assertIn('dummy: warning import_stage', logs) + self.assertNotIn('dummy: info import_stage', logs) + self.assertNotIn('dummy: debug import_stage', logs) + + self.config['verbose'] = True + with helper.capture_log() as logs: + importer = self.create_importer() + importer.run() + self.assertIn('dummy: warning import_stage', logs) + self.assertIn('dummy: info import_stage', logs) + self.assertNotIn('dummy: debug import_stage', logs) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 327b62b6103771bd19465f10a6d4f0412c2b9f1c Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 10 Feb 2015 16:55:06 +0100 Subject: [PATCH 005/129] Improve logging management for plugins: fixes Delete the remaining usages of BeetsPlugin.listen(). Since BeetsPlugin.listeners are wrapped by a loglevel-setting function, we cannot easily check their unicity anymore. BeetsPlugin._raw_listeners set holds the raw listeners. Legacy plugins that did not handle enough arguments in their listenings functions may break: dedicated code is now deleted for it would not work with the decorated listeners. Tests got fixed. Some modifications were done empirically: if it passes then it's okay. --- beets/plugins.py | 19 ++++++++++--------- beetsplug/chroma.py | 2 +- beetsplug/fromfilename.py | 5 +++-- beetsplug/permissions.py | 5 +++-- test/test_plugins.py | 17 +++++++---------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index b7d4c9445..58d9bc0ee 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -18,7 +18,6 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) import traceback -import inspect import re from collections import defaultdict from functools import wraps @@ -180,17 +179,21 @@ class BeetsPlugin(object): mediafile.MediaFile.add_field(name, descriptor) library.Item._media_fields.add(name) + _raw_listeners = None listeners = None def register_listener(self, event, func): """Add a function as a listener for the specified event. """ - func = self._set_log_level(logging.WARNING, func) + wrapped_func = self._set_log_level(logging.WARNING, func) - if self.listeners is None: - self.listeners = defaultdict(list) - if func not in self.listeners[event]: - self.listeners[event].append(func) + cls = self.__class__ + if cls.listeners is None or cls._raw_listeners is None: + cls._raw_listeners = defaultdict(list) + cls.listeners = defaultdict(list) + if func not in cls._raw_listeners[event]: + cls._raw_listeners[event].append(func) + cls.listeners[event].append(wrapped_func) template_funcs = None template_fields = None @@ -440,9 +443,7 @@ def send(event, **arguments): results = [] for handler in event_handlers()[event]: # Don't break legacy plugins if we want to pass more arguments - argspec = inspect.getargspec(handler).args - args = dict((k, v) for k, v in arguments.items() if k in argspec) - result = handler(**args) + result = handler(**arguments) if result is not None: results.append(result) return results diff --git a/beetsplug/chroma.py b/beetsplug/chroma.py index 928f90479..8a83c0002 100644 --- a/beetsplug/chroma.py +++ b/beetsplug/chroma.py @@ -136,6 +136,7 @@ class AcoustidPlugin(plugins.BeetsPlugin): if self.config['auto']: self.register_listener('import_task_start', self.fingerprint_task) + self.register_listener('import_task_apply', apply_acoustid_metadata) def fingerprint_task(self, task, session): return fingerprint_task(self._log, task, session) @@ -211,7 +212,6 @@ def fingerprint_task(log, task, session): acoustid_match(log, item.path) -@AcoustidPlugin.listen('import_task_apply') def apply_acoustid_metadata(task, session): """Apply Acoustid metadata (fingerprint and ID) to the task's items. """ diff --git a/beetsplug/fromfilename.py b/beetsplug/fromfilename.py index 169c02ff6..dc040a0a2 100644 --- a/beetsplug/fromfilename.py +++ b/beetsplug/fromfilename.py @@ -140,10 +140,11 @@ def apply_matches(d): # Plugin structure and hook into import process. class FromFilenamePlugin(plugins.BeetsPlugin): - pass + def __init__(self): + super(FromFilenamePlugin, self).__init__() + self.register_listener('import_task_start', filename_task) -@FromFilenamePlugin.listen('import_task_start') def filename_task(task, session): """Examine each item in the task to see if we can extract a title from the filename. Try to match all filenames to a number of diff --git a/beetsplug/permissions.py b/beetsplug/permissions.py index 256f09e52..5068c2a0a 100644 --- a/beetsplug/permissions.py +++ b/beetsplug/permissions.py @@ -37,9 +37,10 @@ class Permissions(BeetsPlugin): u'file': 644 }) + self.register_listener('item_imported', permissions) + self.register_listener('album_imported', permissions) + -@Permissions.listen('item_imported') -@Permissions.listen('album_imported') def permissions(lib, item=None, album=None): """Running the permission fixer. """ diff --git a/test/test_plugins.py b/test/test_plugins.py index d46c5bd5d..c9c5be502 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -35,6 +35,7 @@ class TestHelper(helper.TestHelper): def setup_plugin_loader(self): # FIXME the mocking code is horrific, but this is the lowest and # earliest level of the plugin mechanism we can hook into. + self.load_plugins() self._plugin_loader_patch = patch('beets.plugins.load_plugins') self._plugin_classes = set() load_plugins = self._plugin_loader_patch.start() @@ -95,7 +96,7 @@ class ItemWriteTest(unittest.TestCase, TestHelper): class EventListenerPlugin(plugins.BeetsPlugin): pass - self.event_listener_plugin = EventListenerPlugin + self.event_listener_plugin = EventListenerPlugin() self.register_plugin(EventListenerPlugin) def tearDown(self): @@ -298,19 +299,15 @@ class ListenersTest(unittest.TestCase, TestHelper): pass d = DummyPlugin() - self.assertEqual(DummyPlugin.listeners['cli_exit'], [d.dummy]) + self.assertEqual(DummyPlugin._raw_listeners['cli_exit'], [d.dummy]) d2 = DummyPlugin() - DummyPlugin.register_listener('cli_exit', d.dummy) - self.assertEqual(DummyPlugin.listeners['cli_exit'], + self.assertEqual(DummyPlugin._raw_listeners['cli_exit'], [d.dummy, d2.dummy]) - @DummyPlugin.listen('cli_exit') - def dummy(lib): - pass - - self.assertEqual(DummyPlugin.listeners['cli_exit'], - [d.dummy, d2.dummy, dummy]) + d.register_listener('cli_exit', d2.dummy) + self.assertEqual(DummyPlugin._raw_listeners['cli_exit'], + [d.dummy, d2.dummy]) def suite(): From f1e13cf886a23a3b933a5f39163b32c44bdda5e6 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 10 Feb 2015 17:13:15 +0100 Subject: [PATCH 006/129] Offer verbose and very verbose modes 'verbose' is now an int and not a boolean. '-v' is level 1, '-vv' level 2. In the configuration it can be set with 'verbose: 1' or 'verbose: 2'. Improve #1244: auditing current log levels of plugins remains. --- beets/config_default.yaml | 2 +- beets/plugins.py | 5 ++--- beets/ui/__init__.py | 4 ++-- test/helper.py | 2 +- test/test_logging.py | 38 +++++++++++++++++++++++++++----------- test/test_spotify.py | 2 +- 6 files changed, 34 insertions(+), 19 deletions(-) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index c448585d9..b98ffbfc6 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -43,7 +43,7 @@ pluginpath: [] threaded: yes timeout: 5.0 per_disc_numbering: no -verbose: no +verbose: 0 terminal_encoding: utf8 original_date: no id3v23: no diff --git a/beets/plugins.py b/beets/plugins.py index 58d9bc0ee..9235883d0 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -112,9 +112,8 @@ class BeetsPlugin(object): def wrapper(*args, **kwargs): assert self._log.level == logging.NOTSET - log_level = base_log_level - if beets.config['verbose'].get(bool): - log_level -= 10 + verbosity = beets.config['verbose'].get(int) + log_level = max(logging.DEBUG, base_log_level - 10 * verbosity) self._log.setLevel(log_level) try: diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 02a7a9478..e041db95e 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -858,7 +858,7 @@ def _configure(options): config.set_args(options) # Configure the logger. - if config['verbose'].get(bool): + if config['verbose'].get(int): log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) @@ -917,7 +917,7 @@ def _raw_main(args, lib=None): help='library database file to use') parser.add_option('-d', '--directory', dest='directory', help="destination music directory") - parser.add_option('-v', '--verbose', dest='verbose', action='store_true', + parser.add_option('-v', '--verbose', dest='verbose', action='count', help='print debugging information') parser.add_option('-c', '--config', dest='config', help='path to configuration file') diff --git a/test/helper.py b/test/helper.py index e929eecff..8d0dbf8a6 100644 --- a/test/helper.py +++ b/test/helper.py @@ -167,7 +167,7 @@ class TestHelper(object): self.config.read() self.config['plugins'] = [] - self.config['verbose'] = True + self.config['verbose'] = 1 self.config['ui']['color'] = False self.config['threaded'] = False diff --git a/test/test_logging.py b/test/test_logging.py index bc8ade0aa..a4e2cfbe7 100644 --- a/test/test_logging.py +++ b/test/test_logging.py @@ -78,37 +78,45 @@ class LoggingLevelTest(unittest.TestCase, helper.TestHelper): sys.modules.pop('beetsplug.dummy') def test_command_logging(self): - self.config['verbose'] = False + self.config['verbose'] = 0 with helper.capture_log() as logs: self.run_command('dummy') self.assertIn('dummy: warning cmd', logs) self.assertIn('dummy: info cmd', logs) self.assertNotIn('dummy: debug cmd', logs) - self.config['verbose'] = True - with helper.capture_log() as logs: - self.run_command('dummy') - self.assertIn('dummy: warning cmd', logs) - self.assertIn('dummy: info cmd', logs) - self.assertIn('dummy: debug cmd', logs) + for level in (1, 2): + self.config['verbose'] = level + with helper.capture_log() as logs: + self.run_command('dummy') + self.assertIn('dummy: warning cmd', logs) + self.assertIn('dummy: info cmd', logs) + self.assertIn('dummy: debug cmd', logs) def test_listener_logging(self): - self.config['verbose'] = False + self.config['verbose'] = 0 with helper.capture_log() as logs: plugins.send('dummy_event') self.assertIn('dummy: warning listener', logs) self.assertNotIn('dummy: info listener', logs) self.assertNotIn('dummy: debug listener', logs) - self.config['verbose'] = True + self.config['verbose'] = 1 with helper.capture_log() as logs: plugins.send('dummy_event') self.assertIn('dummy: warning listener', logs) self.assertIn('dummy: info listener', logs) self.assertNotIn('dummy: debug listener', logs) + self.config['verbose'] = 2 + with helper.capture_log() as logs: + plugins.send('dummy_event') + self.assertIn('dummy: warning listener', logs) + self.assertIn('dummy: info listener', logs) + self.assertIn('dummy: debug listener', logs) + def test_import_stage_logging(self): - self.config['verbose'] = False + self.config['verbose'] = 0 with helper.capture_log() as logs: importer = self.create_importer() importer.run() @@ -116,7 +124,7 @@ class LoggingLevelTest(unittest.TestCase, helper.TestHelper): self.assertNotIn('dummy: info import_stage', logs) self.assertNotIn('dummy: debug import_stage', logs) - self.config['verbose'] = True + self.config['verbose'] = 1 with helper.capture_log() as logs: importer = self.create_importer() importer.run() @@ -124,6 +132,14 @@ class LoggingLevelTest(unittest.TestCase, helper.TestHelper): self.assertIn('dummy: info import_stage', logs) self.assertNotIn('dummy: debug import_stage', logs) + self.config['verbose'] = 2 + with helper.capture_log() as logs: + importer = self.create_importer() + importer.run() + self.assertIn('dummy: warning import_stage', logs) + self.assertIn('dummy: info import_stage', logs) + self.assertIn('dummy: debug import_stage', logs) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/test/test_spotify.py b/test/test_spotify.py index 3d4d75bde..3025163bb 100644 --- a/test/test_spotify.py +++ b/test/test_spotify.py @@ -17,7 +17,7 @@ class ArgumentsMock(object): def __init__(self, mode, show_failures): self.mode = mode self.show_failures = show_failures - self.verbose = True + self.verbose = 1 def _params(url): From a014750e2dba00386c300cc62e06898bf591c472 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 10 Feb 2015 17:26:56 +0100 Subject: [PATCH 007/129] Update docs: mention multi-level logging If you think what I wrote suck, it's because it does. --- docs/changelog.rst | 7 ++++++- docs/dev/plugins.rst | 27 +++++++++++++++++---------- docs/reference/cli.rst | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81493428c..b8bdc15cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog Features: +* Verbosity is now an integer in the configuration since multiple levels are + supported (like e.g. apt-get). On the CLI one can stack verbose flags (i.e. + `-vv`). :bug:`1244` * The summary shown to compare duplicate albums during import now displays the old and new filesizes. :bug:`1291` * The colors used are now configurable via the new config option ``colors``, @@ -106,7 +109,9 @@ For developers: * The logging system in beets has been overhauled. Plugins now each have their own logger, which helps by automatically adjusting the verbosity level in - import mode and by prefixing the plugin's name. Also, logging calls can (and + import mode and by prefixing the plugin's name. Logging levels are + dynamically set when a plugin is called, depending on how it is called + (import stage, event or direct command). Finally, logging calls can (and should!) use modern ``{}``-style string formatting lazily. See :ref:`plugin-logging` in the plugin API docs. * A new ``import_task_created`` event lets you manipulate import tasks diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 97da193f0..f448b5dfa 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -480,14 +480,21 @@ str.format-style string formatting. So you can write logging calls like this:: .. _PEP 3101: https://www.python.org/dev/peps/pep-3101/ .. _standard Python logging module: https://docs.python.org/2/library/logging.html -The per-plugin loggers have two convenient features: +When beets is in verbose mode, plugin messages are prefixed with the plugin +name to make them easier to see. + +What messages will be logged depends on the logging level and the action +performed: + +* On import stages and event, the default is ``WARNING`` messages. +* On direct actions, the default is ``INFO`` and ``WARNING`` message. + +The verbosity can be increased with ``--verbose`` flags: each flags lowers the +level by a notch. + +This addresses a common pattern where plugins need to use the same code for a +command and an import stage, but the command needs to print more messages than +the import stage. (For example, you'll want to log "found lyrics for this song" +when you're run explicitly as a command, but you don't want to noisily +interrupt the importer interface when running automatically.) -* When beets is in verbose mode, messages are prefixed with the plugin name to - make them easier to see. -* Messages at the ``INFO`` logging level are hidden when the plugin is running - in an importer stage (see above). This addresses a common pattern where - plugins need to use the same code for a command and an import stage, but the - command needs to print more messages than the import stage. (For example, - you'll want to log "found lyrics for this song" when you're run explicitly - as a command, but you don't want to noisily interrupt the importer interface - when running automatically.) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index e569c073c..f889be678 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -369,7 +369,7 @@ import ...``. * ``-l LIBPATH``: specify the library database file to use. * ``-d DIRECTORY``: specify the library root directory. * ``-v``: verbose mode; prints out a deluge of debugging information. Please use - this flag when reporting bugs. + this flag when reporting bugs. It can be stacked twice. * ``-c FILE``: read a specified YAML :doc:`configuration file `. Beets also uses the ``BEETSDIR`` environment variable to look for From 228e5c043281e1ca6b969fba6e5d075f8436237a Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Tue, 10 Feb 2015 18:17:37 +0100 Subject: [PATCH 008/129] Importer metadata source is set as a field TrackInfo and AlbumInfo were already keeping track of this, so just had to add it as an actual field to Item and Album See #1311 --- beets/autotag/__init__.py | 5 ++++- beets/autotag/hooks.py | 2 ++ beets/autotag/mb.py | 1 + beets/library.py | 5 +++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index fab0fbbae..2f632f1ee 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -42,6 +42,8 @@ def apply_item_metadata(item, track_info): item.mb_trackid = track_info.track_id if track_info.artist_id: item.mb_artistid = track_info.artist_id + if track_info.data_source: + item.data_source = track_info.data_source # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? @@ -125,7 +127,8 @@ def apply_metadata(album_info, mapping): 'language', 'country', 'albumstatus', - 'albumdisambig'): + 'albumdisambig', + 'data_source',): value = getattr(album_info, field) if value is not None: item[field] = value diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 12d11a0b3..3a4f96548 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -137,6 +137,8 @@ class TrackInfo(object): - ``artist_sort``: name of the track artist for sorting - ``disctitle``: name of the individual medium (subtitle) - ``artist_credit``: Recording-specific artist name + - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) + - ``data_url``: The data source release URL. Only ``title`` and ``track_id`` are required. The rest of the fields may be None. The indices ``index``, ``medium``, and ``medium_index`` diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index c25599751..b9402f3dc 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -161,6 +161,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, medium=medium, medium_index=medium_index, medium_total=medium_total, + data_source='MusicBrainz', data_url=track_url(recording['id']), ) diff --git a/beets/library.py b/beets/library.py index a57ed1642..fed42da5f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -388,6 +388,8 @@ class Item(LibModel): 'channels': types.INTEGER, 'mtime': DateType(), 'added': DateType(), + + 'data_source': types.STRING, } _search_fields = ('artist', 'title', 'comments', @@ -790,6 +792,8 @@ class Album(LibModel): 'original_year': types.PaddedInt(4), 'original_month': types.PaddedInt(2), 'original_day': types.PaddedInt(2), + + 'data_source': types.STRING, } _search_fields = ('album', 'albumartist', 'genre') @@ -832,6 +836,7 @@ class Album(LibModel): 'original_year', 'original_month', 'original_day', + 'data_source', ] """List of keys that are set on an album's items. """ From 5a1f499c6413a054fb53a527a61dca92f6032b00 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 11 Feb 2015 09:06:13 +0100 Subject: [PATCH 009/129] Discogs plugin: fix event listener params Also delete related deprecated comment. --- beets/plugins.py | 1 - beetsplug/discogs.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 9235883d0..59aa9763c 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -441,7 +441,6 @@ def send(event, **arguments): log.debug(u'Sending event: {0}', event) results = [] for handler in event_handlers()[event]: - # Don't break legacy plugins if we want to pass more arguments result = handler(**arguments) if result is not None: results.append(result) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 267041bcc..a8b02f349 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -58,7 +58,7 @@ class DiscogsPlugin(BeetsPlugin): self.discogs_client = None self.register_listener('import_begin', self.setup) - def setup(self): + def setup(self, session): """Create the `discogs_client` field. Authenticate if necessary. """ c_key = self.config['apikey'].get(unicode) From 06cc5346f243e49f599208f62086bbd82523f050 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 11 Feb 2015 09:11:34 +0100 Subject: [PATCH 010/129] Make less assumptions on plugin log levels --- beets/plugins.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 59aa9763c..bee4d9f32 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -105,13 +105,15 @@ class BeetsPlugin(object): def _set_log_level(self, base_log_level, func): """Wrap `func` to temporarily set this plugin's logger level to - `base_log_level` + config options (and restore it to NOTSET after the - function returns). + `base_log_level` + config options (and restore it to its previous + value after the function returns). + + Note that that value may not be NOTSET, e.g. if a plugin import stage + triggers an event that is listened this very same plugin """ @wraps(func) def wrapper(*args, **kwargs): - assert self._log.level == logging.NOTSET - + old_log_level = self._log.level verbosity = beets.config['verbose'].get(int) log_level = max(logging.DEBUG, base_log_level - 10 * verbosity) self._log.setLevel(log_level) @@ -119,7 +121,7 @@ class BeetsPlugin(object): try: return func(*args, **kwargs) finally: - self._log.setLevel(logging.NOTSET) + self._log.setLevel(old_log_level) return wrapper def queries(self): From 20ae26dd776b1a5a3ece81b7375913d8d457505e Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Wed, 11 Feb 2015 10:14:56 +0100 Subject: [PATCH 011/129] Importer metadata source is set as a field: tests See #1311 --- test/test_autotag.py | 7 +++++++ test/test_importer.py | 7 +++++++ test/test_mb.py | 5 +++++ 3 files changed, 19 insertions(+) diff --git a/test/test_autotag.py b/test/test_autotag.py index 7e018a4c1..6393a0f2f 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -789,6 +789,13 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[0].month, 2) self.assertEqual(self.items[0].day, 3) + def test_data_source_applied(self): + my_info = copy.deepcopy(self.info) + my_info.data_source = 'MusicBrainz' + self._apply(info=my_info) + + self.assertEqual(self.items[0].data_source, 'MusicBrainz') + class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def setUp(self): diff --git a/test/test_importer.py b/test/test_importer.py index 45901bc6a..200c10ec6 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -625,6 +625,13 @@ class ImportTest(_common.TestCase, ImportHelper): self.assertIn('No files imported from {0}'.format(import_dir), logs) + def test_asis_no_data_source(self): + self.assertEqual(self.lib.items().get(), None) + + self.importer.add_choice(importer.action.ASIS) + self.importer.run() + self.assertEqual(self.lib.items().get().data_source, '') + class ImportTracksTest(_common.TestCase, ImportHelper): """Test TRACKS and APPLY choice. diff --git a/test/test_mb.py b/test/test_mb.py index 3e5f721ff..c1c93bbdc 100644 --- a/test/test_mb.py +++ b/test/test_mb.py @@ -316,6 +316,11 @@ class MBAlbumInfoTest(_common.TestCase): self.assertEqual(track.artist_sort, 'TRACK ARTIST SORT NAME') self.assertEqual(track.artist_credit, 'TRACK ARTIST CREDIT') + def test_data_source(self): + release = self._make_release() + d = mb.album_info(release) + self.assertEqual(d.data_source, 'MusicBrainz') + class ParseIDTest(_common.TestCase): def test_parse_id_correct(self): From c286ea38de71fdd0ba998fe4f0e26088ced81ea8 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Wed, 11 Feb 2015 10:29:13 +0100 Subject: [PATCH 012/129] Importer metadata source is set as a field: docs docs + changelog See #1311 --- docs/changelog.rst | 2 ++ docs/reference/pathformat.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81493428c..cc1f5f3f0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,8 @@ Features: * A new ``filesize`` field on items indicates the number of bytes in the file. :bug:`1291` * The number of missing/unmatched tracks is shown during import. :bug:`1088` +* The data source used during import (e.g., MusicBrainz) is now saved as a + track/album's field :bug:`1311` Core changes: diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index 8efe54cd2..f4bef453a 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -229,6 +229,8 @@ Library metadata: * mtime: The modification time of the audio file. * added: The date and time that the music was added to your library. * path: The item's filename. +* data_source: The data source used during import to tag the item + (e.g., 'MusicBrainz') .. _templ_plugins: From d267741ff3dcbebf7c5b0b576d26c7d353252b0d Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 11 Feb 2015 16:19:19 +0100 Subject: [PATCH 013/129] Delete 'format' variables that shadow the built-in Also cleanup the 'the' plugin a bit: delete unused variables. Relates to #1300. --- beets/library.py | 4 ++-- beets/ui/commands.py | 4 ++-- beetsplug/convert.py | 34 +++++++++++++++++----------------- beetsplug/the.py | 6 ------ test/test_the.py | 1 - 5 files changed, 21 insertions(+), 28 deletions(-) diff --git a/beets/library.py b/beets/library.py index a57ed1642..b1093676f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1296,11 +1296,11 @@ class DefaultTemplateFunctions(object): return unidecode(s) @staticmethod - def tmpl_time(s, format): + def tmpl_time(s, fmt): """Format a time value using `strftime`. """ cur_fmt = beets.config['time_format'].get(unicode) - return time.strftime(format, time.strptime(s, cur_fmt)) + return time.strftime(fmt, time.strptime(s, cur_fmt)) def tmpl_aunique(self, keys=None, disam=None): """Generate a string that is guaranteed to be unique among all diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 32201f58d..6612ca2cd 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -438,8 +438,8 @@ def summarize_items(items, singleton): summary_parts.append(items[0].format) else: # Enumerate all the formats. - for format, count in format_counts.iteritems(): - summary_parts.append('{0} {1}'.format(format, count)) + for fmt, count in format_counts.iteritems(): + summary_parts.append('{0} {1}'.format(fmt, count)) average_bitrate = sum([item.bitrate for item in items]) / len(items) total_duration = sum([item.length for item in items]) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 2f49b5ef8..352c3c94d 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -49,25 +49,25 @@ def replace_ext(path, ext): return os.path.splitext(path)[0] + b'.' + ext -def get_format(format=None): +def get_format(fmt=None): """Return the command tempate and the extension from the config. """ - if not format: - format = config['convert']['format'].get(unicode).lower() - format = ALIASES.get(format, format) + if not fmt: + fmt = config['convert']['format'].get(unicode).lower() + fmt = ALIASES.get(fmt, fmt) try: - format_info = config['convert']['formats'][format].get(dict) + format_info = config['convert']['formats'][fmt].get(dict) command = format_info['command'] extension = format_info['extension'] except KeyError: raise ui.UserError( u'convert: format {0} needs "command" and "extension" fields' - .format(format) + .format(fmt) ) except ConfigTypeError: - command = config['convert']['formats'][format].get(bytes) - extension = format + command = config['convert']['formats'][fmt].get(bytes) + extension = fmt # Convenience and backwards-compatibility shortcuts. keys = config['convert'].keys() @@ -84,7 +84,7 @@ def get_format(format=None): return (command.encode('utf8'), extension.encode('utf8')) -def should_transcode(item, format): +def should_transcode(item, fmt): """Determine whether the item should be transcoded as part of conversion (i.e., its bitrate is high or it has the wrong format). """ @@ -92,7 +92,7 @@ def should_transcode(item, format): not (item.format.lower() in LOSSLESS_FORMATS): return False maxbr = config['convert']['max_bitrate'].get(int) - return format.lower() != item.format.lower() or \ + return fmt.lower() != item.format.lower() or \ item.bitrate >= 1000 * maxbr @@ -209,9 +209,9 @@ class ConvertPlugin(BeetsPlugin): self._log.info(u'Finished encoding {0}', util.displayable_path(source)) - def convert_item(self, dest_dir, keep_new, path_formats, format, + def convert_item(self, dest_dir, keep_new, path_formats, fmt, pretend=False): - command, ext = get_format(format) + command, ext = get_format(fmt) item, original, converted = None, None, None while True: item = yield (item, original, converted) @@ -224,11 +224,11 @@ class ConvertPlugin(BeetsPlugin): if keep_new: original = dest converted = item.path - if should_transcode(item, format): + if should_transcode(item, fmt): converted = replace_ext(converted, ext) else: original = item.path - if should_transcode(item, format): + if should_transcode(item, fmt): dest = replace_ext(dest, ext) converted = dest @@ -254,7 +254,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(original)) util.move(item.path, original) - if should_transcode(item, format): + if should_transcode(item, fmt): try: self.encode(command, original, converted, pretend) except subprocess.CalledProcessError: @@ -386,8 +386,8 @@ class ConvertPlugin(BeetsPlugin): """Transcode a file automatically after it is imported into the library. """ - format = self.config['format'].get(unicode).lower() - if should_transcode(item, format): + fmt = self.config['format'].get(unicode).lower() + if should_transcode(item, fmt): command, ext = get_format() fd, dest = tempfile.mkstemp('.' + ext) os.close(fd) diff --git a/beetsplug/the.py b/beetsplug/the.py index 3fb16dcf9..538b81245 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -30,12 +30,6 @@ FORMAT = u'{0}, {1}' class ThePlugin(BeetsPlugin): - _instance = None - - the = True - a = True - format = u'' - strip = False patterns = [] def __init__(self): diff --git a/test/test_the.py b/test/test_the.py index 1b33c390e..e04db1aec 100644 --- a/test/test_the.py +++ b/test/test_the.py @@ -44,7 +44,6 @@ class ThePluginTest(_common.TestCase): def test_template_function_with_defaults(self): ThePlugin().patterns = [PATTERN_THE, PATTERN_A] - ThePlugin().format = FORMAT self.assertEqual(ThePlugin().the_template_func('The The'), 'The, The') self.assertEqual(ThePlugin().the_template_func('An A'), 'A, An') From e1e46df1b3f9f38028bf4e84eaa1f335e2ff48df Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 12 Feb 2015 11:56:54 +0100 Subject: [PATCH 014/129] Fix path truncation test: really use bytes Test is to use bytes but because of __future__.unicode_literals it used unicode. --- test/test_library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_library.py b/test/test_library.py index 0bac0f173..3515c855c 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -1007,8 +1007,8 @@ class PathStringTest(_common.TestCase): class PathTruncationTest(_common.TestCase): def test_truncate_bytestring(self): with _common.platform_posix(): - p = util.truncate_path('abcde/fgh', 4) - self.assertEqual(p, 'abcd/fgh') + p = util.truncate_path(b'abcde/fgh', 4) + self.assertEqual(p, b'abcd/fgh') def test_truncate_unicode(self): with _common.platform_posix(): From 9cdd541943a481cf7d866cc28f7355f634d0bf1d Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Fri, 13 Feb 2015 12:24:21 +0100 Subject: [PATCH 015/129] Error handling for 'filesize' field - Logs a warning and returns 0 if getsize fails - Add tests for this Fix #1326 --- beets/library.py | 9 ++++++++- test/test_library.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index a57ed1642..4cf1d6d74 100644 --- a/beets/library.py +++ b/beets/library.py @@ -421,7 +421,7 @@ class Item(LibModel): getters = plugins.item_field_getters() getters['singleton'] = lambda i: i.album_id is None # Filesize is given in bytes - getters['filesize'] = lambda i: os.path.getsize(syspath(i.path)) + getters['filesize'] = lambda i: i.try_filesize() return getters @classmethod @@ -605,6 +605,13 @@ class Item(LibModel): """ return int(os.path.getmtime(syspath(self.path))) + def try_filesize(self): + try: + return os.path.getsize(syspath(self.path)) + except (OSError, Exception) as exc: + log.warning(u'could not get filesize: {0}', exc) + return 0 + # Model methods. def remove(self, delete=False, with_album=True): diff --git a/test/test_library.py b/test/test_library.py index 3515c855c..6bb88076e 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -1172,6 +1172,22 @@ class ItemReadTest(unittest.TestCase): item.read('/thisfiledoesnotexist') +class FilesizeTest(unittest.TestCase, TestHelper): + def setUp(self): + self.setup_beets() + + def tearDown(self): + self.teardown_beets() + + def test_filesize(self): + item = self.add_item_fixture() + self.assertNotEquals(item.filesize, 0) + + def test_nonexistent_file(self): + item = beets.library.Item() + self.assertEqual(item.filesize, 0) + + class ParseQueryTest(unittest.TestCase): def test_parse_invalid_query_string(self): with self.assertRaises(beets.dbcore.InvalidQueryError) as raised: From b80713ce5b76a5f2db0d73da4f5f50a31c72e116 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 13 Feb 2015 17:37:55 -0800 Subject: [PATCH 016/129] Lambda-free reference to instance method Amends 9cdd541943a481cf7d866cc28f7355f634d0bf1d, the fix for #1326. --- beets/library.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index 4cf1d6d74..8d44455e8 100644 --- a/beets/library.py +++ b/beets/library.py @@ -420,8 +420,7 @@ class Item(LibModel): def _getters(cls): getters = plugins.item_field_getters() getters['singleton'] = lambda i: i.album_id is None - # Filesize is given in bytes - getters['filesize'] = lambda i: i.try_filesize() + getters['filesize'] = Item.try_filesize # In bytes. return getters @classmethod @@ -606,6 +605,10 @@ class Item(LibModel): return int(os.path.getmtime(syspath(self.path))) def try_filesize(self): + """Get the size of the underlying file in bytes. + + If the file is missing, return 0 (and log a warning). + """ try: return os.path.getsize(syspath(self.path)) except (OSError, Exception) as exc: From 8f7f7b92b8b028c8a4ff4b23a2e987ab988c196c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 13 Feb 2015 17:40:32 -0800 Subject: [PATCH 017/129] play: unicode_literals fix --- beetsplug/play.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/play.py b/beetsplug/play.py index f6a59098f..5de5dcf32 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -94,9 +94,9 @@ def play_music(lib, opts, args, log): m3u = NamedTemporaryFile('w', suffix='.m3u', delete=False) for item in paths: if relative_to: - m3u.write(relpath(item, relative_to) + '\n') + m3u.write(relpath(item, relative_to) + b'\n') else: - m3u.write(item + '\n') + m3u.write(item + b'\n') m3u.close() command.append(m3u.name) From 13e47472ac7622b57178c734b1933d826e8b9fdb Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 15 Feb 2015 15:34:02 +0100 Subject: [PATCH 018/129] Play plugin: byte newlines in files written Avoid unicode errors when writing '\n' around file paths. --- beetsplug/play.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/play.py b/beetsplug/play.py index f6a59098f..5de5dcf32 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -94,9 +94,9 @@ def play_music(lib, opts, args, log): m3u = NamedTemporaryFile('w', suffix='.m3u', delete=False) for item in paths: if relative_to: - m3u.write(relpath(item, relative_to) + '\n') + m3u.write(relpath(item, relative_to) + b'\n') else: - m3u.write(item + '\n') + m3u.write(item + b'\n') m3u.close() command.append(m3u.name) From 1555d3fe177a3f32291d4f5098120b7f1c309bb4 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Sun, 15 Feb 2015 17:46:00 +0100 Subject: [PATCH 019/129] Importer metadata source is saved as flex attr Saving a file "as is" keeps the data_source attribute unset --- beets/library.py | 12 ++++++------ docs/changelog.rst | 2 +- docs/reference/pathformat.rst | 2 -- test/test_importer.py | 4 +++- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/beets/library.py b/beets/library.py index fed42da5f..7d9d69988 100644 --- a/beets/library.py +++ b/beets/library.py @@ -388,13 +388,15 @@ class Item(LibModel): 'channels': types.INTEGER, 'mtime': DateType(), 'added': DateType(), - - 'data_source': types.STRING, } _search_fields = ('artist', 'title', 'comments', 'album', 'albumartist', 'genre') + _types = { + 'data_source': types.STRING, + } + _media_fields = set(MediaFile.readable_fields()) \ .intersection(_fields.keys()) """Set of item fields that are backed by `MediaFile` fields. @@ -792,14 +794,13 @@ class Album(LibModel): 'original_year': types.PaddedInt(4), 'original_month': types.PaddedInt(2), 'original_day': types.PaddedInt(2), - - 'data_source': types.STRING, } _search_fields = ('album', 'albumartist', 'genre') _types = { - 'path': PathType(), + 'path': PathType(), + 'data_source': types.STRING, } _sorts = { @@ -836,7 +837,6 @@ class Album(LibModel): 'original_year', 'original_month', 'original_day', - 'data_source', ] """List of keys that are set on an album's items. """ diff --git a/docs/changelog.rst b/docs/changelog.rst index cc1f5f3f0..329770102 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -43,7 +43,7 @@ Features: :bug:`1291` * The number of missing/unmatched tracks is shown during import. :bug:`1088` * The data source used during import (e.g., MusicBrainz) is now saved as a - track/album's field :bug:`1311` + flexible attribute `data_source` of an Item/Album. :bug:`1311` Core changes: diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index f4bef453a..8efe54cd2 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -229,8 +229,6 @@ Library metadata: * mtime: The modification time of the audio file. * added: The date and time that the music was added to your library. * path: The item's filename. -* data_source: The data source used during import to tag the item - (e.g., 'MusicBrainz') .. _templ_plugins: diff --git a/test/test_importer.py b/test/test_importer.py index 200c10ec6..a9863b926 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -630,7 +630,9 @@ class ImportTest(_common.TestCase, ImportHelper): self.importer.add_choice(importer.action.ASIS) self.importer.run() - self.assertEqual(self.lib.items().get().data_source, '') + + with self.assertRaises(AttributeError): + self.lib.items().get().data_source class ImportTracksTest(_common.TestCase, ImportHelper): From eafdd9e3790ee845e6a992305fa82c45ef14db1b Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Sun, 15 Feb 2015 19:39:39 +0100 Subject: [PATCH 020/129] Extraction of cover art of albums with non ascii characters lead to a crash. --- beetsplug/embedart.py | 5 +++-- docs/changelog.rst | 2 ++ test/test_embedart.py | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index b705ed7e7..f567e0d2c 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -107,7 +107,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): u"for -n") return for album in lib.albums(decargs(args)): - artpath = normpath(os.path.join(album.path, filename)) + albumpath = album.path.decode('utf-8') + artpath = normpath(os.path.join(albumpath, filename)) artpath = self.extract_first(artpath, album.items()) if artpath and opts.associate: album.set_art(artpath) @@ -265,7 +266,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): self._log.warning(u'Unknown image type in {0}.', displayable_path(item.path)) return - outpath += '.' + ext + outpath = outpath.decode('utf-8') + '.' + ext self._log.info(u'Extracting album art from: {0} to: {1}', item, displayable_path(outpath)) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81493428c..ec60f5abf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -101,6 +101,8 @@ Fixes: Unicode filenames. :bug:`1297` * :doc:`/plugins/discogs`: Handle and log more kinds of communication errors. :bug:`1299` :bug:`1305` +* :doc:`/plugins/embedart`: Fix a crash that occured when the album path + contains non ascii characters. For developers: diff --git a/test/test_embedart.py b/test/test_embedart.py index 6347c32d1..5b49e6e66 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -16,6 +16,7 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) import os.path +import shutil from mock import Mock, patch from test import _common @@ -41,7 +42,7 @@ def require_artresizer_compare(test): return wrapper -class EmbedartCliTest(unittest.TestCase, TestHelper): +class EmbedartCliTest(_common.TestCase, TestHelper): small_artpath = os.path.join(_common.RSRC, 'image-2x3.jpg') abbey_artpath = os.path.join(_common.RSRC, 'abbey.jpg') @@ -114,6 +115,19 @@ class EmbedartCliTest(unittest.TestCase, TestHelper): 'Image written is not {0}'.format( self.abbey_similarpath)) + def test_non_ascii_album_path(self): + import_dir = os.path.join(self.temp_dir, 'testsrcdir\xe4') + resource_path = os.path.join(_common.RSRC, 'image.mp3') + album = self.add_album_fixture() + trackpath = album.items()[0].path + albumpath = album.path + shutil.copy(resource_path, trackpath.decode('utf-8')) + + self.run_command('extractart', '-n', 'extracted') + + self.assertExists(os.path.join(albumpath.decode('utf-8'), + 'extracted.png')) + class EmbedartTest(unittest.TestCase): @patch('beetsplug.embedart.subprocess') From b3fc489305156dfee4247df83c14272fb1014702 Mon Sep 17 00:00:00 2001 From: Malte Ried Date: Sun, 15 Feb 2015 19:50:11 +0100 Subject: [PATCH 021/129] Fixed the flake8 check... --- test/test_embedart.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_embedart.py b/test/test_embedart.py index 5b49e6e66..729f5853d 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -116,7 +116,6 @@ class EmbedartCliTest(_common.TestCase, TestHelper): self.abbey_similarpath)) def test_non_ascii_album_path(self): - import_dir = os.path.join(self.temp_dir, 'testsrcdir\xe4') resource_path = os.path.join(_common.RSRC, 'image.mp3') album = self.add_album_fixture() trackpath = album.items()[0].path From d70a82449e14084af8175bb6b56d4a81c755d272 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Mon, 16 Feb 2015 10:34:35 +0100 Subject: [PATCH 022/129] Fields: show computed and flexible fields - Computed fields from the _getters - Flexible fields for which a type is defined (in _types or in a plugins' item_types/album_types) Fix #1030 --- beets/ui/commands.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 6612ca2cd..68965c0b1 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -96,11 +96,15 @@ def fields_func(lib, opts, args): _print_rows(plugin_fields) print("Item fields:") - _print_rows(library.Item._fields.keys()) + _print_rows(library.Item._fields.keys() + + library.Item._getters().keys() + + library.Item._types.keys()) _show_plugin_fields(False) print("\nAlbum fields:") - _print_rows(library.Album._fields.keys()) + _print_rows(library.Album._fields.keys() + + library.Album._getters().keys() + + library.Album._types.keys()) _show_plugin_fields(True) From 1d8160f9ff77df37fb5df1e2f01c63abbc93ea1a Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Feb 2015 10:50:41 +0100 Subject: [PATCH 023/129] Play plugin: use OO, don't pass log around --- beetsplug/play.py | 176 +++++++++++++++++++++++----------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/beetsplug/play.py b/beetsplug/play.py index 5de5dcf32..4b931a68f 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -17,8 +17,6 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) -from functools import partial - from beets.plugins import BeetsPlugin from beets.ui import Subcommand from beets import config @@ -30,91 +28,6 @@ import shlex from tempfile import NamedTemporaryFile -def play_music(lib, opts, args, log): - """Execute query, create temporary playlist and execute player - command passing that playlist. - """ - command_str = config['play']['command'].get() - use_folders = config['play']['use_folders'].get(bool) - relative_to = config['play']['relative_to'].get() - if relative_to: - relative_to = util.normpath(relative_to) - if command_str: - command = shlex.split(command_str) - else: - # If a command isn't set, then let the OS decide how to open the - # playlist. - sys_name = platform.system() - if sys_name == 'Darwin': - command = ['open'] - elif sys_name == 'Windows': - command = ['start'] - else: - # If not Mac or Windows, then assume Unixy. - command = ['xdg-open'] - - # Preform search by album and add folders rather then tracks to playlist. - if opts.album: - selection = lib.albums(ui.decargs(args)) - paths = [] - - for album in selection: - if use_folders: - paths.append(album.item_dir()) - else: - # TODO use core's sorting functionality - paths.extend([item.path for item in sorted( - album.items(), key=lambda item: (item.disc, item.track))]) - item_type = 'album' - - # Preform item query and add tracks to playlist. - else: - selection = lib.items(ui.decargs(args)) - paths = [item.path for item in selection] - item_type = 'track' - - item_type += 's' if len(selection) > 1 else '' - - if not selection: - ui.print_(ui.colorize('text_warning', - 'No {0} to play.'.format(item_type))) - return - - # Warn user before playing any huge playlists. - if len(selection) > 100: - ui.print_(ui.colorize( - 'text_warning', - 'You are about to queue {0} {1}.'.format(len(selection), item_type) - )) - - if ui.input_options(('Continue', 'Abort')) == 'a': - return - - # Create temporary m3u file to hold our playlist. - m3u = NamedTemporaryFile('w', suffix='.m3u', delete=False) - for item in paths: - if relative_to: - m3u.write(relpath(item, relative_to) + b'\n') - else: - m3u.write(item + b'\n') - m3u.close() - - command.append(m3u.name) - - # Invoke the command and log the output. - output = util.command_output(command) - if output: - log.debug(u'Output of {0}: {1}', - util.displayable_path(command[0]), - output.decode('utf8', 'ignore')) - else: - log.debug(u'no output') - - ui.print_(u'Playing {0} {1}.'.format(len(selection), item_type)) - - util.remove(m3u.name) - - class PlayPlugin(BeetsPlugin): def __init__(self): @@ -136,5 +49,92 @@ class PlayPlugin(BeetsPlugin): action='store_true', default=False, help='query and load albums rather than tracks' ) - play_command.func = partial(play_music, log=self._log) + play_command.func = self.play_music return [play_command] + + def play_music(self, lib, opts, args): + """Execute query, create temporary playlist and execute player + command passing that playlist. + """ + command_str = config['play']['command'].get() + use_folders = config['play']['use_folders'].get(bool) + relative_to = config['play']['relative_to'].get() + if relative_to: + relative_to = util.normpath(relative_to) + if command_str: + command = shlex.split(command_str) + else: + # If a command isn't set, then let the OS decide how to open the + # playlist. + sys_name = platform.system() + if sys_name == 'Darwin': + command = ['open'] + elif sys_name == 'Windows': + command = ['start'] + else: + # If not Mac or Windows, then assume Unixy. + command = ['xdg-open'] + + # Preform search by album and add folders rather than tracks to + # playlist. + if opts.album: + selection = lib.albums(ui.decargs(args)) + paths = [] + + for album in selection: + if use_folders: + paths.append(album.item_dir()) + else: + # TODO use core's sorting functionality + paths.extend([item.path for item in sorted( + album.items(), + key=lambda item: (item.disc, item.track))]) + item_type = 'album' + + # Preform item query and add tracks to playlist. + else: + selection = lib.items(ui.decargs(args)) + paths = [item.path for item in selection] + item_type = 'track' + + item_type += 's' if len(selection) > 1 else '' + + if not selection: + ui.print_(ui.colorize('text_warning', + 'No {0} to play.'.format(item_type))) + return + + # Warn user before playing any huge playlists. + if len(selection) > 100: + ui.print_(ui.colorize( + 'text_warning', + 'You are about to queue {0} {1}.'.format(len(selection), + item_type) + )) + + if ui.input_options(('Continue', 'Abort')) == 'a': + return + + # Create temporary m3u file to hold our playlist. + m3u = NamedTemporaryFile('w', suffix='.m3u', delete=False) + for item in paths: + if relative_to: + m3u.write(relpath(item, relative_to) + b'\n') + else: + m3u.write(item + b'\n') + m3u.close() + + command.append(m3u.name) + + # Invoke the command and log the output. + output = util.command_output(command) + if output: + self._log.debug(u'Output of {0}: {1}', + util.displayable_path(command[0]), + output.decode('utf8', 'ignore')) + else: + self._log.debug(u'no output') + + ui.print_(u'Playing {0} {1}.'.format(len(selection), item_type)) + + util.remove(m3u.name) From 5d9128aff38ad4782ca4b2c31fe274acbc4e65f1 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Feb 2015 11:21:38 +0100 Subject: [PATCH 024/129] util.interactive_open() return open/xdg-open/start Depending on the platform return the correct automatic execution command. --- beets/ui/commands.py | 11 ++--------- beets/util/__init__.py | 14 ++++++++++++++ beetsplug/play.py | 12 +----------- test/test_config_command.py | 20 ++++---------------- test/test_util.py | 23 +++++++++++++++++++++++ 5 files changed, 44 insertions(+), 36 deletions(-) create mode 100644 test/test_util.py diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 6612ca2cd..98658a57d 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -20,7 +20,6 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) import os -import platform import re import shlex @@ -1503,15 +1502,9 @@ def config_edit(): editor = [editor] args = editor + [path] args.insert(1, args[0]) - elif platform.system() == 'Darwin': - args = ['open', 'open', '-n', path] - elif platform.system() == 'Windows': - # On windows we can execute arbitrary files. The os will - # take care of starting an appropriate application - args = [path, path] else: - # Assume Unix - args = ['xdg-open', 'xdg-open', path] + base = util.open_anything() + args = [base, base, path] try: os.execlp(*args) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 01a0257b0..b072d11a5 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -683,3 +683,17 @@ def max_filename_length(path, limit=MAX_FILENAME_LENGTH): return min(res[9], limit) else: return limit + + +def open_anything(): + """Return the system command that dispatches execution to the correct + program. + """ + sys_name = platform.system() + if sys_name == 'Darwin': + base_cmd = 'open' + elif sys_name == 'Windows': + base_cmd = 'start' + else: # Assume Unix + base_cmd = 'xdg-open' + return base_cmd diff --git a/beetsplug/play.py b/beetsplug/play.py index 4b931a68f..dabce2b43 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -23,7 +23,6 @@ from beets import config from beets import ui from beets import util from os.path import relpath -import platform import shlex from tempfile import NamedTemporaryFile @@ -64,16 +63,7 @@ class PlayPlugin(BeetsPlugin): if command_str: command = shlex.split(command_str) else: - # If a command isn't set, then let the OS decide how to open the - # playlist. - sys_name = platform.system() - if sys_name == 'Darwin': - command = ['open'] - elif sys_name == 'Windows': - command = ['start'] - else: - # If not Mac or Windows, then assume Unixy. - command = ['xdg-open'] + command = [util.open_anything()] # Preform search by album and add folders rather than tracks to # playlist. diff --git a/test/test_config_command.py b/test/test_config_command.py index d48afe2d9..82417f14f 100644 --- a/test/test_config_command.py +++ b/test/test_config_command.py @@ -81,25 +81,13 @@ class ConfigCommandTest(unittest.TestCase, TestHelper): execlp.assert_called_once_with( 'myeditor', 'myeditor', self.config_path) - def test_edit_config_with_open(self): - with _common.system_mock('Darwin'): + def test_edit_config_with_automatic_open(self): + with patch('beets.util.open_anything') as open: + open.return_value = 'please_open' with patch('os.execlp') as execlp: self.run_command('config', '-e') execlp.assert_called_once_with( - 'open', 'open', '-n', self.config_path) - - def test_edit_config_with_xdg_open(self): - with _common.system_mock('Linux'): - with patch('os.execlp') as execlp: - self.run_command('config', '-e') - execlp.assert_called_once_with( - 'xdg-open', 'xdg-open', self.config_path) - - def test_edit_config_with_windows_exec(self): - with _common.system_mock('Windows'): - with patch('os.execlp') as execlp: - self.run_command('config', '-e') - execlp.assert_called_once_with(self.config_path, self.config_path) + 'please_open', 'please_open', self.config_path) def test_config_editor_not_found(self): with self.assertRaises(ui.UserError) as user_error: diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 000000000..c7cc9790a --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,23 @@ +from test._common import unittest +from test import _common + +from beets import util + + +class UtilTest(unittest.TestCase): + def test_open_anything(self): + with _common.system_mock('Windows'): + self.assertEqual(util.open_anything(), 'start') + + with _common.system_mock('Darwin'): + self.assertEqual(util.open_anything(), 'open') + + with _common.system_mock('Tagada'): + self.assertEqual(util.open_anything(), 'xdg-open') + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == b'__main__': + unittest.main(defaultTest='suite') From 489b6b2e7e4bd8b121ee512abb32e53833a93280 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Feb 2015 11:42:24 +0100 Subject: [PATCH 025/129] Move many tests from test_library to test_util Sice they only test functions from beets/util/__init__.py there's not reason for them to reside in test/test_library.py. --- test/test_library.py | 113 ----------------------------------- test/test_util.py | 136 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 114 deletions(-) diff --git a/test/test_library.py b/test/test_library.py index 6bb88076e..1a2812b61 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -240,27 +240,6 @@ class DestinationTest(_common.TestCase): self.assertFalse('two \\ three' in p) self.assertFalse('two / three' in p) - def test_sanitize_unix_replaces_leading_dot(self): - with _common.platform_posix(): - p = util.sanitize_path(u'one/.two/three') - self.assertFalse('.' in p) - - def test_sanitize_windows_replaces_trailing_dot(self): - with _common.platform_windows(): - p = util.sanitize_path(u'one/two./three') - self.assertFalse('.' in p) - - def test_sanitize_windows_replaces_illegal_chars(self): - with _common.platform_windows(): - p = util.sanitize_path(u':*?"<>|') - self.assertFalse(':' in p) - self.assertFalse('*' in p) - self.assertFalse('?' in p) - self.assertFalse('"' in p) - self.assertFalse('<' in p) - self.assertFalse('>' in p) - self.assertFalse('|' in p) - def test_path_with_format(self): self.lib.path_formats = [('default', '$artist/$album ($format)')] p = self.i.destination() @@ -337,11 +316,6 @@ class DestinationTest(_common.TestCase): ] self.assertEqual(self.i.destination(), np('one/three')) - def test_sanitize_windows_replaces_trailing_space(self): - with _common.platform_windows(): - p = util.sanitize_path(u'one/two /three') - self.assertFalse(' ' in p) - def test_get_formatted_does_not_replace_separators(self): with _common.platform_posix(): name = os.path.join('a', 'b') @@ -407,25 +381,6 @@ class DestinationTest(_common.TestCase): p = self.i.destination() self.assertEqual(p.rsplit(os.path.sep, 1)[1], 'something') - def test_sanitize_path_works_on_empty_string(self): - with _common.platform_posix(): - p = util.sanitize_path(u'') - self.assertEqual(p, u'') - - def test_sanitize_with_custom_replace_overrides_built_in_sub(self): - with _common.platform_posix(): - p = util.sanitize_path(u'a/.?/b', [ - (re.compile(r'foo'), u'bar'), - ]) - self.assertEqual(p, u'a/.?/b') - - def test_sanitize_with_custom_replace_adds_replacements(self): - with _common.platform_posix(): - p = util.sanitize_path(u'foo/bar', [ - (re.compile(r'foo'), u'bar'), - ]) - self.assertEqual(p, u'bar/bar') - def test_unicode_normalized_nfd_on_mac(self): instr = unicodedata.normalize('NFC', u'caf\xe9') self.lib.path_formats = [('default', instr)] @@ -474,14 +429,6 @@ class DestinationTest(_common.TestCase): self.assertEqual(self.i.destination(), np('base/ber/foo')) - @unittest.skip('unimplemented: #359') - def test_sanitize_empty_component(self): - with _common.platform_posix(): - p = util.sanitize_path(u'foo//bar', [ - (re.compile(r'^$'), u'_'), - ]) - self.assertEqual(p, u'foo/_/bar') - @unittest.skip('unimplemented: #359') def test_destination_with_empty_component(self): self.lib.directory = 'base' @@ -700,49 +647,6 @@ class DisambiguationTest(_common.TestCase, PathFormattingMixin): self._assert_dest('/base/foo [foo_bar]/the title', self.i1) -class PathConversionTest(_common.TestCase): - def test_syspath_windows_format(self): - with _common.platform_windows(): - path = os.path.join('a', 'b', 'c') - outpath = util.syspath(path) - self.assertTrue(isinstance(outpath, unicode)) - self.assertTrue(outpath.startswith(u'\\\\?\\')) - - def test_syspath_windows_format_unc_path(self): - # The \\?\ prefix on Windows behaves differently with UNC - # (network share) paths. - path = '\\\\server\\share\\file.mp3' - with _common.platform_windows(): - outpath = util.syspath(path) - self.assertTrue(isinstance(outpath, unicode)) - self.assertEqual(outpath, u'\\\\?\\UNC\\server\\share\\file.mp3') - - def test_syspath_posix_unchanged(self): - with _common.platform_posix(): - path = os.path.join('a', 'b', 'c') - outpath = util.syspath(path) - self.assertEqual(path, outpath) - - def _windows_bytestring_path(self, path): - old_gfse = sys.getfilesystemencoding - sys.getfilesystemencoding = lambda: 'mbcs' - try: - with _common.platform_windows(): - return util.bytestring_path(path) - finally: - sys.getfilesystemencoding = old_gfse - - def test_bytestring_path_windows_encodes_utf8(self): - path = u'caf\xe9' - outpath = self._windows_bytestring_path(path) - self.assertEqual(path, outpath.decode('utf8')) - - def test_bytesting_path_windows_removes_magic_prefix(self): - path = u'\\\\?\\C:\\caf\xe9' - outpath = self._windows_bytestring_path(path) - self.assertEqual(outpath, u'C:\\caf\xe9'.encode('utf8')) - - class PluginDestinationTest(_common.TestCase): def setUp(self): super(PluginDestinationTest, self).setUp() @@ -1004,23 +908,6 @@ class PathStringTest(_common.TestCase): self.assert_(isinstance(alb.artpath, bytes)) -class PathTruncationTest(_common.TestCase): - def test_truncate_bytestring(self): - with _common.platform_posix(): - p = util.truncate_path(b'abcde/fgh', 4) - self.assertEqual(p, b'abcd/fgh') - - def test_truncate_unicode(self): - with _common.platform_posix(): - p = util.truncate_path(u'abcde/fgh', 4) - self.assertEqual(p, u'abcd/fgh') - - def test_truncate_preserves_extension(self): - with _common.platform_posix(): - p = util.truncate_path(u'abcde/fgh.ext', 5) - self.assertEqual(p, u'abcde/f.ext') - - class MtimeTest(_common.TestCase): def setUp(self): super(MtimeTest, self).setUp() diff --git a/test/test_util.py b/test/test_util.py index c7cc9790a..c38dcdf83 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,6 +1,27 @@ +# This file is part of beets. +# Copyright 2015, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +"""Tests for base utils from the beets.util package. +""" +from __future__ import (division, absolute_import, print_function, + unicode_literals) + +import sys +import re +import os + from test._common import unittest from test import _common - from beets import util @@ -15,6 +36,119 @@ class UtilTest(unittest.TestCase): with _common.system_mock('Tagada'): self.assertEqual(util.open_anything(), 'xdg-open') + def test_sanitize_unix_replaces_leading_dot(self): + with _common.platform_posix(): + p = util.sanitize_path(u'one/.two/three') + self.assertFalse('.' in p) + + def test_sanitize_windows_replaces_trailing_dot(self): + with _common.platform_windows(): + p = util.sanitize_path(u'one/two./three') + self.assertFalse('.' in p) + + def test_sanitize_windows_replaces_illegal_chars(self): + with _common.platform_windows(): + p = util.sanitize_path(u':*?"<>|') + self.assertFalse(':' in p) + self.assertFalse('*' in p) + self.assertFalse('?' in p) + self.assertFalse('"' in p) + self.assertFalse('<' in p) + self.assertFalse('>' in p) + self.assertFalse('|' in p) + + def test_sanitize_windows_replaces_trailing_space(self): + with _common.platform_windows(): + p = util.sanitize_path(u'one/two /three') + self.assertFalse(' ' in p) + + def test_sanitize_path_works_on_empty_string(self): + with _common.platform_posix(): + p = util.sanitize_path(u'') + self.assertEqual(p, u'') + + def test_sanitize_with_custom_replace_overrides_built_in_sub(self): + with _common.platform_posix(): + p = util.sanitize_path(u'a/.?/b', [ + (re.compile(r'foo'), u'bar'), + ]) + self.assertEqual(p, u'a/.?/b') + + def test_sanitize_with_custom_replace_adds_replacements(self): + with _common.platform_posix(): + p = util.sanitize_path(u'foo/bar', [ + (re.compile(r'foo'), u'bar'), + ]) + self.assertEqual(p, u'bar/bar') + + @unittest.skip('unimplemented: #359') + def test_sanitize_empty_component(self): + with _common.platform_posix(): + p = util.sanitize_path(u'foo//bar', [ + (re.compile(r'^$'), u'_'), + ]) + self.assertEqual(p, u'foo/_/bar') + + +class PathConversionTest(_common.TestCase): + def test_syspath_windows_format(self): + with _common.platform_windows(): + path = os.path.join('a', 'b', 'c') + outpath = util.syspath(path) + self.assertTrue(isinstance(outpath, unicode)) + self.assertTrue(outpath.startswith(u'\\\\?\\')) + + def test_syspath_windows_format_unc_path(self): + # The \\?\ prefix on Windows behaves differently with UNC + # (network share) paths. + path = '\\\\server\\share\\file.mp3' + with _common.platform_windows(): + outpath = util.syspath(path) + self.assertTrue(isinstance(outpath, unicode)) + self.assertEqual(outpath, u'\\\\?\\UNC\\server\\share\\file.mp3') + + def test_syspath_posix_unchanged(self): + with _common.platform_posix(): + path = os.path.join('a', 'b', 'c') + outpath = util.syspath(path) + self.assertEqual(path, outpath) + + def _windows_bytestring_path(self, path): + old_gfse = sys.getfilesystemencoding + sys.getfilesystemencoding = lambda: 'mbcs' + try: + with _common.platform_windows(): + return util.bytestring_path(path) + finally: + sys.getfilesystemencoding = old_gfse + + def test_bytestring_path_windows_encodes_utf8(self): + path = u'caf\xe9' + outpath = self._windows_bytestring_path(path) + self.assertEqual(path, outpath.decode('utf8')) + + def test_bytesting_path_windows_removes_magic_prefix(self): + path = u'\\\\?\\C:\\caf\xe9' + outpath = self._windows_bytestring_path(path) + self.assertEqual(outpath, u'C:\\caf\xe9'.encode('utf8')) + + +class PathTruncationTest(_common.TestCase): + def test_truncate_bytestring(self): + with _common.platform_posix(): + p = util.truncate_path(b'abcde/fgh', 4) + self.assertEqual(p, b'abcd/fgh') + + def test_truncate_unicode(self): + with _common.platform_posix(): + p = util.truncate_path(u'abcde/fgh', 4) + self.assertEqual(p, u'abcd/fgh') + + def test_truncate_preserves_extension(self): + with _common.platform_posix(): + p = util.truncate_path(u'abcde/fgh.ext', 5) + self.assertEqual(p, u'abcde/f.ext') + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From c47221555f2f2dae727b236805dc344f3dcd71fe Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Feb 2015 11:55:26 +0100 Subject: [PATCH 026/129] Add beets.util.interactive_open() find cmd + execute interactive_open() takes a target and an optional command, if it does not receive a command then it uses open_anything(). It parses command and lexes it with shlex.split(), revieling the client from that task. "config -e" command uses it, and gives a better error message in case of problem. "play" plugin uses it as well, as side-effect being that the command is now interactive, as requested in issue #1321. Fix issue #1321. --- beets/ui/commands.py | 25 +++++++------------------ beets/util/__init__.py | 24 ++++++++++++++++++++++++ beetsplug/play.py | 24 +++++++----------------- docs/changelog.rst | 2 ++ docs/plugins/play.rst | 4 ++-- test/test_config_command.py | 5 +++-- test/test_util.py | 13 +++++++++++++ 7 files changed, 58 insertions(+), 39 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 98658a57d..d6fb726ec 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -21,7 +21,6 @@ from __future__ import (division, absolute_import, print_function, import os import re -import shlex import beets from beets import ui @@ -1494,24 +1493,14 @@ def config_edit(): """ path = config.user_config_path() - if 'EDITOR' in os.environ: - editor = os.environ['EDITOR'].encode('utf8') - try: - editor = [e.decode('utf8') for e in shlex.split(editor)] - except ValueError: # Malformed shell tokens. - editor = [editor] - args = editor + [path] - args.insert(1, args[0]) - else: - base = util.open_anything() - args = [base, base, path] - + editor = os.environ.get('EDITOR') try: - os.execlp(*args) - except OSError: - raise ui.UserError("Could not edit configuration. Please " - "set the EDITOR environment variable.") - + util.interactive_open(path, editor) + except OSError as exc: + message = "Could not edit configuration: {0}".format(exc) + if not editor: + message += ". Please set the EDITOR environment variable" + raise ui.UserError(message) config_cmd = ui.Subcommand('config', help='show or edit the user configuration') diff --git a/beets/util/__init__.py b/beets/util/__init__.py index b072d11a5..2d6968e13 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -26,6 +26,7 @@ from collections import defaultdict import traceback import subprocess import platform +import shlex MAX_FILENAME_LENGTH = 200 @@ -697,3 +698,26 @@ def open_anything(): else: # Assume Unix base_cmd = 'xdg-open' return base_cmd + + +def interactive_open(target, command=None): + """Open `target` file with `command` or, in not available, ask the OS to + deal with it. + + The executed program will have stdin, stdout and stderr. + OSError may be raised, it is left to the caller to catch them. + """ + if command: + command = command.encode('utf8') + try: + command = [c.decode('utf8') + for c in shlex.split(command)] + except ValueError: # Malformed shell tokens. + command = [command] + command.insert(0, command[0]) # for argv[0] + else: + base_cmd = open_anything() + command = [base_cmd, base_cmd] + + command.append(target) + return os.execlp(*command) diff --git a/beetsplug/play.py b/beetsplug/play.py index dabce2b43..db3348210 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -23,7 +23,6 @@ from beets import config from beets import ui from beets import util from os.path import relpath -import shlex from tempfile import NamedTemporaryFile @@ -60,10 +59,6 @@ class PlayPlugin(BeetsPlugin): relative_to = config['play']['relative_to'].get() if relative_to: relative_to = util.normpath(relative_to) - if command_str: - command = shlex.split(command_str) - else: - command = [util.open_anything()] # Preform search by album and add folders rather than tracks to # playlist. @@ -114,17 +109,12 @@ class PlayPlugin(BeetsPlugin): m3u.write(item + b'\n') m3u.close() - command.append(m3u.name) - - # Invoke the command and log the output. - output = util.command_output(command) - if output: - self._log.debug(u'Output of {0}: {1}', - util.displayable_path(command[0]), - output.decode('utf8', 'ignore')) - else: - self._log.debug(u'no output') - ui.print_(u'Playing {0} {1}.'.format(len(selection), item_type)) - util.remove(m3u.name) + try: + util.interactive_open(m3u.name, command_str) + except OSError as exc: + raise ui.UserError("Could not play the music playlist: " + "{0}".format(exc)) + finally: + util.remove(m3u.name) diff --git a/docs/changelog.rst b/docs/changelog.rst index 329770102..d68b92812 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog Features: +* :doc:`/plugins/play` gives full interaction with the command invoked. + :bug:`1321` * The summary shown to compare duplicate albums during import now displays the old and new filesizes. :bug:`1291` * The colors used are now configurable via the new config option ``colors``, diff --git a/docs/plugins/play.rst b/docs/plugins/play.rst index 325656aaa..939e3bec4 100644 --- a/docs/plugins/play.rst +++ b/docs/plugins/play.rst @@ -26,8 +26,8 @@ would on the command-line):: play: command: /usr/bin/command --option1 --option2 some_other_option -Enable beets' verbose logging to see the command's output if you need to -debug. +While playing you'll be able to interact with the player if it is a +command-line oriented, and you'll get its output in real time. Configuration ------------- diff --git a/test/test_config_command.py b/test/test_config_command.py index 82417f14f..7289daf13 100644 --- a/test/test_config_command.py +++ b/test/test_config_command.py @@ -92,10 +92,11 @@ class ConfigCommandTest(unittest.TestCase, TestHelper): def test_config_editor_not_found(self): with self.assertRaises(ui.UserError) as user_error: with patch('os.execlp') as execlp: - execlp.side_effect = OSError() + execlp.side_effect = OSError('here is problem') self.run_command('config', '-e') self.assertIn('Could not edit configuration', - unicode(user_error.exception.args[0])) + unicode(user_error.exception)) + self.assertIn('here is problem', unicode(user_error.exception)) def test_edit_invalid_config_file(self): self.lib = Library(':memory:') diff --git a/test/test_util.py b/test/test_util.py index c38dcdf83..20c7708d5 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -20,6 +20,8 @@ import sys import re import os +from mock import patch + from test._common import unittest from test import _common from beets import util @@ -36,6 +38,17 @@ class UtilTest(unittest.TestCase): with _common.system_mock('Tagada'): self.assertEqual(util.open_anything(), 'xdg-open') + @patch('os.execlp') + @patch('beets.util.open_anything') + def test_interactive_open(self, mock_open, mock_execlp): + mock_open.return_value = 'tagada' + util.interactive_open('foo') + mock_execlp.assert_called_once_with('tagada', 'tagada', 'foo') + mock_execlp.reset_mock() + + util.interactive_open('foo', 'bar') + mock_execlp.assert_called_once_with('bar', 'bar', 'foo') + def test_sanitize_unix_replaces_leading_dot(self): with _common.platform_posix(): p = util.sanitize_path(u'one/.two/three') From 39a6145d2d47a9525b15e5b89f5a16bed82eeef7 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Feb 2015 12:19:05 +0100 Subject: [PATCH 027/129] Plugin play uses default item sort in album mode Offer library.get_default_{item,album}_sort for that purpose. --- beets/library.py | 22 ++++++++++++++-------- beetsplug/play.py | 11 +++++------ docs/changelog.rst | 2 ++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/beets/library.py b/beets/library.py index 8d95561f7..c05206dea 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1188,21 +1188,27 @@ class Library(dbcore.Database): model_cls, query, sort ) + def get_default_album_sort(self): + """Get a :class:`Sort` object for albums from the config option. + """ + return dbcore.sort_from_strings( + Album, beets.config['sort_album'].as_str_seq()) + + def get_default_item_sort(self): + """Get a :class:`Sort` object for items from the config option. + """ + return dbcore.sort_from_strings( + Item, beets.config['sort_item'].as_str_seq()) + def albums(self, query=None, sort=None): """Get :class:`Album` objects matching the query. """ - sort = sort or dbcore.sort_from_strings( - Album, beets.config['sort_album'].as_str_seq() - ) - return self._fetch(Album, query, sort) + return self._fetch(Album, query, sort or self.get_default_album_sort()) def items(self, query=None, sort=None): """Get :class:`Item` objects matching the query. """ - sort = sort or dbcore.sort_from_strings( - Item, beets.config['sort_item'].as_str_seq() - ) - return self._fetch(Item, query, sort) + return self._fetch(Item, query, sort or self.get_default_item_sort()) # Convenience accessors. diff --git a/beetsplug/play.py b/beetsplug/play.py index db3348210..8a912505c 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -60,23 +60,22 @@ class PlayPlugin(BeetsPlugin): if relative_to: relative_to = util.normpath(relative_to) - # Preform search by album and add folders rather than tracks to + # Perform search by album and add folders rather than tracks to # playlist. if opts.album: selection = lib.albums(ui.decargs(args)) paths = [] + sort = lib.get_default_album_sort() for album in selection: if use_folders: paths.append(album.item_dir()) else: - # TODO use core's sorting functionality - paths.extend([item.path for item in sorted( - album.items(), - key=lambda item: (item.disc, item.track))]) + paths.extend(item.path + for item in sort.sort(album.items())) item_type = 'album' - # Preform item query and add tracks to playlist. + # Perform item query and add tracks to playlist. else: selection = lib.items(ui.decargs(args)) paths = [item.path for item in selection] diff --git a/docs/changelog.rst b/docs/changelog.rst index d68b92812..d65d1e416 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog Features: +* :doc:`/plugins/play` will sort items according to the configured option when + used in album mode. * :doc:`/plugins/play` gives full interaction with the command invoked. :bug:`1321` * The summary shown to compare duplicate albums during import now displays From 12ecb2ce35e144d110ab381d61b8378843f5f2b0 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Mon, 16 Feb 2015 18:21:56 +0100 Subject: [PATCH 028/129] Fix formatting to new PEP8 version (1.6.2) PEP8 1.6.2 (2015-02-15): - added check for breaking around a binary operator This caused Travis to fail on "W503 line break before binary operator" --- beets/util/confit.py | 8 ++++---- beets/util/functemplate.py | 4 ++-- beetsplug/mpdstats.py | 4 ++-- setup.py | 5 ++--- test/test_ui.py | 4 ++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/beets/util/confit.py b/beets/util/confit.py index c30e52c5d..110d00001 100644 --- a/beets/util/confit.py +++ b/beets/util/confit.py @@ -602,11 +602,11 @@ class Dumper(yaml.SafeDumper): for item_key, item_value in mapping: node_key = self.represent_data(item_key) node_value = self.represent_data(item_value) - if not (isinstance(node_key, yaml.ScalarNode) - and not node_key.style): + if not (isinstance(node_key, yaml.ScalarNode) and + not node_key.style): best_style = False - if not (isinstance(node_value, yaml.ScalarNode) - and not node_value.style): + if not (isinstance(node_value, yaml.ScalarNode) and + not node_value.style): best_style = False value.append((node_key, node_value)) if flow_style is None: diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index f1066761d..3d5b0e871 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -309,8 +309,8 @@ class Parser(object): # A non-special character. Skip to the next special # character, treating the interstice as literal text. next_pos = ( - self.special_char_re.search(self.string[self.pos:]).start() - + self.pos + self.special_char_re.search( + self.string[self.pos:]).start() + self.pos ) text_parts.append(self.string[self.pos:next_pos]) self.pos = next_pos diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index c021d7465..e40c3a7d4 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -166,8 +166,8 @@ class MPDStats(object): else: rolling = (rating + (1.0 - rating) / 2.0) stable = (play_count + 1.0) / (play_count + skip_count + 2.0) - return (self.rating_mix * stable - + (1.0 - self.rating_mix) * rolling) + return (self.rating_mix * stable + + (1.0 - self.rating_mix) * rolling) def get_item(self, path): """Return the beets item related to path. diff --git a/setup.py b/setup.py index 76669c62c..a87b155bc 100755 --- a/setup.py +++ b/setup.py @@ -82,9 +82,8 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', - ] - + (['colorama'] if (sys.platform == 'win32') else []) - + (['ordereddict'] if sys.version_info < (2, 7, 0) else []), + ] + (['colorama'] if (sys.platform == 'win32') else []) + + (['ordereddict'] if sys.version_info < (2, 7, 0) else []), tests_require=[ 'beautifulsoup4', diff --git a/test/test_ui.py b/test/test_ui.py index 07add2fcd..d2a2ea80e 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -968,8 +968,8 @@ class ShowChangeTest(_common.TestCase): self.items[0].title = u'' self.items[0].path = u'/path/to/caf\xe9.mp3'.encode('utf8') msg = re.sub(r' +', ' ', self._show_change()) - self.assertTrue(u'caf\xe9.mp3 -> the title' in msg - or u'caf.mp3 ->' in msg) + self.assertTrue(u'caf\xe9.mp3 -> the title' in msg or + u'caf.mp3 ->' in msg) class PathFormatTest(_common.TestCase): From 21aedeb51acede828d88fe3fe88bf9c4d1aee295 Mon Sep 17 00:00:00 2001 From: Marvin Steadfast Date: Wed, 11 Feb 2015 12:58:57 +0100 Subject: [PATCH 029/129] Updated permissions plugin to change directory permissions too. --- beetsplug/permissions.py | 76 +++++++++++++++++++++++++++++++++++----- test/test_permissions.py | 41 +++++++++++++++++----- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/beetsplug/permissions.py b/beetsplug/permissions.py index 256f09e52..afff3ce38 100644 --- a/beetsplug/permissions.py +++ b/beetsplug/permissions.py @@ -6,10 +6,13 @@ like the following in your config.yaml to configure: permissions: file: 644 + dir: 755 """ import os +from collections import OrderedDict from beets import config, util from beets.plugins import BeetsPlugin +from beets.util import displayable_path def convert_perm(perm): @@ -28,13 +31,43 @@ def check_permissions(path, permission): return oct(os.stat(path).st_mode & 0o777) == oct(permission) +def get_music_directories(music_directory, imported_item): + """Creates a list of directories the imported item is in. + """ + # Checks for the directory in config and if it has a tilde in it. + # If its that way it will be expanded to the full path. + if '~' in music_directory: + music_directory = os.path.expanduser(music_directory) + + # Getting the absolute path of the directory. + music_directory = os.path.abspath(music_directory) + + # Creates a differential path list of the directory config path and + # the path of the imported item. + differential_path_list = os.path.split( + displayable_path(imported_item).split( + music_directory)[1])[0].split('/')[1:] + + # Creating a list with full paths of all directories in the music library + # we need to look at for chaning permissions. + directory_list = [] + for path in differential_path_list: + if len(directory_list) > 0: + directory_list.append(os.path.join(directory_list[-1], path)) + else: + directory_list.append(os.path.join(music_directory, path)) + + return directory_list + + class Permissions(BeetsPlugin): def __init__(self): super(Permissions, self).__init__() # Adding defaults. self.config.add({ - u'file': 644 + u'file': 644, + u'dir': 755 }) @@ -45,21 +78,25 @@ def permissions(lib, item=None, album=None): """ # Getting the config. file_perm = config['permissions']['file'].get() + dir_perm = config['permissions']['dir'].get() - # Converts file permissions to oct. + # Converts permissions to oct. file_perm = convert_perm(file_perm) + dir_perm = convert_perm(dir_perm) # Create chmod_queue. - chmod_queue = [] + file_chmod_queue = [] if item: - chmod_queue.append(item.path) + file_chmod_queue.append(item.path) elif album: for album_item in album.items(): - chmod_queue.append(album_item.path) + file_chmod_queue.append(album_item.path) - # Setting permissions for every path in the queue. - for path in chmod_queue: - # Changing permissions on the destination path. + # A list of directories to set permissions for. + dir_chmod_queue = [] + + for path in file_chmod_queue: + # Changing permissions on the destination file. os.chmod(util.bytestring_path(path), file_perm) # Checks if the destination path has the permissions configured. @@ -67,3 +104,26 @@ def permissions(lib, item=None, album=None): message = 'There was a problem setting permission on {}'.format( path) print(message) + + # Adding directories to the chmod queue. + dir_chmod_queue.append( + get_music_directories(config['directory'].get(), path)) + + # Unpack sublists. + dir_chmod_queue = [directory + for dir_list in dir_chmod_queue + for directory in dir_list] + + # Get rid of the duplicates. + dir_chmod_queue = list(OrderedDict.fromkeys(dir_chmod_queue)) + + # Change permissions for the directories. + for path in dir_chmod_queue: + # Chaning permissions on the destination directory. + os.chmod(util.bytestring_path(path), dir_perm) + # Checks if the destination path has the permissions configured. + + if not check_permissions(util.bytestring_path(path), dir_perm): + message = 'There was a problem setting permission on {}'.format( + path) + print(message) diff --git a/test/test_permissions.py b/test/test_permissions.py index 1e16a1c34..9f24820d6 100644 --- a/test/test_permissions.py +++ b/test/test_permissions.py @@ -5,7 +5,9 @@ from __future__ import (division, absolute_import, print_function, from test._common import unittest from test.helper import TestHelper -from beetsplug.permissions import check_permissions, convert_perm +from beetsplug.permissions import (check_permissions, + convert_perm, + get_music_directories) class PermissionsPluginTest(unittest.TestCase, TestHelper): @@ -14,7 +16,8 @@ class PermissionsPluginTest(unittest.TestCase, TestHelper): self.load_plugins('permissions') self.config['permissions'] = { - 'file': 777} + 'file': 777, + 'dir': 777} def tearDown(self): self.teardown_beets() @@ -24,23 +27,45 @@ class PermissionsPluginTest(unittest.TestCase, TestHelper): self.importer = self.create_importer() self.importer.run() item = self.lib.items().get() - config_perm = self.config['permissions']['file'].get() - config_perm = convert_perm(config_perm) - self.assertTrue(check_permissions(item.path, config_perm)) + file_perm = self.config['permissions']['file'].get() + file_perm = convert_perm(file_perm) + + dir_perm = self.config['permissions']['dir'].get() + dir_perm = convert_perm(dir_perm) + + music_dirs = get_music_directories(self.config['directory'].get(), + item.path) + + self.assertTrue(check_permissions(item.path, file_perm)) self.assertFalse(check_permissions(item.path, convert_perm(644))) + for path in music_dirs: + self.assertTrue(check_permissions(path, dir_perm)) + self.assertFalse(check_permissions(path, convert_perm(644))) + def test_permissions_on_item_imported(self): self.config['import']['singletons'] = True self.importer = self.create_importer() self.importer.run() item = self.lib.items().get() - config_perm = self.config['permissions']['file'].get() - config_perm = convert_perm(config_perm) - self.assertTrue(check_permissions(item.path, config_perm)) + file_perm = self.config['permissions']['file'].get() + file_perm = convert_perm(file_perm) + + dir_perm = self.config['permissions']['dir'].get() + dir_perm = convert_perm(dir_perm) + + music_dirs = get_music_directories(self.config['directory'].get(), + item.path) + + self.assertTrue(check_permissions(item.path, file_perm)) self.assertFalse(check_permissions(item.path, convert_perm(644))) + for path in music_dirs: + self.assertTrue(check_permissions(path, dir_perm)) + self.assertFalse(check_permissions(path, convert_perm(644))) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 8b08ec568c33a76edf78e69a58dc1f450b3c8f95 Mon Sep 17 00:00:00 2001 From: Marvin Steadfast Date: Wed, 11 Feb 2015 13:27:18 +0100 Subject: [PATCH 030/129] permissions plugin now uses a set instead of a OrderedDict to find duplicates in a list --- beetsplug/permissions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beetsplug/permissions.py b/beetsplug/permissions.py index afff3ce38..7d50c34ab 100644 --- a/beetsplug/permissions.py +++ b/beetsplug/permissions.py @@ -9,7 +9,6 @@ like the following in your config.yaml to configure: dir: 755 """ import os -from collections import OrderedDict from beets import config, util from beets.plugins import BeetsPlugin from beets.util import displayable_path @@ -114,15 +113,15 @@ def permissions(lib, item=None, album=None): for dir_list in dir_chmod_queue for directory in dir_list] - # Get rid of the duplicates. - dir_chmod_queue = list(OrderedDict.fromkeys(dir_chmod_queue)) + # Get rid of duplicates. + dir_chmod_queue = list(set(dir_chmod_queue)) # Change permissions for the directories. for path in dir_chmod_queue: # Chaning permissions on the destination directory. os.chmod(util.bytestring_path(path), dir_perm) - # Checks if the destination path has the permissions configured. + # Checks if the destination path has the permissions configured. if not check_permissions(util.bytestring_path(path), dir_perm): message = 'There was a problem setting permission on {}'.format( path) From 27f4732d3d93fb7c9d2d1232df35e002535008ba Mon Sep 17 00:00:00 2001 From: Marvin Steadfast Date: Wed, 11 Feb 2015 13:48:53 +0100 Subject: [PATCH 031/129] Updated permissions plugin docs --- docs/plugins/permissions.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugins/permissions.rst b/docs/plugins/permissions.rst index 06b034b51..9c4cdc0aa 100644 --- a/docs/plugins/permissions.rst +++ b/docs/plugins/permissions.rst @@ -2,7 +2,7 @@ Permissions Plugin ================== The ``permissions`` plugin allows you to set file permissions for imported -music files. +music files and its directories. To use the ``permissions`` plugin, enable it in your configuration (see :ref:`using-plugins`). Permissions will be adjusted automatically on import. @@ -12,9 +12,10 @@ Configuration To configure the plugin, make an ``permissions:`` section in your configuration file. The ``file`` config value therein uses **octal modes** to specify the -desired permissions. The default flags are octal 644. +desired permissions. The default flags for files are octal 644 and 755 for directories. Here's an example:: permissions: file: 644 + dir: 755 From dd0de2f04b6e61253cac72ab0b8fd0e2c4da6d10 Mon Sep 17 00:00:00 2001 From: Marvin Steadfast Date: Mon, 16 Feb 2015 13:26:38 +0100 Subject: [PATCH 032/129] Made the permissions plugin simpler. Got rid of some non-needed code and use the ancestors function instead of writing something new. --- beetsplug/permissions.py | 52 ++++++++++------------------------------ test/test_permissions.py | 10 ++++---- 2 files changed, 17 insertions(+), 45 deletions(-) diff --git a/beetsplug/permissions.py b/beetsplug/permissions.py index 7d50c34ab..224819658 100644 --- a/beetsplug/permissions.py +++ b/beetsplug/permissions.py @@ -11,7 +11,7 @@ like the following in your config.yaml to configure: import os from beets import config, util from beets.plugins import BeetsPlugin -from beets.util import displayable_path +from beets.util import ancestry def convert_perm(perm): @@ -30,33 +30,12 @@ def check_permissions(path, permission): return oct(os.stat(path).st_mode & 0o777) == oct(permission) -def get_music_directories(music_directory, imported_item): - """Creates a list of directories the imported item is in. +def dirs_in_library(library, item): + """Creates a list of ancestor directories in the beets library path. """ - # Checks for the directory in config and if it has a tilde in it. - # If its that way it will be expanded to the full path. - if '~' in music_directory: - music_directory = os.path.expanduser(music_directory) - - # Getting the absolute path of the directory. - music_directory = os.path.abspath(music_directory) - - # Creates a differential path list of the directory config path and - # the path of the imported item. - differential_path_list = os.path.split( - displayable_path(imported_item).split( - music_directory)[1])[0].split('/')[1:] - - # Creating a list with full paths of all directories in the music library - # we need to look at for chaning permissions. - directory_list = [] - for path in differential_path_list: - if len(directory_list) > 0: - directory_list.append(os.path.join(directory_list[-1], path)) - else: - directory_list.append(os.path.join(music_directory, path)) - - return directory_list + return [ancestor + for ancestor in ancestry(item) + if library in ancestor][1:] class Permissions(BeetsPlugin): @@ -91,8 +70,8 @@ def permissions(lib, item=None, album=None): for album_item in album.items(): file_chmod_queue.append(album_item.path) - # A list of directories to set permissions for. - dir_chmod_queue = [] + # A set of directories to change permissions for. + dir_chmod_queue = set() for path in file_chmod_queue: # Changing permissions on the destination file. @@ -104,17 +83,10 @@ def permissions(lib, item=None, album=None): path) print(message) - # Adding directories to the chmod queue. - dir_chmod_queue.append( - get_music_directories(config['directory'].get(), path)) - - # Unpack sublists. - dir_chmod_queue = [directory - for dir_list in dir_chmod_queue - for directory in dir_list] - - # Get rid of duplicates. - dir_chmod_queue = list(set(dir_chmod_queue)) + # Adding directories to the directory chmod queue. + dir_chmod_queue.update( + dirs_in_library(config['directory'].get(), + path)) # Change permissions for the directories. for path in dir_chmod_queue: diff --git a/test/test_permissions.py b/test/test_permissions.py index 9f24820d6..20e33b7d2 100644 --- a/test/test_permissions.py +++ b/test/test_permissions.py @@ -7,7 +7,7 @@ from test._common import unittest from test.helper import TestHelper from beetsplug.permissions import (check_permissions, convert_perm, - get_music_directories) + dirs_in_library) class PermissionsPluginTest(unittest.TestCase, TestHelper): @@ -34,8 +34,8 @@ class PermissionsPluginTest(unittest.TestCase, TestHelper): dir_perm = self.config['permissions']['dir'].get() dir_perm = convert_perm(dir_perm) - music_dirs = get_music_directories(self.config['directory'].get(), - item.path) + music_dirs = dirs_in_library(self.config['directory'].get(), + item.path) self.assertTrue(check_permissions(item.path, file_perm)) self.assertFalse(check_permissions(item.path, convert_perm(644))) @@ -56,8 +56,8 @@ class PermissionsPluginTest(unittest.TestCase, TestHelper): dir_perm = self.config['permissions']['dir'].get() dir_perm = convert_perm(dir_perm) - music_dirs = get_music_directories(self.config['directory'].get(), - item.path) + music_dirs = dirs_in_library(self.config['directory'].get(), + item.path) self.assertTrue(check_permissions(item.path, file_perm)) self.assertFalse(check_permissions(item.path, convert_perm(644))) From b9174d176f96232b184998faf49a7b64b0696d5b Mon Sep 17 00:00:00 2001 From: Marvin Steadfast Date: Tue, 17 Feb 2015 11:43:56 +0100 Subject: [PATCH 033/129] The permissions plugin now uses startswith for finding ancestors in the library path. --- beetsplug/permissions.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/permissions.py b/beetsplug/permissions.py index 224819658..7039f491a 100644 --- a/beetsplug/permissions.py +++ b/beetsplug/permissions.py @@ -35,7 +35,7 @@ def dirs_in_library(library, item): """ return [ancestor for ancestor in ancestry(item) - if library in ancestor][1:] + if ancestor.startswith(library)][1:] class Permissions(BeetsPlugin): diff --git a/docs/changelog.rst b/docs/changelog.rst index 329770102..ca96f1d3f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,8 @@ Features: * The number of missing/unmatched tracks is shown during import. :bug:`1088` * The data source used during import (e.g., MusicBrainz) is now saved as a flexible attribute `data_source` of an Item/Album. :bug:`1311` +* :doc:`/plugins/permissions`: Now handles also the permissions of the + directories. :bug:`1308` bug:`1324` Core changes: From eae98aff0e6af64810e67302baeb808ab6fe2b48 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 17 Feb 2015 12:19:21 +0100 Subject: [PATCH 034/129] PathQuery is case-{,in}sensitive on {UNIX,Windows} PathQuery use LIKE on Windows and instr() = 1 on UNIX. Fix #1165. --- beets/library.py | 27 +++++++++++++++++++++------ test/test_query.py | 13 +++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 8d95561f7..35b763321 100644 --- a/beets/library.py +++ b/beets/library.py @@ -24,6 +24,7 @@ import unicodedata import time import re from unidecode import unidecode +import platform from beets import logging from beets.mediafile import MediaFile, MutagenError, UnreadableFileError @@ -42,30 +43,44 @@ log = logging.getLogger('beets') # Library-specific query types. class PathQuery(dbcore.FieldQuery): - """A query that matches all items under a given path.""" + """A query that matches all items under a given path. + + On Windows paths are case-insensitive, contratly to UNIX platforms. + """ escape_re = re.compile(r'[\\_%]') escape_char = b'\\' + _is_windows = platform.system() == 'Windows' + def __init__(self, field, pattern, fast=True): super(PathQuery, self).__init__(field, pattern, fast) + if self._is_windows: + pattern = pattern.lower() + # Match the path as a single file. self.file_path = util.bytestring_path(util.normpath(pattern)) # As a directory (prefix). self.dir_path = util.bytestring_path(os.path.join(self.file_path, b'')) def match(self, item): - return (item.path == self.file_path) or \ - item.path.startswith(self.dir_path) + path = item.path.lower() if self._is_windows else item.path + return (path == self.file_path) or path.startswith(self.dir_path) def col_clause(self): + file_blob = buffer(self.file_path) + + if not self._is_windows: + dir_blob = buffer(self.dir_path) + return '({0} = ?) || (instr({0}, ?) = 1)'.format(self.field), \ + (file_blob, dir_blob) + escape = lambda m: self.escape_char + m.group(0) dir_pattern = self.escape_re.sub(escape, self.dir_path) - dir_pattern = buffer(dir_pattern + b'%') - file_blob = buffer(self.file_path) + dir_blob = buffer(dir_pattern + b'%') return '({0} = ?) || ({0} LIKE ? ESCAPE ?)'.format(self.field), \ - (file_blob, dir_pattern, self.escape_char) + (file_blob, dir_blob, self.escape_char) # Library-specific field types. diff --git a/test/test_query.py b/test/test_query.py index a9b1058bd..c6aec6185 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -17,6 +17,8 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +from mock import patch + from test import _common from test._common import unittest from test import helper @@ -461,6 +463,17 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, ['album with backslash']) + def test_case_sensitivity(self): + self.add_album(path='/A/B/C2.mp3', title='caps path') + q = b'path:/A/B' + with patch('beets.library.PathQuery._is_windows', False): + results = self.lib.items(q) + self.assert_items_matched(results, ['caps path']) + + with patch('beets.library.PathQuery._is_windows', True): + results = self.lib.items(q) + self.assert_items_matched(results, ['path item', 'caps path']) + class IntQueryTest(unittest.TestCase, TestHelper): From 83e34322e911ab3e3a74751442edea36ab31ef7b Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 17 Feb 2015 13:13:30 +0100 Subject: [PATCH 035/129] Update changelog & docs --- docs/changelog.rst | 1 + docs/reference/query.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 329770102..6b9c57781 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -65,6 +65,7 @@ Core changes: Fixes: +* Path queries are case-sensitive on UNIX OSes. :bug:`1165` * :doc:`/plugins/lyrics`: Silence a warning about insecure requests in the new MusixMatch backend. :bug:`1204` * Fix a crash when ``beet`` is invoked without arguments. :bug:`1205` diff --git a/docs/reference/query.rst b/docs/reference/query.rst index 7dc79461a..af676a50d 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -184,6 +184,8 @@ Note that this only matches items that are *already in your library*, so a path query won't necessarily find *all* the audio files in a directory---just the ones you've already added to your beets library. +Such queries are case-sensitive on UNIX and case-insensitive on Microsoft +Windows. .. _query-sort: From c05dea123e77efe84bfd3f3e4039abbe1956b95b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 17 Feb 2015 17:12:42 -0500 Subject: [PATCH 036/129] Docs clarity for verbosity levels (#1320) --- docs/changelog.rst | 6 +++--- docs/reference/cli.rst | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 20e7edb50..fc9799886 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,9 +6,9 @@ Changelog Features: -* Verbosity is now an integer in the configuration since multiple levels are - supported (like e.g. apt-get). On the CLI one can stack verbose flags (i.e. - `-vv`). :bug:`1244` +* There are now multiple levels of verbosity. On the command line, you can + make beets somewhat verbose with ``-v`` or very verbose with ``-vv``. + :bug:`1244` * The summary shown to compare duplicate albums during import now displays the old and new filesizes. :bug:`1291` * The colors used are now configurable via the new config option ``colors``, diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index f889be678..321b766d4 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -369,7 +369,8 @@ import ...``. * ``-l LIBPATH``: specify the library database file to use. * ``-d DIRECTORY``: specify the library root directory. * ``-v``: verbose mode; prints out a deluge of debugging information. Please use - this flag when reporting bugs. It can be stacked twice. + this flag when reporting bugs. You can use it twice, as in ``-vv``, to make + beets even more verbose. * ``-c FILE``: read a specified YAML :doc:`configuration file `. Beets also uses the ``BEETSDIR`` environment variable to look for From 6089fb7899db2031aa6cc0c681951ee1d5628f38 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 17 Feb 2015 17:23:02 -0500 Subject: [PATCH 037/129] Remove unused import --- test/test_config_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_config_command.py b/test/test_config_command.py index 7289daf13..c206c7ac8 100644 --- a/test/test_config_command.py +++ b/test/test_config_command.py @@ -10,7 +10,6 @@ from shutil import rmtree from beets import ui from beets import config -from test import _common from test._common import unittest from test.helper import TestHelper, capture_stdout from beets.library import Library From 7bcd3a383bedd79b7ca398ee411999a717a59738 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Feb 2015 12:29:26 +0100 Subject: [PATCH 038/129] Library.get_default_{item, album}_sort: static methods See #1329. --- beets/library.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index c05206dea..6e53352d7 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1188,13 +1188,15 @@ class Library(dbcore.Database): model_cls, query, sort ) - def get_default_album_sort(self): + @staticmethod + def get_default_album_sort(): """Get a :class:`Sort` object for albums from the config option. """ return dbcore.sort_from_strings( Album, beets.config['sort_album'].as_str_seq()) - def get_default_item_sort(self): + @staticmethod + def get_default_item_sort(): """Get a :class:`Sort` object for items from the config option. """ return dbcore.sort_from_strings( From 457afdc55d2c9103a96faa6b2b8da22d78360b91 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Feb 2015 13:30:06 +0100 Subject: [PATCH 039/129] Auto re-auth for discogs plugin upon error 401 This goes in the direction of #1299 and #1305. --- beetsplug/discogs.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index a8b02f349..ac0f516ff 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -32,6 +32,7 @@ import time import json import socket import httplib +import os # Silence spurious INFO log lines generated by urllib3. @@ -78,6 +79,12 @@ class DiscogsPlugin(BeetsPlugin): self.discogs_client = Client(USER_AGENT, c_key, c_secret, token, secret) + def reset_auth(self): + """Delete toke file & redo the auth steps. + """ + os.remove(self._tokenfile()) + self.setup() + def _tokenfile(self): """Get the path to the JSON file for storing the OAuth token. """ @@ -130,7 +137,11 @@ class DiscogsPlugin(BeetsPlugin): return self.get_albums(query) except DiscogsAPIError as e: self._log.debug(u'API Error: {0} (query: {1})', e, query) - return [] + if e.status_code == 401: + self.reset_auth() + return self.candidates(items, artist, album, va_likely) + else: + return [] except CONNECTION_ERRORS as e: self._log.debug(u'HTTP Connection Error: {0}', e) return [] @@ -156,8 +167,11 @@ class DiscogsPlugin(BeetsPlugin): try: getattr(result, 'title') except DiscogsAPIError as e: - if e.message != '404 Not Found': + if e.status_code != 404: self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) + if e.status_code == 401: + self.reset_auth() + return self.album_for_id(album_id) return None except CONNECTION_ERRORS as e: self._log.debug(u'HTTP Connection Error: {0}', e) From 6fc678e9470e1469ba93b5011867ee29d406df82 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Feb 2015 18:52:22 +0100 Subject: [PATCH 040/129] PathQuery: use substr() instead of instr() substr() is only available in SQLite 3.7.15+, which is not available yet on Debian stable, CentOS & co. Use substr() instead. --- beets/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index 35b763321..c5503dd0a 100644 --- a/beets/library.py +++ b/beets/library.py @@ -73,8 +73,8 @@ class PathQuery(dbcore.FieldQuery): if not self._is_windows: dir_blob = buffer(self.dir_path) - return '({0} = ?) || (instr({0}, ?) = 1)'.format(self.field), \ - (file_blob, dir_blob) + return '({0} = ?) || (substr({0}, 1, ?) = ?)'.format(self.field), \ + (file_blob, len(dir_blob), dir_blob) escape = lambda m: self.escape_char + m.group(0) dir_pattern = self.escape_re.sub(escape, self.dir_path) From 53cefa328e8f50db9ec83ec5af6a927561ad1fa6 Mon Sep 17 00:00:00 2001 From: mried Date: Wed, 18 Feb 2015 19:06:58 +0100 Subject: [PATCH 041/129] Changed the path format from unicode to bytes. --- beetsplug/embedart.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index f567e0d2c..19e9a145d 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -26,7 +26,7 @@ from beets.plugins import BeetsPlugin from beets import mediafile from beets import ui from beets.ui import decargs -from beets.util import syspath, normpath, displayable_path +from beets.util import syspath, normpath, displayable_path, bytestring_path from beets.util.artresizer import ArtResizer from beets import config @@ -101,14 +101,14 @@ class EmbedCoverArtPlugin(BeetsPlugin): self.extract_first(normpath(opts.outpath), lib.items(decargs(args))) else: - filename = opts.filename or config['art_filename'].get() + filename = bytestring_path(opts.filename or + config['art_filename'].get()) if os.path.dirname(filename) != '': self._log.error(u"Only specify a name rather than a path " u"for -n") return for album in lib.albums(decargs(args)): - albumpath = album.path.decode('utf-8') - artpath = normpath(os.path.join(albumpath, filename)) + artpath = normpath(os.path.join(album.path, filename)) artpath = self.extract_first(artpath, album.items()) if artpath and opts.associate: album.set_art(artpath) @@ -266,7 +266,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): self._log.warning(u'Unknown image type in {0}.', displayable_path(item.path)) return - outpath = outpath.decode('utf-8') + '.' + ext + outpath = outpath + b'.' + ext self._log.info(u'Extracting album art from: {0} to: {1}', item, displayable_path(outpath)) From 8d3822723d39040d86b73b64c0696c9f2abbb039 Mon Sep 17 00:00:00 2001 From: mried Date: Wed, 18 Feb 2015 19:10:17 +0100 Subject: [PATCH 042/129] Removed the obsolete changelog entry. --- docs/changelog.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ec60f5abf..81493428c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -101,8 +101,6 @@ Fixes: Unicode filenames. :bug:`1297` * :doc:`/plugins/discogs`: Handle and log more kinds of communication errors. :bug:`1299` :bug:`1305` -* :doc:`/plugins/embedart`: Fix a crash that occured when the album path - contains non ascii characters. For developers: From e00d7b7ddcecd3696d93e0d22b3b05c0f1733ee4 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Feb 2015 19:28:03 +0100 Subject: [PATCH 043/129] PathQuery: simple utf8 comparison Test usqge of SQL's substr() with a UTF8 example. The ideal would be to test with non-UTF8 code points, however it is impossible to perform such a query: queries can only be unicode or utf8. --- test/test_query.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_query.py b/test/test_query.py index c6aec6185..e53efc29f 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -474,6 +474,12 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.items(q) self.assert_items_matched(results, ['path item', 'caps path']) + def test_utf8_bytes(self): + self.add_album(path=b'/\xc3\xa0/b/c.mp3', title='latin byte') + q = b'path:/\xc3\xa0/b/c.mp3' + results = self.lib.items(q) + self.assert_items_matched(results, ['latin byte']) + class IntQueryTest(unittest.TestCase, TestHelper): From 9e5e7a28e5574b45b92e5a282abbd796cbb18b28 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Feb 2015 19:31:07 +0100 Subject: [PATCH 044/129] InvalidQueryError: resist to any query Even though queries may not contain non-utf8 code points InvalidQueryError ought to be prudent, for such an invalid query would raise an InvalidQueryError which therefore has to be able to manipulate the invalid query. --- beets/dbcore/query.py | 8 +++++++- test/test_library.py | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index cd891148e..e80010ccf 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -37,7 +37,13 @@ class InvalidQueryError(ParsingError): def __init__(self, query, explanation): if isinstance(query, list): query = " ".join(query) - message = "'{0}': {1}".format(query, explanation) + try: + message = "'{0}': {1}".format(query, explanation) + except UnicodeDecodeError: + # queries are unicode. however if for an unholy reason it's not + # the case, an InvalidQueryError may be raised -- and report it + # correctly than fail again here + message = "{0!r}: {1}".format(query, explanation) super(InvalidQueryError, self).__init__(message) diff --git a/test/test_library.py b/test/test_library.py index 6bb88076e..d2193b25f 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -1195,6 +1195,13 @@ class ParseQueryTest(unittest.TestCase): self.assertIsInstance(raised.exception, beets.dbcore.query.ParsingError) + def test_parse_byte_string(self): + with self.assertRaises(beets.dbcore.InvalidQueryError) as raised: + beets.library.parse_query_string(b'f\xf2o', None) + self.assertIn("can't decode", unicode(raised.exception)) + self.assertIsInstance(raised.exception, + beets.dbcore.query.ParsingError) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From c91e8cb782412c4af760bff6eb9084476eeb9d59 Mon Sep 17 00:00:00 2001 From: mried Date: Thu, 19 Feb 2015 19:43:07 +0100 Subject: [PATCH 045/129] Sometimes the extract art test failed because the file type of the extracted image might be PNG or JPG. Belongs to #1328. --- beetsplug/embedart.py | 2 +- test/test_embedart.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 19e9a145d..35feb4d9c 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -266,7 +266,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): self._log.warning(u'Unknown image type in {0}.', displayable_path(item.path)) return - outpath = outpath + b'.' + ext + outpath += b'.' + ext self._log.info(u'Extracting album art from: {0} to: {1}', item, displayable_path(outpath)) diff --git a/test/test_embedart.py b/test/test_embedart.py index 729f5853d..4a4f157ae 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -124,8 +124,11 @@ class EmbedartCliTest(_common.TestCase, TestHelper): self.run_command('extractart', '-n', 'extracted') - self.assertExists(os.path.join(albumpath.decode('utf-8'), - 'extracted.png')) + path = os.path.join(albumpath.decode('utf-8'), 'extracted.') + found = any(map(lambda ext: os.path.exists(path + ext), + ['png', 'jpeg', 'jpg'])) + + self.assertTrue(found, 'Extracted image was not found!') class EmbedartTest(unittest.TestCase): From 2dec90de7a244bd2310accfd1e02ca4f41fc523e Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Fri, 20 Feb 2015 12:52:59 +0100 Subject: [PATCH 046/129] Fix assertTags() in mediafile tests --- test/test_mediafile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_mediafile.py b/test/test_mediafile.py index 8b23d1cee..717364128 100644 --- a/test/test_mediafile.py +++ b/test/test_mediafile.py @@ -662,7 +662,7 @@ class ReadWriteTestBase(ArtTestMixin, GenreListTestMixin, errors.append('Tag %s does not exist' % key) else: if value2 != value: - errors.append('Tag %s: %s != %s' % (key, value2, value)) + errors.append('Tag %s: %r != %r' % (key, value2, value)) if any(errors): errors = ['Tags did not match'] + errors self.fail('\n '.join(errors)) From a8477264ac7452187f0cb14b9fa6470543c11fd0 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Fri, 20 Feb 2015 12:53:43 +0100 Subject: [PATCH 047/129] MediaFile: improve cover art detection Filter media file images on their type. Detection is still not deterministic when 0 or multiple image have type ImageType.front. Fix #1332. --- beets/mediafile.py | 21 +++++++++++++++++---- test/test_mediafile.py | 13 ++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 6c5d0a2c2..22899dee0 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1246,18 +1246,31 @@ class DateItemField(MediaField): class CoverArtField(MediaField): """A descriptor that provides access to the *raw image data* for the - first image on a file. This is used for backwards compatibility: the + cover image on a file. This is used for backwards compatibility: the full `ImageListField` provides richer `Image` objects. + + When there are multiple images we try to pick the most likely to be a front + cover. """ def __init__(self): pass def __get__(self, mediafile, _): - try: - return mediafile.images[0].data - except IndexError: + candidates = mediafile.images + if candidates: + return self.guess_cover_image(candidates).data + else: return None + @staticmethod + def guess_cover_image(candidates): + if len(candidates) == 1: + return candidates[0] + try: + return next(c for c in candidates if c.type == ImageType.front) + except StopIteration: + return candidates[0] + def __set__(self, mediafile, data): if data: mediafile.images = [Image(data=data)] diff --git a/test/test_mediafile.py b/test/test_mediafile.py index 717364128..cd149e7e4 100644 --- a/test/test_mediafile.py +++ b/test/test_mediafile.py @@ -28,7 +28,7 @@ from test import _common from test._common import unittest from beets.mediafile import MediaFile, MediaField, Image, \ MP3DescStorageStyle, StorageStyle, MP4StorageStyle, \ - ASFStorageStyle, ImageType + ASFStorageStyle, ImageType, CoverArtField from beets.library import Item from beets.plugins import BeetsPlugin @@ -161,6 +161,13 @@ class ImageStructureTestMixin(ArtTestMixin): mediafile = MediaFile(mediafile.path) self.assertEqual(len(mediafile.images), 0) + def test_guess_cover(self): + mediafile = self._mediafile_fixture('image') + self.assertEqual(len(mediafile.images), 2) + cover = CoverArtField.guess_cover_image(mediafile.images) + self.assertEqual(cover.desc, 'album cover') + self.assertEqual(mediafile.art, cover.data) + def assertExtendedImageAttributes(self, image, **kwargs): """Ignore extended image attributes in the base tests. """ @@ -758,6 +765,10 @@ class MP4Test(ReadWriteTestBase, PartialTestMixin, with self.assertRaises(ValueError): mediafile.images = [Image(data=self.tiff_data)] + def test_guess_cover(self): + # There is no metadata associated with images, we pick one at random + pass + class AlacTest(ReadWriteTestBase, unittest.TestCase): extension = 'alac.m4a' From 88baf1979e3fd1bd216e2adc9b7839abaa6befa6 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Fri, 20 Feb 2015 12:55:23 +0100 Subject: [PATCH 048/129] Revert "Sometimes the extract art test failed because the file type of the extracted image might be PNG or JPG. Belongs to #1328." This reverts commit c91e8cb782412c4af760bff6eb9084476eeb9d59. --- test/test_embedart.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/test_embedart.py b/test/test_embedart.py index 4a4f157ae..729f5853d 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -124,11 +124,8 @@ class EmbedartCliTest(_common.TestCase, TestHelper): self.run_command('extractart', '-n', 'extracted') - path = os.path.join(albumpath.decode('utf-8'), 'extracted.') - found = any(map(lambda ext: os.path.exists(path + ext), - ['png', 'jpeg', 'jpg'])) - - self.assertTrue(found, 'Extracted image was not found!') + self.assertExists(os.path.join(albumpath.decode('utf-8'), + 'extracted.png')) class EmbedartTest(unittest.TestCase): From f14dbf4e89ea74d09026da5028e4ecf59192c9bd Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 24 Feb 2015 14:37:37 +0100 Subject: [PATCH 049/129] SubcommandsOptionParser: use super() --- beets/ui/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index e041db95e..ad1614bad 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -635,7 +635,7 @@ class Subcommand(object): root_parser.get_prog_name().decode('utf8'), self.name) -class SubcommandsOptionParser(optparse.OptionParser): +class SubcommandsOptionParser(optparse.OptionParser, object): """A variant of OptionParser that parses subcommands and their arguments. """ @@ -653,7 +653,7 @@ class SubcommandsOptionParser(optparse.OptionParser): kwargs['add_help_option'] = False # Super constructor. - optparse.OptionParser.__init__(self, *args, **kwargs) + super(SubcommandsOptionParser, self).__init__(*args, **kwargs) # Our root parser needs to stop on the first unrecognized argument. self.disable_interspersed_args() @@ -670,7 +670,7 @@ class SubcommandsOptionParser(optparse.OptionParser): # Add the list of subcommands to the help message. def format_help(self, formatter=None): # Get the original help message, to which we will append. - out = optparse.OptionParser.format_help(self, formatter) + out = super(SubcommandsOptionParser, self).format_help(formatter) if formatter is None: formatter = self.formatter From 65a88e2bf40a57010e89918ff8e9e96d44b423f4 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 24 Feb 2015 19:24:16 +0100 Subject: [PATCH 050/129] Fix StrubPlugin.write_item() expected arguments Fix #1338. --- beetsplug/scrub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/scrub.py b/beetsplug/scrub.py index f6a3bed27..ac0017474 100644 --- a/beetsplug/scrub.py +++ b/beetsplug/scrub.py @@ -139,7 +139,7 @@ class ScrubPlugin(BeetsPlugin): self._log.error(u'could not scrub {0}: {1}', util.displayable_path(path), exc) - def write_item(self, path): + def write_item(self, item, path, tags): """Automatically embed art into imported albums.""" if not scrubbing and self.config['auto']: self._log.debug(u'auto-scrubbing {0}', util.displayable_path(path)) From ccbe9079718160a89a60a70fe6153028bba7b176 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 24 Feb 2015 22:18:05 -0800 Subject: [PATCH 051/129] Add (skipped) test for #496 --- test/test_library.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/test_library.py b/test/test_library.py index 1a2812b61..e968141a3 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -452,6 +452,22 @@ class DestinationTest(_common.TestCase): self.assertEqual(self.i.destination(), np('base/one/_.mp3')) + @unittest.skip('unimplemented: #496') + def test_truncation_does_not_conflict_with_replacement(self): + # Use a replacement that should always replace the last X in any + # path component with a Z. + self.lib.replacements = [ + (re.compile(r'X$'), u'Z'), + ] + + # Construct an item whose untruncated path ends with a Y but whose + # truncated version ends with an X. + self.i.title = 'X' * 300 + 'Y' + + # The final path should reflect the replacement. + dest = self.i.destination() + self.assertTrue('XZ' in dest) + class ItemFormattedMappingTest(_common.LibTestCase): def test_formatted_item_value(self): From 1385ce11cab8f7fc5c98ad313f0515ea1f660bfb Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Sat, 28 Feb 2015 15:35:48 +0100 Subject: [PATCH 052/129] Added support for bs1770gain, a loudness-scanner --- beetsplug/replaygain.py | 111 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ce41cad57..655152f8e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -21,6 +21,7 @@ import collections import itertools import sys import warnings +import re from beets import logging from beets import ui @@ -83,6 +84,109 @@ class Backend(object): raise NotImplementedError() +# BS1770GAIN CLI tool backend. +class bs1770gainBackend(Backend): + def __init__(self, config, log): + super(bs1770gainBackend, self).__init__(config, log) + self.command = 'bs1770gain' + self.method = config["method"].get(unicode) + if self.command: + # Check whether the program is in $PATH. + for cmd in ('bs1770gain'): + try: + call([cmd]) + self.command = cmd + except OSError: + pass + if not self.command: + raise FatalReplayGainError( + 'no bs1770gain command found: install bs1770gain' + ) + def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of TrackGain objects. + """ + #supported_items = filter(self.format_supported, items) + output = self.compute_gain(items, False) + return output + + def compute_album_gain(self, album): + """Computes the album gain of the given album, returns an + AlbumGain object. + """ + # TODO: What should be done when not all tracks in the album are + # supported? + + supported_items = album.items() + output = self.compute_gain(supported_items, True) + + return AlbumGain(output[-1], output[:-1]) + + def format_supported(self, item): + """Checks whether the given item is supported by the selected tool. + """ + if 'mp3gain' in self.command and item.format != 'MP3': + return False + elif 'aacgain' in self.command and item.format not in ('MP3', 'AAC'): + return False + return True + + def compute_gain(self, items, is_album): + """Computes the track or album gain of a list of items, returns + a list of TrackGain objects. + When computing album gain, the last TrackGain object returned is + the album gain + """ + + if len(items) == 0: + return [] + + """Compute ReplayGain values and return a list of results + dictionaries as given by `parse_tool_output`. + """ + # Construct shell command. + cmd = [self.command] + cmd = cmd + [self.method] + cmd = cmd + ['-it'] + cmd = cmd + [syspath(i.path) for i in items] + + self._log.debug(u'analyzing {0} files', len(items)) + self._log.debug(u"executing {0}", " ".join(map(displayable_path, cmd))) + output = call(cmd) + self._log.debug(u'analysis finished') + results = self.parse_tool_output(output, + len(items) + (1 if is_album else 0)) + return results + + def parse_tool_output(self, text, num_lines): + """Given the output from bs1770gain, parse the text and + return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + data = unicode(text,errors='ignore') + results=re.findall(r'(\s{2,2}\[\d+\/\d+\].*?|\[ALBUM\].*?)(?=\s{2,2}\[\d+\/\d+\]|\s{2,2}\[ALBUM\]:|done\.$)',data,re.S|re.M) + + for ll in results[0:num_lines]: + parts = ll.split(b'\n') + if len(parts) == 0: + self._log.debug(u'bad tool output: {0}', text) + raise ReplayGainError('bs1770gain failed') + + d = { + 'file': parts[0], + 'gain': float((parts[1].split('/'))[1].split('LU')[0]), + 'peak': float(parts[2].split('/')[1]), + } + + self._log.info('analysed {}gain={};peak={}', + d['file'].rstrip(), d['gain'], d['peak']) + out.append(Gain(d['gain'], d['peak'])) + return out + + +# GStreamer-based backend. + # mpgain/aacgain CLI tool backend. @@ -179,6 +283,7 @@ class CommandBackend(Backend): else: # Disable clipping warning. cmd = cmd + ['-c'] + cmd = cmd + ['-a' if is_album else '-r'] cmd = cmd + ['-d', bytes(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] @@ -598,9 +703,10 @@ class ReplayGainPlugin(BeetsPlugin): """ backends = { - "command": CommandBackend, + "command": CommandBackend, "gstreamer": GStreamerBackend, - "audiotools": AudioToolsBackend + "audiotools": AudioToolsBackend, + "bs1770gain": bs1770gainBackend } def __init__(self): @@ -688,6 +794,7 @@ class ReplayGainPlugin(BeetsPlugin): ) self.store_album_gain(album, album_gain.album_gain) + for item, track_gain in itertools.izip(album.items(), album_gain.track_gains): self.store_track_gain(item, track_gain) From 952081e5edf6f87fec85537c7d14e4e02ae02149 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 1 Mar 2015 14:52:31 +0100 Subject: [PATCH 053/129] Revert "InvalidQueryError: resist to any query" This reverts commit 9e5e7a28e5574b45b92e5a282abbd796cbb18b28. --- beets/dbcore/query.py | 8 +------- test/test_library.py | 7 ------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index e80010ccf..cd891148e 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -37,13 +37,7 @@ class InvalidQueryError(ParsingError): def __init__(self, query, explanation): if isinstance(query, list): query = " ".join(query) - try: - message = "'{0}': {1}".format(query, explanation) - except UnicodeDecodeError: - # queries are unicode. however if for an unholy reason it's not - # the case, an InvalidQueryError may be raised -- and report it - # correctly than fail again here - message = "{0!r}: {1}".format(query, explanation) + message = "'{0}': {1}".format(query, explanation) super(InvalidQueryError, self).__init__(message) diff --git a/test/test_library.py b/test/test_library.py index d2193b25f..6bb88076e 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -1195,13 +1195,6 @@ class ParseQueryTest(unittest.TestCase): self.assertIsInstance(raised.exception, beets.dbcore.query.ParsingError) - def test_parse_byte_string(self): - with self.assertRaises(beets.dbcore.InvalidQueryError) as raised: - beets.library.parse_query_string(b'f\xf2o', None) - self.assertIn("can't decode", unicode(raised.exception)) - self.assertIsInstance(raised.exception, - beets.dbcore.query.ParsingError) - def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From cb504ad163ec81348c2b30c5d169a7cb9f448495 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 1 Mar 2015 14:57:10 +0100 Subject: [PATCH 054/129] library.parse_query_string: assert query is unicode --- beets/library.py | 5 +++-- test/test_library.py | 4 ++++ test/test_query.py | 8 +------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/beets/library.py b/beets/library.py index c5503dd0a..0c1acba22 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1107,11 +1107,12 @@ def parse_query_string(s, model_cls): The string is split into components using shell-like syntax. """ + assert isinstance(s, unicode), "Query is not unicode: {0!r}".format(s) + # A bug in Python < 2.7.3 prevents correct shlex splitting of # Unicode strings. # http://bugs.python.org/issue6988 - if isinstance(s, unicode): - s = s.encode('utf8') + s = s.encode('utf8') try: parts = [p.decode('utf8') for p in shlex.split(s)] except ValueError as exc: diff --git a/test/test_library.py b/test/test_library.py index 6bb88076e..9ec43c2bf 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -1195,6 +1195,10 @@ class ParseQueryTest(unittest.TestCase): self.assertIsInstance(raised.exception, beets.dbcore.query.ParsingError) + def test_parse_bytes(self): + with self.assertRaises(AssertionError): + beets.library.parse_query_string(b"query", None) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/test/test_query.py b/test/test_query.py index e53efc29f..2511aa841 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -465,7 +465,7 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): def test_case_sensitivity(self): self.add_album(path='/A/B/C2.mp3', title='caps path') - q = b'path:/A/B' + q = 'path:/A/B' with patch('beets.library.PathQuery._is_windows', False): results = self.lib.items(q) self.assert_items_matched(results, ['caps path']) @@ -474,12 +474,6 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.items(q) self.assert_items_matched(results, ['path item', 'caps path']) - def test_utf8_bytes(self): - self.add_album(path=b'/\xc3\xa0/b/c.mp3', title='latin byte') - q = b'path:/\xc3\xa0/b/c.mp3' - results = self.lib.items(q) - self.assert_items_matched(results, ['latin byte']) - class IntQueryTest(unittest.TestCase, TestHelper): From 9efcfbb8fa32c1a4ac2a2147a6250414f0d53d0c Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 1 Mar 2015 18:10:07 +0100 Subject: [PATCH 055/129] PathQuery: add 'case_sensitivity' param - fully tested - default value is platform-aware --- beets/library.py | 16 +++++++++++----- test/test_query.py | 20 +++++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/beets/library.py b/beets/library.py index 0c1acba22..b6d8f4c07 100644 --- a/beets/library.py +++ b/beets/library.py @@ -45,7 +45,8 @@ log = logging.getLogger('beets') class PathQuery(dbcore.FieldQuery): """A query that matches all items under a given path. - On Windows paths are case-insensitive, contratly to UNIX platforms. + On Windows paths are case-insensitive by default, contrarly to UNIX + platforms. """ escape_re = re.compile(r'[\\_%]') @@ -53,11 +54,16 @@ class PathQuery(dbcore.FieldQuery): _is_windows = platform.system() == 'Windows' - def __init__(self, field, pattern, fast=True): + def __init__(self, field, pattern, fast=True, case_sensitive=None): super(PathQuery, self).__init__(field, pattern, fast) - if self._is_windows: + if case_sensitive is None: + # setting this value as the default one would make it un-patchable + # and therefore un-testable + case_sensitive = not self._is_windows + if not case_sensitive: pattern = pattern.lower() + self.case_sensitive = case_sensitive # Match the path as a single file. self.file_path = util.bytestring_path(util.normpath(pattern)) @@ -65,13 +71,13 @@ class PathQuery(dbcore.FieldQuery): self.dir_path = util.bytestring_path(os.path.join(self.file_path, b'')) def match(self, item): - path = item.path.lower() if self._is_windows else item.path + path = item.path if self.case_sensitive else item.path.lower() return (path == self.file_path) or path.startswith(self.dir_path) def col_clause(self): file_blob = buffer(self.file_path) - if not self._is_windows: + if self.case_sensitive: dir_blob = buffer(self.dir_path) return '({0} = ?) || (substr({0}, 1, ?) = ?)'.format(self.field), \ (file_blob, len(dir_blob), dir_blob) diff --git a/test/test_query.py b/test/test_query.py index 2511aa841..ee0f3d0ba 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -17,6 +17,7 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +from functools import partial from mock import patch from test import _common @@ -465,14 +466,23 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): def test_case_sensitivity(self): self.add_album(path='/A/B/C2.mp3', title='caps path') - q = 'path:/A/B' + + makeq = partial(beets.library.PathQuery, 'path', '/A/B') + + results = self.lib.items(makeq(case_sensitive=True)) + self.assert_items_matched(results, ['caps path']) + + results = self.lib.items(makeq(case_sensitive=False)) + self.assert_items_matched(results, ['path item', 'caps path']) + + # test platform-aware default sensitivity with patch('beets.library.PathQuery._is_windows', False): - results = self.lib.items(q) - self.assert_items_matched(results, ['caps path']) + q = makeq() + self.assertEqual(q.case_sensitive, True) with patch('beets.library.PathQuery._is_windows', True): - results = self.lib.items(q) - self.assert_items_matched(results, ['path item', 'caps path']) + q = makeq() + self.assertEqual(q.case_sensitive, False) class IntQueryTest(unittest.TestCase, TestHelper): From ddf86af3a0fa1129bb12781aa196dbc7532aa585 Mon Sep 17 00:00:00 2001 From: Taeyeon Mori Date: Sun, 1 Mar 2015 19:49:31 +0100 Subject: [PATCH 056/129] DOCS The plugin stages now receive the ImportSession as first argument --- docs/dev/plugins.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index f448b5dfa..79a3f7354 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -382,7 +382,7 @@ Multiple stages run in parallel but each stage processes only one task at a time and each task is processed by only one stage at a time. Plugins provide stages as functions that take two arguments: ``config`` and -``task``, which are ``ImportConfig`` and ``ImportTask`` objects (both defined in +``task``, which are ``ImportSession`` and ``ImportTask`` objects (both defined in ``beets.importer``). Add such a function to the plugin's ``import_stages`` field to register it:: @@ -391,7 +391,7 @@ to register it:: def __init__(self): super(ExamplePlugin, self).__init__() self.import_stages = [self.stage] - def stage(self, config, task): + def stage(self, session, task): print('Importing something!') .. _extend-query: From eec8d5d2be922cd295f7888f656eef2545346168 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 1 Mar 2015 17:09:36 -0800 Subject: [PATCH 057/129] Doc rewording for #1330 --- beets/library.py | 17 ++++++++++++----- docs/changelog.rst | 2 +- docs/reference/query.rst | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/beets/library.py b/beets/library.py index fe9bee081..9b22fd4df 100644 --- a/beets/library.py +++ b/beets/library.py @@ -45,8 +45,9 @@ log = logging.getLogger('beets') class PathQuery(dbcore.FieldQuery): """A query that matches all items under a given path. - On Windows paths are case-insensitive by default, contrarly to UNIX - platforms. + Matching can either base case-sensitive or case-sensitive. By + default, the behavior depends on the OS: case-insensitive on Windows + and case-sensitive otherwise. """ escape_re = re.compile(r'[\\_%]') @@ -55,15 +56,21 @@ class PathQuery(dbcore.FieldQuery): _is_windows = platform.system() == 'Windows' def __init__(self, field, pattern, fast=True, case_sensitive=None): + """Create a path query. + + `case_sensitive` can be a bool or `None`, indicating that the + behavior should depend on the platform (the default). + """ super(PathQuery, self).__init__(field, pattern, fast) + # By default, the case sensitivity depends on the platform. if case_sensitive is None: - # setting this value as the default one would make it un-patchable - # and therefore un-testable case_sensitive = not self._is_windows + self.case_sensitive = case_sensitive + + # Use a normalized-case pattern for case-insensitive matches. if not case_sensitive: pattern = pattern.lower() - self.case_sensitive = case_sensitive # Match the path as a single file. self.file_path = util.bytestring_path(util.normpath(pattern)) diff --git a/docs/changelog.rst b/docs/changelog.rst index 449eb4017..11a64d3ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -76,7 +76,7 @@ Fixes: * :doc:`/plugins/replaygain`: Stop applying replaygain directly to source files when using the mp3gain backend. :bug:`1316` -* Path queries are case-sensitive on UNIX OSes. :bug:`1165` +* Path queries are case-sensitive on non-Windows OSes. :bug:`1165` * :doc:`/plugins/lyrics`: Silence a warning about insecure requests in the new MusixMatch backend. :bug:`1204` * Fix a crash when ``beet`` is invoked without arguments. :bug:`1205` diff --git a/docs/reference/query.rst b/docs/reference/query.rst index af676a50d..20c5360f8 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -184,7 +184,7 @@ Note that this only matches items that are *already in your library*, so a path query won't necessarily find *all* the audio files in a directory---just the ones you've already added to your beets library. -Such queries are case-sensitive on UNIX and case-insensitive on Microsoft +Path queries are case-sensitive on most platforms but case-insensitive on Windows. .. _query-sort: From 31c7c4a87735eb1dcb1ffe0616c62760a4d84ccc Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 1 Mar 2015 17:11:59 -0800 Subject: [PATCH 058/129] Avoid a little global state (#1330) For even clearer interaction with the environment. --- beets/library.py | 4 +--- test/test_query.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 9b22fd4df..793e5ca40 100644 --- a/beets/library.py +++ b/beets/library.py @@ -53,8 +53,6 @@ class PathQuery(dbcore.FieldQuery): escape_re = re.compile(r'[\\_%]') escape_char = b'\\' - _is_windows = platform.system() == 'Windows' - def __init__(self, field, pattern, fast=True, case_sensitive=None): """Create a path query. @@ -65,7 +63,7 @@ class PathQuery(dbcore.FieldQuery): # By default, the case sensitivity depends on the platform. if case_sensitive is None: - case_sensitive = not self._is_windows + case_sensitive = platform.system() != 'Windows' self.case_sensitive = case_sensitive # Use a normalized-case pattern for case-insensitive matches. diff --git a/test/test_query.py b/test/test_query.py index ee0f3d0ba..4334488df 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -18,7 +18,6 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) from functools import partial -from mock import patch from test import _common from test._common import unittest @@ -476,11 +475,11 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): self.assert_items_matched(results, ['path item', 'caps path']) # test platform-aware default sensitivity - with patch('beets.library.PathQuery._is_windows', False): + with _common.platform_posix(): q = makeq() self.assertEqual(q.case_sensitive, True) - with patch('beets.library.PathQuery._is_windows', True): + with _common.platform_windows(): q = makeq() self.assertEqual(q.case_sensitive, False) From 9c4492752ff1b3e16c61d8be867d7688de811058 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 1 Mar 2015 17:33:11 -0800 Subject: [PATCH 059/129] Fix a test fix :flushed: for #1330 --- test/test_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 4334488df..d512e02b8 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -475,11 +475,11 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): self.assert_items_matched(results, ['path item', 'caps path']) # test platform-aware default sensitivity - with _common.platform_posix(): + with _common.system_mock('Darwin'): q = makeq() self.assertEqual(q.case_sensitive, True) - with _common.platform_windows(): + with _common.system_mock('Windows'): q = makeq() self.assertEqual(q.case_sensitive, False) From 226a90d12a85abc2fa19efde23feacec97d7dc34 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 2 Mar 2015 08:51:59 +0100 Subject: [PATCH 060/129] PathQuery: fix docstring --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 793e5ca40..0dd3acaf1 100644 --- a/beets/library.py +++ b/beets/library.py @@ -45,7 +45,7 @@ log = logging.getLogger('beets') class PathQuery(dbcore.FieldQuery): """A query that matches all items under a given path. - Matching can either base case-sensitive or case-sensitive. By + Matching can either be case-insensitive or case-sensitive. By default, the behavior depends on the OS: case-insensitive on Windows and case-sensitive otherwise. """ From 72c5db88765bafb4e43ce072e5bdd21c37467f45 Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Mon, 2 Mar 2015 15:38:33 +0100 Subject: [PATCH 061/129] add doc, clean-up code --- beetsplug/replaygain.py | 114 +++++++++++++++++++++++++++++++++++- docs/plugins/replaygain.rst | 31 +++++++++- 2 files changed, 139 insertions(+), 6 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ce41cad57..cd0022846 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -21,6 +21,7 @@ import collections import itertools import sys import warnings +import re from beets import logging from beets import ui @@ -32,12 +33,14 @@ from beets import config # Utilities. class ReplayGainError(Exception): + """Raised when a local (to a track or an album) error occurs in one of the backends. """ class FatalReplayGainError(Exception): + """Raised when a fatal error occurs in one of the backends. """ @@ -66,8 +69,10 @@ AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") class Backend(object): + """An abstract class representing engine for calculating RG values. """ + def __init__(self, config, log): """Initialize the backend with the configuration view for the plugin. @@ -83,10 +88,108 @@ class Backend(object): raise NotImplementedError() +# bsg1770gain backend + + +class Bs1770gainBackend(Backend): + + def __init__(self, config, log): + super(Bs1770gainBackend, self).__init__(config, log) + cmd = 'bs1770gain' + + try: + self.method = '--' + config['method'].get(unicode) + except: + self.method = '--replaygain' + + try: + call([cmd, self.method]) + self.command = cmd + except OSError: + pass + if not self.command: + raise FatalReplayGainError( + 'no replaygain command found: install bs1770gain' + ) + + def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of TrackGain objects. + """ + output = self.compute_gain(items, False) + return output + + def compute_album_gain(self, album): + """Computes the album gain of the given album, returns an + AlbumGain object. + """ + # TODO: What should be done when not all tracks in the album are + # supported? + + supported_items = album.items() + output = self.compute_gain(supported_items, True) + + return AlbumGain(output[-1], output[:-1]) + + def compute_gain(self, items, is_album): + """Computes the track or album gain of a list of items, returns + a list of TrackGain objects. + When computing album gain, the last TrackGain object returned is + the album gain + """ + + if len(items) == 0: + return [] + + """Compute ReplayGain values and return a list of results + dictionaries as given by `parse_tool_output`. + """ + # Construct shell command. + cmd = [self.command] + cmd = cmd + [self.method] + cmd = cmd + ['-it'] + cmd = cmd + [syspath(i.path) for i in items] + + self._log.debug(u'analyzing {0} files', len(items)) + self._log.debug(u"executing {0}", " ".join(map(displayable_path, cmd))) + output = call(cmd) + self._log.debug(u'analysis finished') + results = self.parse_tool_output(output, + len(items) + is_album) + return results + + def parse_tool_output(self, text, num_lines): + """Given the output from bs1770gain, parse the text and + return a list of dictionaries + containing information about each analyzed file. + """ + out = [] + data = unicode(text, errors='ignore') + regex = ("(\s{2,2}\[\d+\/\d+\].*?|\[ALBUM\].*?)(?=\s{2,2}\[\d+\/\d+\]" + "|\s{2,2}\[ALBUM\]:|done\.$)") + + results = re.findall(regex, data, re.S | re.M) + for ll in results[0:num_lines]: + parts = ll.split(b'\n') + if len(parts) == 0: + self._log.debug(u'bad tool output: {0}', text) + raise ReplayGainError('bs1770gain failed') + + d = { + 'file': parts[0], + 'gain': float((parts[1].split('/'))[1].split('LU')[0]), + 'peak': float(parts[2].split('/')[1]), + } + + self._log.info('analysed {}gain={};peak={}', + d['file'].rstrip(), d['gain'], d['peak']) + out.append(Gain(d['gain'], d['peak'])) + return out + + # mpgain/aacgain CLI tool backend. - - class CommandBackend(Backend): + def __init__(self, config, log): super(CommandBackend, self).__init__(config, log) config.add({ @@ -218,6 +321,7 @@ class CommandBackend(Backend): # GStreamer-based backend. class GStreamerBackend(Backend): + def __init__(self, config, log): super(GStreamerBackend, self).__init__(config, log) self._import_gst() @@ -466,10 +570,12 @@ class GStreamerBackend(Backend): class AudioToolsBackend(Backend): + """ReplayGain backend that uses `Python Audio Tools `_ and its capabilities to read more file formats and compute ReplayGain values using it replaygain module. """ + def __init__(self, config, log): super(CommandBackend, self).__init__(config, log) self._import_audiotools() @@ -594,13 +700,15 @@ class AudioToolsBackend(Backend): # Main plugin logic. class ReplayGainPlugin(BeetsPlugin): + """Provides ReplayGain analysis. """ backends = { "command": CommandBackend, "gstreamer": GStreamerBackend, - "audiotools": AudioToolsBackend + "audiotools": AudioToolsBackend, + "bs1770gain": Bs1770gainBackend } def __init__(self): diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index d2584a648..8c0bf610d 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -10,9 +10,9 @@ playback levels. Installation ------------ -This plugin can use one of three backends to compute the ReplayGain values: -GStreamer, mp3gain (and its cousin, aacgain), and Python Audio Tools. mp3gain -can be easier to install but GStreamer and Audio Tools support more audio +This plugin can use one of four backends to compute the ReplayGain values: +GStreamer, mp3gain (and its cousin, aacgain), Python Audio Tools and bs1770gain. mp3gain +can be easier to install but GStreamer, Audio Tools and bs1770gain support more audio formats. Once installed, this plugin analyzes all files during the import process. This @@ -75,6 +75,22 @@ On OS X, most of the dependencies can be installed with `Homebrew`_:: .. _Python Audio Tools: http://audiotools.sourceforge.net +bs1770gain +`````````` + +In order to use this backend, you will need to install the bs1770gain command-line tool. Here are some hints: + +* goto `bs1770gain`_ and follow the download instructions +* make sure it is in your $PATH + +.. _bs1770gain: bs1770gain.sourceforge.net + +Then, enable the plugin (see :ref:`using-plugins`) and specify the +backend in your configuration file:: + + replaygain: + backend: bs1770gain + Configuration ------------- @@ -100,6 +116,15 @@ These options only work with the "command" backend: would keep clipping from occurring. Default: ``yes``. +This option only works with the "bs1770gain" backend: + +- **method**:either replaygain, ebu or atsc. Default: replaygain + + replaygain measures loudness with a reference level of -18 LUFS. + ebu measures loudness with a reference level of -23 LUFS. + atsc measures loudness with a reference level of -24 LUFS. + + Manual Analysis --------------- From 8bd0633496da9038e020493cafadc333245ff02d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 2 Mar 2015 11:30:14 -0800 Subject: [PATCH 062/129] replaygain: Fix `super` call in AudioTools Fix #1342. --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ce41cad57..378e9466e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -471,7 +471,7 @@ class AudioToolsBackend(Backend): file formats and compute ReplayGain values using it replaygain module. """ def __init__(self, config, log): - super(CommandBackend, self).__init__(config, log) + super(AudioToolsBackend, self).__init__(config, log) self._import_audiotools() def _import_audiotools(self): From 5bc8ef700910aa84753a0a1fdc87f88bb938a945 Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Mon, 2 Mar 2015 22:11:33 +0100 Subject: [PATCH 063/129] fix some formating --- beetsplug/replaygain.py | 19 ++++++++----------- docs/plugins/replaygain.rst | 8 ++++---- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index c52cbe053..a13b58985 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -88,12 +88,14 @@ class Backend(object): raise NotImplementedError() - # bsg1770gain backend - - class Bs1770gainBackend(Backend): + """bs1770gain is a loudness scanner compliant with ITU-R BS.1770 and its + flavors EBU R128,ATSC A/85 and Replaygain 2.0. It uses a special + designed algorithm to normalize audio to the same level. + """ + def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) cmd = 'bs1770gain' @@ -107,19 +109,17 @@ class Bs1770gainBackend(Backend): call([cmd, self.method]) self.command = cmd except OSError: - pass + pass if not self.command: raise FatalReplayGainError( 'no replaygain command found: install bs1770gain' ) - def compute_track_gain(self, items): """Computes the track gain of the given tracks, returns a list of TrackGain objects. """ - output = self.compute_gain(items, False) return output @@ -135,8 +135,6 @@ class Bs1770gainBackend(Backend): return AlbumGain(output[-1], output[:-1]) - - def compute_gain(self, items, is_album): """Computes the track or album gain of a list of items, returns a list of TrackGain objects. @@ -147,7 +145,6 @@ class Bs1770gainBackend(Backend): if len(items) == 0: return [] - """Compute ReplayGain values and return a list of results dictionaries as given by `parse_tool_output`. """ @@ -171,7 +168,7 @@ class Bs1770gainBackend(Backend): containing information about each analyzed file. """ out = [] - data = unicode(text, errors='ignore') + data = text.decode('utf8', errors='ignore') regex = ("(\s{2,2}\[\d+\/\d+\].*?|\[ALBUM\].*?)(?=\s{2,2}\[\d+\/\d+\]" "|\s{2,2}\[ALBUM\]:|done\.$)") @@ -179,7 +176,7 @@ class Bs1770gainBackend(Backend): for ll in results[0:num_lines]: parts = ll.split(b'\n') if len(parts) == 0: - self._log.debug(u'bad tool output: {0}', text) + self._log.debug(u'bad tool output: {0!r}', text) raise ReplayGainError('bs1770gain failed') d = { diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 8c0bf610d..f08a23953 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -118,11 +118,11 @@ These options only work with the "command" backend: This option only works with the "bs1770gain" backend: -- **method**:either replaygain, ebu or atsc. Default: replaygain +- **method**: The loudness scanning standard: either 'replaygain' for ReplayGain 2.0, + 'ebu' for EBU R128 or 'atsc' for ATSC A/85. + This dictates the reference level: -18, -23, or -24 LUFS respectively. Default: replaygain + - replaygain measures loudness with a reference level of -18 LUFS. - ebu measures loudness with a reference level of -23 LUFS. - atsc measures loudness with a reference level of -24 LUFS. Manual Analysis From 80c49ab36098398a4fbcb2a0ec920f4385e151ab Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Tue, 3 Mar 2015 11:04:03 +0100 Subject: [PATCH 064/129] removed line 288 --- beetsplug/replaygain.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index bbaa47d0b..dcb98ff3a 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -286,7 +286,6 @@ class CommandBackend(Backend): else: # Disable clipping warning. cmd = cmd + ['-c'] - cmd = cmd + ['-a' if is_album else '-r'] cmd = cmd + ['-d', bytes(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] From a3e32fd410118e9f479cce53bc8e487d7d434b1d Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Tue, 3 Mar 2015 11:23:45 +0100 Subject: [PATCH 065/129] added fatalreplaygainerror --- beetsplug/replaygain.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index dcb98ff3a..39d1f0382 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -109,7 +109,9 @@ class Bs1770gainBackend(Backend): call([cmd, self.method]) self.command = cmd except OSError: - pass + raise FatalReplayGainError( + 'Is bs1770gain installed? Is your method in conifg correct?' + ) if not self.command: raise FatalReplayGainError( 'no replaygain command found: install bs1770gain' From 5d7d402adb5642e8604aa766b2efc088b2baa00e Mon Sep 17 00:00:00 2001 From: jmwatte Date: Tue, 3 Mar 2015 13:11:25 +0100 Subject: [PATCH 066/129] correct typing error --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 39d1f0382..e9d71619f 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -110,7 +110,7 @@ class Bs1770gainBackend(Backend): self.command = cmd except OSError: raise FatalReplayGainError( - 'Is bs1770gain installed? Is your method in conifg correct?' + 'Is bs1770gain installed? Is your method in config correct?' ) if not self.command: raise FatalReplayGainError( From 48671cbdf1cba0ae7dedf63b037571b49a1465a3 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 3 Mar 2015 10:38:01 -0800 Subject: [PATCH 067/129] Changelog for #1343 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 11a64d3ad..ab9918ae3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog Features: +* :doc:`/plugins/replaygain`: There is a new backend for the `bs1770gain`_ + tool. Thanks to :user:`jmwatte`. :bug:`1343` * There are now multiple levels of verbosity. On the command line, you can make beets somewhat verbose with ``-v`` or very verbose with ``-vv``. :bug:`1244` @@ -129,6 +131,8 @@ For developers: immediately after they are initialized. It's also possible to replace the originally created tasks by returning new ones using this event. +.. _bs1770gain: http://bs1770gain.sourceforge.net + 1.3.10 (January 5, 2015) ------------------------ From 8113c7ba85febeb8c976081adecae6f0e04e5eef Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 3 Mar 2015 10:40:53 -0800 Subject: [PATCH 068/129] Roll back whitespace changes from #1343 --- beetsplug/replaygain.py | 12 ++---------- docs/plugins/replaygain.rst | 12 ++++++------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index e9d71619f..c4bebc8dd 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -33,14 +33,12 @@ from beets import config # Utilities. class ReplayGainError(Exception): - """Raised when a local (to a track or an album) error occurs in one of the backends. """ class FatalReplayGainError(Exception): - """Raised when a fatal error occurs in one of the backends. """ @@ -69,7 +67,6 @@ AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") class Backend(object): - """An abstract class representing engine for calculating RG values. """ @@ -90,10 +87,8 @@ class Backend(object): # bsg1770gain backend class Bs1770gainBackend(Backend): - - """bs1770gain is a loudness scanner compliant with ITU-R BS.1770 and its - flavors EBU R128,ATSC A/85 and Replaygain 2.0. It uses a special - designed algorithm to normalize audio to the same level. + """bs1770gain is a loudness scanner compliant with ITU-R BS.1770 and + its flavors EBU R128, ATSC A/85 and Replaygain 2.0. """ def __init__(self, config, log): @@ -576,7 +571,6 @@ class GStreamerBackend(Backend): class AudioToolsBackend(Backend): - """ReplayGain backend that uses `Python Audio Tools `_ and its capabilities to read more file formats and compute ReplayGain values using it replaygain module. @@ -706,7 +700,6 @@ class AudioToolsBackend(Backend): # Main plugin logic. class ReplayGainPlugin(BeetsPlugin): - """Provides ReplayGain analysis. """ @@ -802,7 +795,6 @@ class ReplayGainPlugin(BeetsPlugin): ) self.store_album_gain(album, album_gain.album_gain) - for item, track_gain in itertools.izip(album.items(), album_gain.track_gains): self.store_track_gain(item, track_gain) diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index f08a23953..b8f385df8 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -90,7 +90,8 @@ backend in your configuration file:: replaygain: backend: bs1770gain - + + Configuration ------------- @@ -118,11 +119,10 @@ These options only work with the "command" backend: This option only works with the "bs1770gain" backend: -- **method**: The loudness scanning standard: either 'replaygain' for ReplayGain 2.0, - 'ebu' for EBU R128 or 'atsc' for ATSC A/85. - This dictates the reference level: -18, -23, or -24 LUFS respectively. Default: replaygain - - +- **method**: The loudness scanning standard: either `replaygain` for + ReplayGain 2.0, `ebu` for EBU R128, or `atsc` for ATSC A/85. This dictates + the reference level: -18, -23, or -24 LUFS respectively. Default: + `replaygain` Manual Analysis From 614f8eda885ca43dd682f19212522bc9a9f02d4c Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Tue, 3 Mar 2015 23:02:02 +0100 Subject: [PATCH 069/129] added tests for bs1770gain --- test/test_replaygain.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 64d65b006..84306a299 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -33,6 +33,10 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False +if has_program('bs1770gain', ['--replaygain']): + LOUDNESS_PROG_AVAILABLE = True +else: + LOUDNESS_PROG_AVAILABLE = False class ReplayGainCliTestBase(TestHelper): @@ -123,6 +127,11 @@ class ReplayGainCmdCliTest(ReplayGainCliTestBase, unittest.TestCase): backend = u'command' +@unittest.skipIf(not LOUDNESS_PROG_AVAILABLE, 'bs1770gain cannot be found') +class ReplayGainLdnsCliTest(ReplayGainCliTestBase, unittest.TestCase): + backend = u'bs1770gain' + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 5d9bb5059a2b126b964b28f93fe4e0d6e010394f Mon Sep 17 00:00:00 2001 From: jean-marie winters Date: Wed, 4 Mar 2015 07:20:53 +0100 Subject: [PATCH 070/129] corrected typo --- test/test_replaygain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 84306a299..ba2628fef 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -38,6 +38,7 @@ if has_program('bs1770gain', ['--replaygain']): else: LOUDNESS_PROG_AVAILABLE = False + class ReplayGainCliTestBase(TestHelper): def setUp(self): From 69786b8538caa01c1cea772b1f14f10236f70fd5 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 4 Mar 2015 12:09:40 +0100 Subject: [PATCH 071/129] Fix test.helper.has_program(): encode command subprocess.subprocess.check_call() should receive a byte string command otherwise it fails with a UnicodeDecodeError on systems with non-ascii elements in the system path. Discovered while reviewing PR #1344. --- test/helper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/helper.py b/test/helper.py index 8d0dbf8a6..bb32c1c87 100644 --- a/test/helper.py +++ b/test/helper.py @@ -51,6 +51,7 @@ from beets.library import Library, Item, Album from beets import importer from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.mediafile import MediaFile, Image +from beets.ui import _encoding # TODO Move AutotagMock here from test import _common @@ -117,9 +118,13 @@ def capture_stdout(): def has_program(cmd, args=['--version']): """Returns `True` if `cmd` can be executed. """ + full_cmd = [cmd] + args + for i, elem in enumerate(full_cmd): + if isinstance(elem, unicode): + full_cmd[i] = elem.encode(_encoding()) try: with open(os.devnull, 'wb') as devnull: - subprocess.check_call([cmd] + args, stderr=devnull, + subprocess.check_call(full_cmd, stderr=devnull, stdout=devnull, stdin=devnull) except OSError: return False From f8e2ca2c944d5dd2c1d7cec46372d0098192ec1b Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 4 Mar 2015 12:15:56 +0100 Subject: [PATCH 072/129] Replaygain tests: more careful plugins unloading When plugins loading as failed plugins unloading may fail in consequence, swallowing the loading error. This fixes it. --- test/test_replaygain.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 64d65b006..2c623725c 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -42,9 +42,18 @@ class ReplayGainCliTestBase(TestHelper): try: self.load_plugins('replaygain') except: - self.teardown_beets() - self.unload_plugins() - raise + import sys + # store exception info so an error in teardown does not swallow it + exc_info = sys.exc_info() + try: + self.teardown_beets() + self.unload_plugins() + except: + # if load_plugins() failed then setup is incomplete and + # teardown operations may fail. In particular # {Item,Album} + # may not have the _original_types attribute in unload_plugins + pass + raise exc_info[1], None, exc_info[2] self.config['replaygain']['backend'] = self.backend album = self.add_album_fixture(2) From 9750351a1b6b98136a32f9d9b2b23f034479f593 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 4 Mar 2015 14:48:17 +0100 Subject: [PATCH 073/129] beets.util.command_output() & related receives bytes - May fail with unicode, esp. will non-ascii system path entry. - Only send it bytes. - Also applies to subprocess.Popen() & co. --- beets/util/__init__.py | 6 +++--- beets/util/artresizer.py | 6 +++--- beetsplug/embedart.py | 6 +++--- beetsplug/replaygain.py | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 2d6968e13..53156143f 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -624,7 +624,7 @@ def cpu_count(): num = 0 elif sys.platform == b'darwin': try: - num = int(command_output(['sysctl', '-n', 'hw.ncpu'])) + num = int(command_output([b'sysctl', b'-n', b'hw.ncpu'])) except ValueError: num = 0 else: @@ -641,8 +641,8 @@ def cpu_count(): def command_output(cmd, shell=False): """Runs the command and returns its output after it has exited. - ``cmd`` is a list of arguments starting with the command names. If - ``shell`` is true, ``cmd`` is assumed to be a string and passed to a + ``cmd`` is a list of byte string arguments starting with the command names. + If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a shell to execute. If the process exits with a non-zero return code diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index b1920d8ac..bce888209 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -91,8 +91,8 @@ def im_resize(maxwidth, path_in, path_out=None): # compatibility. try: util.command_output([ - 'convert', util.syspath(path_in), - '-resize', '{0}x^>'.format(maxwidth), path_out + b'convert', util.syspath(path_in), + b'-resize', b'{0}x^>'.format(maxwidth), path_out ]) except subprocess.CalledProcessError: log.warn(u'artresizer: IM convert failed for {0}', @@ -187,7 +187,7 @@ class ArtResizer(object): # Try invoking ImageMagick's "convert". try: - out = util.command_output(['identify', '--version']) + out = util.command_output([b'identify', b'--version']) if 'imagemagick' in out.lower(): pattern = r".+ (\d+)\.(\d+)\.(\d+).*" diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 35feb4d9c..d44095687 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -196,13 +196,13 @@ class EmbedCoverArtPlugin(BeetsPlugin): # Converting images to grayscale tends to minimize the weight # of colors in the diff score. convert_proc = subprocess.Popen( - ['convert', syspath(imagepath), syspath(art), - '-colorspace', 'gray', 'MIFF:-'], + [b'convert', syspath(imagepath), syspath(art), + b'-colorspace', b'gray', b'MIFF:-'], stdout=subprocess.PIPE, close_fds=not is_windows, ) compare_proc = subprocess.Popen( - ['compare', '-metric', 'PHASH', '-', 'null:'], + [b'compare', b'-metric', b'PHASH', b'-', b'null:'], stdin=convert_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index c4bebc8dd..41937d455 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -93,12 +93,12 @@ class Bs1770gainBackend(Backend): def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) - cmd = 'bs1770gain' + cmd = b'bs1770gain' try: - self.method = '--' + config['method'].get(unicode) + self.method = b'--' + config['method'].get(str) except: - self.method = '--replaygain' + self.method = b'--replaygain' try: call([cmd, self.method]) @@ -210,9 +210,9 @@ class CommandBackend(Backend): ) else: # Check whether the program is in $PATH. - for cmd in ('mp3gain', 'aacgain'): + for cmd in (b'mp3gain', b'aacgain'): try: - call([cmd, '-v']) + call([cmd, b'-v']) self.command = cmd except OSError: pass @@ -276,14 +276,14 @@ class CommandBackend(Backend): # tag-writing; this turns the mp3gain/aacgain tool into a gain # calculator rather than a tag manipulator because we take care # of changing tags ourselves. - cmd = [self.command, '-o', '-s', 's'] + cmd = [self.command, b'-o', b'-s', b's'] if self.noclip: # Adjust to avoid clipping. - cmd = cmd + ['-k'] + cmd = cmd + [b'-k'] else: # Disable clipping warning. - cmd = cmd + ['-c'] - cmd = cmd + ['-d', bytes(self.gain_offset)] + cmd = cmd + [b'-c'] + cmd = cmd + [b'-d', bytes(self.gain_offset)] cmd = cmd + [syspath(i.path) for i in items] self._log.debug(u'analyzing {0} files', len(items)) From 5a355201d3de084281c1e3f00f7b0e6650b46557 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 4 Mar 2015 15:29:19 +0100 Subject: [PATCH 074/129] =?UTF-8?q?test=5Freplaygain:=20fix=20except=20a,?= =?UTF-8?q?=20b:=20=E2=86=92=20except=20(a,=20b):?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 2c623725c..fe1bdfea5 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -25,7 +25,7 @@ try: import gi gi.require_version('Gst', '1.0') GST_AVAILABLE = True -except ImportError, ValueError: +except (ImportError, ValueError): GST_AVAILABLE = False if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): From f14f47f059459118a5d3f9123c5f72a948ac3a4c Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 4 Mar 2015 16:48:14 +0100 Subject: [PATCH 075/129] Renamed list_format_* into format_* --- beets/config_default.yaml | 4 ++-- beets/library.py | 4 ++-- beets/ui/__init__.py | 11 +++++++++++ docs/changelog.rst | 3 +++ docs/plugins/duplicates.rst | 2 +- docs/plugins/mbsync.rst | 4 ++-- docs/plugins/missing.rst | 2 +- docs/reference/config.rst | 26 ++++++++++++++++++++++---- test/test_library.py | 4 ++-- test/test_mbsync.py | 4 ++-- 10 files changed, 48 insertions(+), 16 deletions(-) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index b98ffbfc6..e0b942d82 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -61,8 +61,8 @@ ui: action_default: turquoise action: blue -list_format_item: $artist - $album - $title -list_format_album: $albumartist - $album +format_item: $artist - $album - $title +format_album: $albumartist - $album time_format: '%Y-%m-%d %H:%M:%S' sort_album: albumartist+ album+ diff --git a/beets/library.py b/beets/library.py index 6e53352d7..333ff4238 100644 --- a/beets/library.py +++ b/beets/library.py @@ -418,7 +418,7 @@ class Item(LibModel): _sorts = {'artist': SmartArtistSort} - _format_config_key = 'list_format_item' + _format_config_key = 'format_item' @classmethod def _getters(cls): @@ -851,7 +851,7 @@ class Album(LibModel): """List of keys that are set on an album's items. """ - _format_config_key = 'list_format_album' + _format_config_key = 'format_album' @classmethod def _getters(cls): diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index ad1614bad..999067f53 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -871,6 +871,17 @@ def _configure(options): u'See documentation for more info.') config['ui']['color'].set(config['color'].get(bool)) + # Compatibility from list_format_{item,album} to format_{item,album} + for elem in ('item', 'album'): + old_key = 'list_format_{0}'.format(elem) + if config[old_key].exists(): + new_key = 'format_{0}'.format(elem) + log.warning('Warning: configuration uses "{0}" which is deprecated' + ' in favor of "{1}" now that it affects all commands. ' + 'See changelog & documentation.'.format(old_key, + new_key)) + config[new_key].set(config[old_key]) + config_path = config.user_config_path() if os.path.isfile(config_path): log.debug(u'user configuration: {0}', diff --git a/docs/changelog.rst b/docs/changelog.rst index 41266a7dc..527806a9a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -71,6 +71,9 @@ Core changes: now affect (almost) every place where objects are printed and logged. (Previously, they only controlled the :ref:`list-cmd` command and a few other scattered pieces.) :bug:`1269` +* :ref:`list_format_album` and :ref:`list_format_album` have respectively been + renamed :ref:`format_album` and :ref:`format_item`. The old names still work + but each triggers a warning message. :bug:`1271` Fixes: diff --git a/docs/plugins/duplicates.rst b/docs/plugins/duplicates.rst index 18c2a8052..ecbda69cf 100644 --- a/docs/plugins/duplicates.rst +++ b/docs/plugins/duplicates.rst @@ -61,7 +61,7 @@ file. The available options mirror the command-line options: or album. This uses the same template syntax as beets' :doc:`path formats`. The usage is inspired by, and therefore similar to, the :ref:`list ` command. - Default: :ref:`list_format_item` + Default: :ref:`format_item` - **full**: List every track or album that has duplicates, not just the duplicates themselves. Default: ``no``. diff --git a/docs/plugins/mbsync.rst b/docs/plugins/mbsync.rst index cf89974e4..a7633a500 100644 --- a/docs/plugins/mbsync.rst +++ b/docs/plugins/mbsync.rst @@ -34,5 +34,5 @@ The command has a few command-line options: plugin will write new metadata to files' tags. To disable this, use the ``-W`` (``--nowrite``) option. * To customize the output of unrecognized items, use the ``-f`` - (``--format``) option. The default output is ``list_format_item`` or - ``list_format_album`` for items and albums, respectively. + (``--format``) option. The default output is ``format_item`` or + ``format_album`` for items and albums, respectively. diff --git a/docs/plugins/missing.rst b/docs/plugins/missing.rst index ffca94052..aab04e71b 100644 --- a/docs/plugins/missing.rst +++ b/docs/plugins/missing.rst @@ -36,7 +36,7 @@ configuration file. The available options are: track. This uses the same template syntax as beets' :doc:`path formats `. The usage is inspired by, and therefore similar to, the :ref:`list ` command. - Default: :ref:`list_format_item`. + Default: :ref:`format_item`. - **total**: Print a single count of missing tracks in all albums. Default: ``no``. diff --git a/docs/reference/config.rst b/docs/reference/config.rst index a64e73749..b7763c0b4 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -167,20 +167,38 @@ Defaults to ``yes``. list_format_item ~~~~~~~~~~~~~~~~ -Format to use when listing *individual items* with the :ref:`list-cmd` -command and other commands that need to print out items. Defaults to -``$artist - $album - $title``. The ``-f`` command-line option overrides -this setting. +Deprecated option, replaced by :ref:`format_item`. .. _list_format_album: list_format_album ~~~~~~~~~~~~~~~~~ +Deprecated option, replaced by :ref:`format_album`. + +.. _format_item: + +format_item +~~~~~~~~~~~ + +Format to use when listing *individual items* with the :ref:`list-cmd` +command and other commands that need to print out items. Defaults to +``$artist - $album - $title``. The ``-f`` command-line option overrides +this setting. + +It used to be named :ref:`list_format_item`. + +.. _format_album: + +format_album +~~~~~~~~~~~~ + Format to use when listing *albums* with :ref:`list-cmd` and other commands. Defaults to ``$albumartist - $album``. The ``-f`` command-line option overrides this setting. +It used to be named :ref:`list_format_album`. + .. _sort_item: sort_item diff --git a/test/test_library.py b/test/test_library.py index 1a2812b61..a496ef6c0 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -972,7 +972,7 @@ class TemplateTest(_common.LibTestCase): self.assertEqual(self.i.evaluate_template('$foo'), 'baz') def test_album_and_item_format(self): - config['list_format_album'] = u'foö $foo' + config['format_album'] = u'foö $foo' album = beets.library.Album() album.foo = 'bar' album.tagada = 'togodo' @@ -981,7 +981,7 @@ class TemplateTest(_common.LibTestCase): self.assertEqual(unicode(album), u"foö bar") self.assertEqual(str(album), b"fo\xc3\xb6 bar") - config['list_format_item'] = 'bar $foo' + config['format_item'] = 'bar $foo' item = beets.library.Item() item.foo = 'bar' item.tagada = 'togodo' diff --git a/test/test_mbsync.py b/test/test_mbsync.py index fc37fe8c3..701227366 100644 --- a/test/test_mbsync.py +++ b/test/test_mbsync.py @@ -73,8 +73,8 @@ class MbsyncCliTest(unittest.TestCase, TestHelper): self.assertEqual(album.album, 'album info') def test_message_when_skipping(self): - config['list_format_item'] = '$artist - $album - $title' - config['list_format_album'] = '$albumartist - $album' + config['format_item'] = '$artist - $album - $title' + config['format_album'] = '$albumartist - $album' # Test album with no mb_albumid. # The default format for an album include $albumartist so From 6234fee67d82b84fe6cd0acb92a9420ed4da0474 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 14:48:03 +0100 Subject: [PATCH 076/129] Option parser: add common options with a method Add a new OptionParser subclass: CommonOptionsOptionParser, which provides facilities for adding --album, --path and --format options. The last one is quite versatile. Update base commands (from beets.ui.commands) to use those. --- beets/library.py | 8 ++-- beets/ui/__init__.py | 106 +++++++++++++++++++++++++++++++++++++++++-- beets/ui/commands.py | 48 ++++---------------- beetsplug/convert.py | 2 +- 4 files changed, 117 insertions(+), 47 deletions(-) diff --git a/beets/library.py b/beets/library.py index 333ff4238..05ef98f48 100644 --- a/beets/library.py +++ b/beets/library.py @@ -233,7 +233,7 @@ class LibModel(dbcore.Model): """Shared concrete functionality for Items and Albums. """ - _format_config_key = None + format_config_key = None """Config key that specifies how an instance should be formatted. """ @@ -256,7 +256,7 @@ class LibModel(dbcore.Model): def __format__(self, spec): if not spec: - spec = beets.config[self._format_config_key].get(unicode) + spec = beets.config[self.format_config_key].get(unicode) result = self.evaluate_template(spec) if isinstance(spec, bytes): # if spec is a byte string then we must return a one as well @@ -418,7 +418,7 @@ class Item(LibModel): _sorts = {'artist': SmartArtistSort} - _format_config_key = 'format_item' + format_config_key = 'format_item' @classmethod def _getters(cls): @@ -851,7 +851,7 @@ class Album(LibModel): """List of keys that are set on an album's items. """ - _format_config_key = 'format_album' + format_config_key = 'format_album' @classmethod def _getters(cls): diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 999067f53..b151f3602 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -592,6 +592,103 @@ def show_model_changes(new, old=None, fields=None, always=False): return bool(changes) +class CommonOptionsParser(optparse.OptionParser, object): + """Offers a simple way to add common formatting options. + + Options available include: + - matching albums instead of tracks: add_album_option() + - showing paths instead of items/albums: add_path_option() + - changing the format of displayed items/albums: add_format_option() + + The last one can have several behaviors: + - against a special target + - with a certain format + - autodetected target with the album option + + Each method is fully documented in the related method. + """ + def __init__(self, *args, **kwargs): + super(CommonOptionsParser, self).__init__(*args, **kwargs) + self._has_album = False + + def add_album_option(self, flags=('-a', '--album')): + """Add a -a/--album option to match albums instead of tracks. + + If used then the format option can auto-detect whether we're setting + the format for items or albums. + Sets the album property on the options extracted from the CLI. + """ + album = optparse.Option(*flags, action='store_true', + help='match albums instead of tracks') + self.add_option(album) + self._has_album = True + + def _set_format(self, option, opt_str, value, parser, target=None, + fmt=None): + """Internal callback that sets the correct format while parsing CLI + arguments. + """ + value = fmt or value and unicode(value) or '' + parser.values.format = value + if target: + config[target.format_config_key].set(value) + else: + if not self._has_album or not parser.values.album: + config[library.Item.format_config_key].set(value) + if not self._has_album or parser.values.album: + config[library.Album.format_config_key].set(value) + + def add_path_option(self, flags=('-p', '--path')): + """Add a -p/--path option to display the path instead of the default + format. + + By default this affects both items and albums. If add_album_option() + is used then the target will be autodetected. + + Sets the format property to u'$path' on the options extracted from the + CLI. + """ + path = optparse.Option(*flags, nargs=0, action='callback', + callback=self._set_format, + callback_kwargs={'fmt': '$path'}, + help='print paths for matched items or albums') + self.add_option(path) + + def add_format_option(self, flags=('-f', '--format'), target=None): + """Add -f/--format option to print some LibModel instances with a + custom format. + + `target` is optional and can be one of ``library.Item``, 'item', + ``library.Album`` and 'album'. + + Several behaviors are available: + - if `target` is given then the format is only applied to that + LibModel + - if the album option is used then the target will be autodetected + - otherwise the format is applied to both items and albums. + + Sets the format property on the options extracted from the CLI. + """ + kwargs = {} + if target: + if isinstance(target, basestring): + target = {'item': library.Item, + 'album': library.Album}[target] + kwargs['target'] = target + + opt = optparse.Option(*flags, action='callback', + callback=self._set_format, + callback_kwargs=kwargs, + help='print with custom format') + self.add_option(opt) + + def add_all_common_options(self): + """Add album, path and format options. + """ + self.add_album_option() + self.add_path_option() + self.add_format_option() + # Subcommand parsing infrastructure. # # This is a fairly generic subcommand parser for optparse. It is @@ -600,6 +697,7 @@ def show_model_changes(new, old=None, fields=None, always=False): # There you will also find a better description of the code and a more # succinct example program. + class Subcommand(object): """A subcommand of a root command-line application that may be invoked by a SubcommandOptionParser. @@ -609,10 +707,10 @@ class Subcommand(object): the subcommand; aliases are alternate names. parser is an OptionParser responsible for parsing the subcommand's options. help is a short description of the command. If no parser is - given, it defaults to a new, empty OptionParser. + given, it defaults to a new, empty CommonOptionsParser. """ self.name = name - self.parser = parser or optparse.OptionParser() + self.parser = parser or CommonOptionsParser() self.aliases = aliases self.help = help self.hide = hide @@ -635,7 +733,7 @@ class Subcommand(object): root_parser.get_prog_name().decode('utf8'), self.name) -class SubcommandsOptionParser(optparse.OptionParser, object): +class SubcommandsOptionParser(CommonOptionsParser): """A variant of OptionParser that parses subcommands and their arguments. """ @@ -924,6 +1022,8 @@ def _raw_main(args, lib=None): handling. """ parser = SubcommandsOptionParser() + parser.add_format_option(flags=('--format-item',), target=library.Item) + parser.add_format_option(flags=('--format-album',), target=library.Album) parser.add_option('-l', '--library', dest='library', help='library database file to use') parser.add_option('-d', '--directory', dest='directory', diff --git a/beets/ui/commands.py b/beets/ui/commands.py index bea5dc91c..5d29093c9 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -957,7 +957,7 @@ default_commands.append(import_cmd) # list: Query and show library contents. -def list_items(lib, query, album, fmt): +def list_items(lib, query, album, fmt=''): """Print out items in lib matching query. If album, then search for albums instead of single items. """ @@ -970,23 +970,11 @@ def list_items(lib, query, album, fmt): def list_func(lib, opts, args): - fmt = '$path' if opts.path else opts.format - list_items(lib, decargs(args), opts.album, fmt) + list_items(lib, decargs(args), opts.album) list_cmd = ui.Subcommand('list', help='query the library', aliases=('ls',)) -list_cmd.parser.add_option( - '-a', '--album', action='store_true', - help='show matching albums instead of tracks' -) -list_cmd.parser.add_option( - '-p', '--path', action='store_true', - help='print paths for matched items or albums' -) -list_cmd.parser.add_option( - '-f', '--format', action='store', - help='print with custom format', default='' -) +list_cmd.parser.add_all_common_options() list_cmd.func = list_func default_commands.append(list_cmd) @@ -1087,10 +1075,8 @@ def update_func(lib, opts, args): update_cmd = ui.Subcommand( 'update', help='update the library', aliases=('upd', 'up',) ) -update_cmd.parser.add_option( - '-a', '--album', action='store_true', - help='match albums instead of tracks' -) +update_cmd.parser.add_album_option() +update_cmd.parser.add_format_option() update_cmd.parser.add_option( '-M', '--nomove', action='store_false', default=True, dest='move', help="don't move files in library" @@ -1099,10 +1085,6 @@ update_cmd.parser.add_option( '-p', '--pretend', action='store_true', help="show all changes but do nothing" ) -update_cmd.parser.add_option( - '-f', '--format', action='store', - help='print with custom format', default='' -) update_cmd.func = update_func default_commands.append(update_cmd) @@ -1151,10 +1133,7 @@ remove_cmd.parser.add_option( "-d", "--delete", action="store_true", help="also remove files from disk" ) -remove_cmd.parser.add_option( - '-a', '--album', action='store_true', - help='match albums instead of tracks' -) +remove_cmd.parser.add_album_option() remove_cmd.func = remove_func default_commands.append(remove_cmd) @@ -1348,18 +1327,12 @@ modify_cmd.parser.add_option( '-W', '--nowrite', action='store_false', dest='write', help="don't write metadata (opposite of -w)" ) -modify_cmd.parser.add_option( - '-a', '--album', action='store_true', - help='modify whole albums instead of tracks' -) +modify_cmd.parser.add_album_option() +modify_cmd.parser.add_format_option(target='item') modify_cmd.parser.add_option( '-y', '--yes', action='store_true', help='skip confirmation' ) -modify_cmd.parser.add_option( - '-f', '--format', action='store', - help='print with custom format', default='' -) modify_cmd.func = modify_func default_commands.append(modify_cmd) @@ -1405,10 +1378,7 @@ move_cmd.parser.add_option( '-c', '--copy', default=False, action='store_true', help='copy instead of moving' ) -move_cmd.parser.add_option( - '-a', '--album', default=False, action='store_true', - help='match whole albums instead of tracks' -) +move_cmd.parser.add_album_option() move_cmd.func = move_func default_commands.append(move_cmd) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 352c3c94d..d4bc6d32b 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -359,7 +359,7 @@ class ConvertPlugin(BeetsPlugin): self.config['pretend'].get(bool) if not pretend: - ui.commands.list_items(lib, ui.decargs(args), opts.album, '') + ui.commands.list_items(lib, ui.decargs(args), opts.album) if not (opts.yes or ui.input_yn("Convert? (Y/n)")): return From 5623d26a9179edfb83be6cf91ed201e095a9f4b7 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 16:07:49 +0100 Subject: [PATCH 077/129] Add tests for the CommonOptionsParser Unit test both the features & do real behaviour tests with the 'list' command. --- test/test_ui.py | 136 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/test/test_ui.py b/test/test_ui.py index d2a2ea80e..90ee43b38 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1033,6 +1033,142 @@ class CompletionTest(_common.TestCase): self.fail('test/test_completion.sh did not execute properly') +class CommonOptionsParserCliTest(unittest.TestCase, TestHelper): + """Test CommonOptionsParser and formatting LibModel formatting on 'list' + command. + """ + def setUp(self): + self.setup_beets() + self.lib = library.Library(':memory:') + self.item = _common.item() + self.item.path = 'xxx/yyy' + self.lib.add(self.item) + self.lib.add_album([self.item]) + + def tearDown(self): + self.teardown_beets() + + def test_base(self): + l = self.run_with_output('ls') + self.assertEqual(l, 'the artist - the album - the title\n') + + l = self.run_with_output('ls', '-a') + self.assertEqual(l, 'the album artist - the album\n') + + def test_path_option(self): + l = self.run_with_output('ls', '-p') + self.assertEqual(l, 'xxx/yyy\n') + + l = self.run_with_output('ls', '-a', '-p') + self.assertEqual(l, 'xxx\n') + + def test_format_option(self): + l = self.run_with_output('ls', '-f', '$artist') + self.assertEqual(l, 'the artist\n') + + l = self.run_with_output('ls', '-a', '-f', '$albumartist') + self.assertEqual(l, 'the album artist\n') + + def test_root_format_option(self): + l = self.run_with_output('--format-item', '$artist', + '--format-album', 'foo', 'ls') + self.assertEqual(l, 'the artist\n') + + l = self.run_with_output('--format-item', 'foo', + '--format-album', '$albumartist', 'ls', '-a') + self.assertEqual(l, 'the album artist\n') + + +class CommonOptionsParserTest(unittest.TestCase, TestHelper): + def setUp(self): + self.setup_beets() + + def tearDown(self): + self.teardown_beets() + + def test_album_option(self): + parser = ui.CommonOptionsParser() + self.assertFalse(parser._has_album) + parser.add_album_option() + self.assertTrue(parser._has_album) + + self.assertEqual(parser.parse_args([]), ({'album': None}, [])) + self.assertEqual(parser.parse_args(['-a']), ({'album': True}, [])) + self.assertEqual(parser.parse_args(['--album']), ({'album': True}, [])) + + def test_path_option(self): + parser = ui.CommonOptionsParser() + parser.add_path_option() + self.assertFalse(parser._has_album) + + config['format_item'].set('$foo') + self.assertEqual(parser.parse_args([]), ({'path': None}, [])) + self.assertEqual(config['format_item'].get(unicode), u'$foo') + + self.assertEqual(parser.parse_args(['-p']), + ({'path': None, 'format': '$path'}, [])) + self.assertEqual(parser.parse_args(['--path']), + ({'path': None, 'format': '$path'}, [])) + + self.assertEqual(config['format_item'].get(unicode), '$path') + self.assertEqual(config['format_album'].get(unicode), '$path') + + def test_format_option(self): + parser = ui.CommonOptionsParser() + parser.add_format_option() + self.assertFalse(parser._has_album) + + config['format_item'].set('$foo') + self.assertEqual(parser.parse_args([]), ({'format': None}, [])) + self.assertEqual(config['format_item'].get(unicode), u'$foo') + + self.assertEqual(parser.parse_args(['-f', '$bar']), + ({'format': '$bar'}, [])) + self.assertEqual(parser.parse_args(['--format', '$baz']), + ({'format': '$baz'}, [])) + + self.assertEqual(config['format_item'].get(unicode), '$baz') + self.assertEqual(config['format_album'].get(unicode), '$baz') + + def test_format_option_with_target(self): + with self.assertRaises(KeyError): + ui.CommonOptionsParser().add_format_option(target='thingy') + + parser = ui.CommonOptionsParser() + parser.add_format_option(target='item') + + config['format_item'].set('$item') + config['format_album'].set('$album') + + self.assertEqual(parser.parse_args(['-f', '$bar']), + ({'format': '$bar'}, [])) + + self.assertEqual(config['format_item'].get(unicode), '$bar') + self.assertEqual(config['format_album'].get(unicode), '$album') + + def test_format_option_with_album(self): + parser = ui.CommonOptionsParser() + parser.add_album_option() + parser.add_format_option() + + config['format_item'].set('$item') + config['format_album'].set('$album') + + parser.parse_args(['-f', '$bar']) + self.assertEqual(config['format_item'].get(unicode), '$bar') + self.assertEqual(config['format_album'].get(unicode), '$album') + + parser.parse_args(['-a', '-f', '$foo']) + self.assertEqual(config['format_item'].get(unicode), '$bar') + self.assertEqual(config['format_album'].get(unicode), '$foo') + + def test_add_all_common_options(self): + parser = ui.CommonOptionsParser() + parser.add_all_common_options() + self.assertEqual(parser.parse_args([]), + ({'album': None, 'path': None, 'format': None}, [])) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 38ca99498d7f23294130005c2d1a93a0f1824f87 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 16:20:07 +0100 Subject: [PATCH 078/129] Bypass sequential args. parsing for --album When setting format and using --album we *need* to know whether we're in album mode. Naively if --album happens after "--format fmt" then we'll set Item format instead of Album format. By looking forward for -a/--album we bypass that problem. --- beets/ui/__init__.py | 20 ++++++++++++++++---- test/test_ui.py | 11 +++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index b151f3602..dfa7f2e09 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -609,7 +609,10 @@ class CommonOptionsParser(optparse.OptionParser, object): """ def __init__(self, *args, **kwargs): super(CommonOptionsParser, self).__init__(*args, **kwargs) - self._has_album = False + self._album_flags = False + # this serves both as an indicator that we offer the feature AND allows + # us to check whether it has been specified on the CLI - bypassing the + # fact that arguments may be in any order def add_album_option(self, flags=('-a', '--album')): """Add a -a/--album option to match albums instead of tracks. @@ -621,7 +624,7 @@ class CommonOptionsParser(optparse.OptionParser, object): album = optparse.Option(*flags, action='store_true', help='match albums instead of tracks') self.add_option(album) - self._has_album = True + self._album_flags = set(flags) def _set_format(self, option, opt_str, value, parser, target=None, fmt=None): @@ -633,9 +636,18 @@ class CommonOptionsParser(optparse.OptionParser, object): if target: config[target.format_config_key].set(value) else: - if not self._has_album or not parser.values.album: + if self._album_flags: + if parser.values.album: + target = library.Album + else: + # the option is either missing either not parsed yet + if self._album_flags & set(parser.rargs): + target = library.Album + else: + target = library.Item + config[target.format_config_key].set(value) + else: config[library.Item.format_config_key].set(value) - if not self._has_album or parser.values.album: config[library.Album.format_config_key].set(value) def add_path_option(self, flags=('-p', '--path')): diff --git a/test/test_ui.py b/test/test_ui.py index 90ee43b38..75905dafb 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1088,9 +1088,9 @@ class CommonOptionsParserTest(unittest.TestCase, TestHelper): def test_album_option(self): parser = ui.CommonOptionsParser() - self.assertFalse(parser._has_album) + self.assertFalse(parser._album_flags) parser.add_album_option() - self.assertTrue(parser._has_album) + self.assertTrue(bool(parser._album_flags)) self.assertEqual(parser.parse_args([]), ({'album': None}, [])) self.assertEqual(parser.parse_args(['-a']), ({'album': True}, [])) @@ -1099,7 +1099,7 @@ class CommonOptionsParserTest(unittest.TestCase, TestHelper): def test_path_option(self): parser = ui.CommonOptionsParser() parser.add_path_option() - self.assertFalse(parser._has_album) + self.assertFalse(parser._album_flags) config['format_item'].set('$foo') self.assertEqual(parser.parse_args([]), ({'path': None}, [])) @@ -1116,7 +1116,7 @@ class CommonOptionsParserTest(unittest.TestCase, TestHelper): def test_format_option(self): parser = ui.CommonOptionsParser() parser.add_format_option() - self.assertFalse(parser._has_album) + self.assertFalse(parser._album_flags) config['format_item'].set('$foo') self.assertEqual(parser.parse_args([]), ({'format': None}, [])) @@ -1162,6 +1162,9 @@ class CommonOptionsParserTest(unittest.TestCase, TestHelper): self.assertEqual(config['format_item'].get(unicode), '$bar') self.assertEqual(config['format_album'].get(unicode), '$foo') + parser.parse_args(['-f', '$foo2', '-a']) + self.assertEqual(config['format_album'].get(unicode), '$foo2') + def test_add_all_common_options(self): parser = ui.CommonOptionsParser() parser.add_all_common_options() From 650305c9a184320df1bcfe6ae5e60d4f5a7ad283 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 17:00:19 +0100 Subject: [PATCH 079/129] All suitable plugins use CommonOptionsParser features --- beetsplug/convert.py | 6 ++---- beetsplug/duplicates.py | 17 +---------------- beetsplug/echonest.py | 5 +---- beetsplug/mbsync.py | 16 +++++++--------- beetsplug/missing.py | 8 +------- beetsplug/play.py | 6 +----- beetsplug/random.py | 7 +------ beetsplug/replaygain.py | 3 +-- test/test_mbsync.py | 4 ++++ 9 files changed, 19 insertions(+), 53 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index d4bc6d32b..11ea1f5ce 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -139,8 +139,6 @@ class ConvertPlugin(BeetsPlugin): cmd = ui.Subcommand('convert', help='convert to external location') cmd.parser.add_option('-p', '--pretend', action='store_true', help='show actions but do nothing') - cmd.parser.add_option('-a', '--album', action='store_true', - help='choose albums instead of tracks') cmd.parser.add_option('-t', '--threads', action='store', type='int', help='change the number of threads, \ defaults to maximum available processors') @@ -149,10 +147,10 @@ class ConvertPlugin(BeetsPlugin): and move the old files') cmd.parser.add_option('-d', '--dest', action='store', help='set the destination directory') - cmd.parser.add_option('-f', '--format', action='store', dest='format', - help='set the destination directory') cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes', help='do not ask for confirmation') + cmd.parser.add_album_option() + cmd.parser.add_format_option(target='item') cmd.func = self.convert_func return [cmd] diff --git a/beetsplug/duplicates.py b/beetsplug/duplicates.py index fb697922b..1739d3530 100644 --- a/beetsplug/duplicates.py +++ b/beetsplug/duplicates.py @@ -125,17 +125,6 @@ class DuplicatesPlugin(BeetsPlugin): self._command = Subcommand('duplicates', help=__doc__, aliases=['dup']) - - self._command.parser.add_option('-f', '--format', dest='format', - action='store', type='string', - help='print with custom format', - metavar='FMT', default='') - - self._command.parser.add_option('-a', '--album', dest='album', - action='store_true', - help='show duplicate albums instead of' - ' tracks') - self._command.parser.add_option('-c', '--count', dest='count', action='store_true', help='show duplicate counts') @@ -168,15 +157,11 @@ class DuplicatesPlugin(BeetsPlugin): action='store', metavar='DEST', help='copy items to dest') - self._command.parser.add_option('-p', '--path', dest='path', - action='store_true', - help='print paths for matched items or' - ' albums') - self._command.parser.add_option('-t', '--tag', dest='tag', action='store', help='tag matched items with \'k=v\'' ' attribute') + self._command.parser.add_all_common_options() def commands(self): diff --git a/beetsplug/echonest.py b/beetsplug/echonest.py index c8d45f3b0..753bfb5c3 100644 --- a/beetsplug/echonest.py +++ b/beetsplug/echonest.py @@ -486,10 +486,7 @@ class EchonestMetadataPlugin(plugins.BeetsPlugin): '-t', '--threshold', dest='threshold', action='store', type='float', default=0.15, help='Set difference threshold' ) - sim_cmd.parser.add_option( - '-f', '--format', action='store', default='${difference}: ${path}', - help='print with custom format' - ) + sim_cmd.parser.add_format_option() def sim_func(lib, opts, args): self.config.set_args(opts) diff --git a/beetsplug/mbsync.py b/beetsplug/mbsync.py index 365d83193..974f7e894 100644 --- a/beetsplug/mbsync.py +++ b/beetsplug/mbsync.py @@ -52,8 +52,7 @@ class MBSyncPlugin(BeetsPlugin): cmd.parser.add_option('-W', '--nowrite', action='store_false', default=config['import']['write'], dest='write', help="don't write updated metadata to files") - cmd.parser.add_option('-f', '--format', action='store', default='', - help='print with custom format') + cmd.parser.add_format_option() cmd.func = self.func return [cmd] @@ -64,17 +63,16 @@ class MBSyncPlugin(BeetsPlugin): pretend = opts.pretend write = opts.write query = ui.decargs(args) - fmt = opts.format - self.singletons(lib, query, move, pretend, write, fmt) - self.albums(lib, query, move, pretend, write, fmt) + self.singletons(lib, query, move, pretend, write) + self.albums(lib, query, move, pretend, write) - def singletons(self, lib, query, move, pretend, write, fmt): + def singletons(self, lib, query, move, pretend, write): """Retrieve and apply info from the autotagger for items matched by query. """ for item in lib.items(query + ['singleton:true']): - item_formatted = format(item, fmt) + item_formatted = format(item) if not item.mb_trackid: self._log.info(u'Skipping singleton with no mb_trackid: {0}', item_formatted) @@ -93,13 +91,13 @@ class MBSyncPlugin(BeetsPlugin): autotag.apply_item_metadata(item, track_info) apply_item_changes(lib, item, move, pretend, write) - def albums(self, lib, query, move, pretend, write, fmt): + def albums(self, lib, query, move, pretend, write): """Retrieve and apply info from the autotagger for albums matched by query and their items. """ # Process matching albums. for a in lib.albums(query): - album_formatted = format(a, fmt) + album_formatted = format(a) if not a.mb_albumid: self._log.info(u'Skipping album with no mb_albumid: {0}', album_formatted) diff --git a/beetsplug/missing.py b/beetsplug/missing.py index 517a20758..7f976f75c 100644 --- a/beetsplug/missing.py +++ b/beetsplug/missing.py @@ -94,19 +94,13 @@ class MissingPlugin(BeetsPlugin): self._command = Subcommand('missing', help=__doc__, aliases=['miss']) - - self._command.parser.add_option('-f', '--format', dest='format', - action='store', type='string', - help='print with custom FORMAT', - metavar='FORMAT', default='') - self._command.parser.add_option('-c', '--count', dest='count', action='store_true', help='count missing tracks per album') - self._command.parser.add_option('-t', '--total', dest='total', action='store_true', help='count total of missing tracks') + self._command.add_format_option() def commands(self): def _miss(lib, opts, args): diff --git a/beetsplug/play.py b/beetsplug/play.py index 8a912505c..65d5b91ec 100644 --- a/beetsplug/play.py +++ b/beetsplug/play.py @@ -42,11 +42,7 @@ class PlayPlugin(BeetsPlugin): 'play', help='send music to a player as a playlist' ) - play_command.parser.add_option( - '-a', '--album', - action='store_true', default=False, - help='query and load albums rather than tracks' - ) + play_command.parser.add_album_option() play_command.func = self.play_music return [play_command] diff --git a/beetsplug/random.py b/beetsplug/random.py index f6a664a4c..180e954f0 100644 --- a/beetsplug/random.py +++ b/beetsplug/random.py @@ -67,16 +67,11 @@ def random_item(lib, opts, args): random_cmd = Subcommand('random', help='chose a random track or album') -random_cmd.parser.add_option('-a', '--album', action='store_true', - help='choose an album instead of track') -random_cmd.parser.add_option('-p', '--path', action='store_true', - help='print the path of the matched item') -random_cmd.parser.add_option('-f', '--format', action='store', - help='print with custom format', default='') random_cmd.parser.add_option('-n', '--number', action='store', type="int", help='number of objects to choose', default=1) random_cmd.parser.add_option('-e', '--equal-chance', action='store_true', help='each artist has the same chance') +random_cmd.parser.add_all_common_options() random_cmd.func = random_item diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ce41cad57..d5b6ae8d4 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -760,7 +760,6 @@ class ReplayGainPlugin(BeetsPlugin): self.handle_track(item, write) cmd = ui.Subcommand('replaygain', help='analyze for ReplayGain') - cmd.parser.add_option('-a', '--album', action='store_true', - help='analyze albums instead of tracks') + cmd.parser.add_album_option() cmd.func = func return [cmd] diff --git a/test/test_mbsync.py b/test/test_mbsync.py index 701227366..ff6e01cf3 100644 --- a/test/test_mbsync.py +++ b/test/test_mbsync.py @@ -99,6 +99,10 @@ class MbsyncCliTest(unittest.TestCase, TestHelper): e = "mbsync: Skipping album with no mb_albumid: 'album info'" self.assertEqual(e, logs[0]) + # restore the config + config['format_item'] = '$artist - $album - $title' + config['format_album'] = '$albumartist - $album' + # Test singleton with no mb_trackid. # The default singleton format includes $artist and $album # so we need to stub them here From ea1dc1eb1921302a7be3cfcfa6e2c1569150f450 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 17:24:57 +0100 Subject: [PATCH 080/129] Plugins conversion cont. --- beetsplug/random.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/random.py b/beetsplug/random.py index 180e954f0..43fb7957c 100644 --- a/beetsplug/random.py +++ b/beetsplug/random.py @@ -26,7 +26,6 @@ from itertools import groupby def random_item(lib, opts, args): query = decargs(args) - fmt = '$path' if opts.path else opts.format if opts.album: objs = list(lib.albums(query)) @@ -63,7 +62,7 @@ def random_item(lib, opts, args): objs = random.sample(objs, number) for item in objs: - print_(format(item, fmt)) + print_(format(item)) random_cmd = Subcommand('random', help='chose a random track or album') From 7798a521b58ac4525f3e121fabca3bd1d6317ac8 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 17:39:56 +0100 Subject: [PATCH 081/129] Fix convert plugin --- beetsplug/convert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 11ea1f5ce..0cce93f7a 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -146,11 +146,12 @@ class ConvertPlugin(BeetsPlugin): dest='keep_new', help='keep only the converted \ and move the old files') cmd.parser.add_option('-d', '--dest', action='store', + help='set the target format of the tracks') + cmd.parser.add_option('-f', '--format', action='store', dest='format', help='set the destination directory') cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes', help='do not ask for confirmation') cmd.parser.add_album_option() - cmd.parser.add_format_option(target='item') cmd.func = self.convert_func return [cmd] From 6fda0e23fc0ab4c16c9a0050eb85c26d4e9e74f2 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 17:13:33 +0100 Subject: [PATCH 082/129] Update docs & changelog --- docs/changelog.rst | 6 ++++++ docs/dev/plugins.rst | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 527806a9a..4da0bb141 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog Features: +* Beets now accept top-level options ``--format-item`` and ``--format-album`` + before any subcommand to control how items and albums are displayed. + :bug:`1271`: * There are now multiple levels of verbosity. On the command line, you can make beets somewhat verbose with ``-v`` or very verbose with ``-vv``. :bug:`1244` @@ -120,6 +123,9 @@ Fixes: For developers: +* the ``OptionParser`` is now a ``CommonOptionsParser`` that offers facilities + for adding usual options (``--album``, ``--path`` and ``--format``). See + :ref:`add_subcommands`. :bug:`1271` * The logging system in beets has been overhauled. Plugins now each have their own logger, which helps by automatically adjusting the verbosity level in import mode and by prefixing the plugin's name. Logging levels are diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index f448b5dfa..9961ad03c 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -87,8 +87,11 @@ The function should use any of the utility functions defined in ``beets.ui``. Try running ``pydoc beets.ui`` to see what's available. You can add command-line options to your new command using the ``parser`` member -of the ``Subcommand`` class, which is an ``OptionParser`` instance. Just use it -like you would a normal ``OptionParser`` in an independent script. +of the ``Subcommand`` class, which is a ``CommonOptionParser`` instance. Just +use it like you would a normal ``OptionParser`` in an independent script. Note +that it offers several methods to add common options: ``--album``, ``--path`` +and ``--format``. This feature is versatile and extensively documented, try +``pydoc beets.ui.CommonOptionParser`` for more information. .. _plugin_events: From 167f067961d1fe7ca755225e52cda11642351729 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 5 Mar 2015 17:23:55 +0100 Subject: [PATCH 083/129] Improve behavior of --path: store_true-like Availability of the 'path' presence in arguments can be important for some plugins such as duplicates, and therefore should be conserved. --- beets/ui/__init__.py | 8 ++++++-- test/test_ui.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index dfa7f2e09..85c5e4824 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -627,10 +627,13 @@ class CommonOptionsParser(optparse.OptionParser, object): self._album_flags = set(flags) def _set_format(self, option, opt_str, value, parser, target=None, - fmt=None): + fmt=None, store_true=False): """Internal callback that sets the correct format while parsing CLI arguments. """ + if store_true: + setattr(parser.values, option.dest, True) + value = fmt or value and unicode(value) or '' parser.values.format = value if target: @@ -662,7 +665,8 @@ class CommonOptionsParser(optparse.OptionParser, object): """ path = optparse.Option(*flags, nargs=0, action='callback', callback=self._set_format, - callback_kwargs={'fmt': '$path'}, + callback_kwargs={'fmt': '$path', + 'store_true': True}, help='print paths for matched items or albums') self.add_option(path) diff --git a/test/test_ui.py b/test/test_ui.py index 75905dafb..0f8bb6306 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1106,9 +1106,9 @@ class CommonOptionsParserTest(unittest.TestCase, TestHelper): self.assertEqual(config['format_item'].get(unicode), u'$foo') self.assertEqual(parser.parse_args(['-p']), - ({'path': None, 'format': '$path'}, [])) + ({'path': True, 'format': '$path'}, [])) self.assertEqual(parser.parse_args(['--path']), - ({'path': None, 'format': '$path'}, [])) + ({'path': True, 'format': '$path'}, [])) self.assertEqual(config['format_item'].get(unicode), '$path') self.assertEqual(config['format_album'].get(unicode), '$path') From d1f6bbaf01269faca6b5868781c03b4e6b1e987a Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Fri, 6 Mar 2015 11:16:20 +0100 Subject: [PATCH 084/129] Discogs plugin: catch client lib ValueErrors When search fails, log the query which caused it and return an empty result. Kind of fixes #1347. --- beetsplug/discogs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index ac0f516ff..0ea40caee 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -192,7 +192,13 @@ class DiscogsPlugin(BeetsPlugin): # Strip medium information from query, Things like "CD1" and "disk 1" # can also negate an otherwise positive result. query = re.sub(r'(?i)\b(CD|disc)\s*\d+', '', query) - releases = self.discogs_client.search(query, type='release').page(1) + try: + releases = self.discogs_client.search(query, + type='release').page(1) + except ValueError as exc: + self._log.debug("Problem whiile searching for {0!r}: " + "{1}".format(query, exc)) + return [] return [self.get_album_info(release) for release in releases[:5]] def get_album_info(self, result): From 0d70db39660132d09564736bf3894381c79a769a Mon Sep 17 00:00:00 2001 From: jmwatte Date: Fri, 6 Mar 2015 20:36:06 +0100 Subject: [PATCH 085/129] Update replaygain.rst --- docs/plugins/replaygain.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index f08a23953..955d46109 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -83,7 +83,7 @@ In order to use this backend, you will need to install the bs1770gain command-li * goto `bs1770gain`_ and follow the download instructions * make sure it is in your $PATH -.. _bs1770gain: bs1770gain.sourceforge.net +.. _bs1770gain:http://bs1770gain.sourceforge.net Then, enable the plugin (see :ref:`using-plugins`) and specify the backend in your configuration file:: From bcc591bf978e4966749d72408d0f4c0c7a1b1880 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 6 Mar 2015 12:01:09 -0800 Subject: [PATCH 086/129] Generalize exception handler for #1347 --- beetsplug/discogs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 0ea40caee..a7e67b922 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -195,8 +195,8 @@ class DiscogsPlugin(BeetsPlugin): try: releases = self.discogs_client.search(query, type='release').page(1) - except ValueError as exc: - self._log.debug("Problem whiile searching for {0!r}: " + except CONNECTION_ERRORS as exc: + self._log.debug("Communication error while searching for {0!r}: " "{1}".format(query, exc)) return [] return [self.get_album_info(release) for release in releases[:5]] From da4a5d079782238d88d01653be74ee195cb82776 Mon Sep 17 00:00:00 2001 From: jmwatte Date: Fri, 6 Mar 2015 22:28:59 +0100 Subject: [PATCH 087/129] fix bs1770gain link --- docs/plugins/replaygain.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index cbb0de129..3411d3d40 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -83,7 +83,7 @@ In order to use this backend, you will need to install the bs1770gain command-li * goto `bs1770gain`_ and follow the download instructions * make sure it is in your $PATH -.. _bs1770gain:http://bs1770gain.sourceforge.net +.. _bs1770gain: http://bs1770gain.sourceforge.net/ Then, enable the plugin (see :ref:`using-plugins`) and specify the backend in your configuration file:: From 679b0a586bc82d7ce3d96c7628e97195294f00cd Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sat, 7 Mar 2015 13:45:58 +0100 Subject: [PATCH 088/129] Remove list_format_{album,item} sections from docs --- docs/changelog.rst | 4 ++-- docs/reference/config.rst | 19 ++++--------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9af5d0a34..693c4b1d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -72,11 +72,11 @@ Core changes: ``albumtotal`` computed attribute that provides the total number of tracks on the album. (The :ref:`per_disc_numbering` option has no influence on this field.) -* The :ref:`list_format_album` and :ref:`list_format_item` configuration keys +* The `list_format_album` and `list_format_item` configuration keys now affect (almost) every place where objects are printed and logged. (Previously, they only controlled the :ref:`list-cmd` command and a few other scattered pieces.) :bug:`1269` -* :ref:`list_format_album` and :ref:`list_format_album` have respectively been +* `list_format_album` and `list_format_album` have respectively been renamed :ref:`format_album` and :ref:`format_item`. The old names still work but each triggers a warning message. :bug:`1271` diff --git a/docs/reference/config.rst b/docs/reference/config.rst index b7763c0b4..fc686a8b8 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -162,20 +162,8 @@ Either ``yes`` or ``no``, indicating whether the autotagger should use multiple threads. This makes things faster but may behave strangely. Defaults to ``yes``. + .. _list_format_item: - -list_format_item -~~~~~~~~~~~~~~~~ - -Deprecated option, replaced by :ref:`format_item`. - -.. _list_format_album: - -list_format_album -~~~~~~~~~~~~~~~~~ - -Deprecated option, replaced by :ref:`format_album`. - .. _format_item: format_item @@ -186,8 +174,9 @@ command and other commands that need to print out items. Defaults to ``$artist - $album - $title``. The ``-f`` command-line option overrides this setting. -It used to be named :ref:`list_format_item`. +It used to be named `list_format_item`. +.. _list_format_album: .. _format_album: format_album @@ -197,7 +186,7 @@ Format to use when listing *albums* with :ref:`list-cmd` and other commands. Defaults to ``$albumartist - $album``. The ``-f`` command-line option overrides this setting. -It used to be named :ref:`list_format_album`. +It used to be named `list_format_album`. .. _sort_item: From e789b04c9a201e6672bb8274f09df498d3dc4335 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sat, 7 Mar 2015 13:51:00 +0100 Subject: [PATCH 089/129] =?UTF-8?q?Rename=20LibModel.format=5Fconfig=5Fkey?= =?UTF-8?q?=20=E2=86=92=20=5Fformat=5Fconfig=5Fkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #1346 --- beets/library.py | 8 ++++---- beets/ui/__init__.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beets/library.py b/beets/library.py index bd88136f1..139cdfec0 100644 --- a/beets/library.py +++ b/beets/library.py @@ -259,7 +259,7 @@ class LibModel(dbcore.Model): """Shared concrete functionality for Items and Albums. """ - format_config_key = None + _format_config_key = None """Config key that specifies how an instance should be formatted. """ @@ -282,7 +282,7 @@ class LibModel(dbcore.Model): def __format__(self, spec): if not spec: - spec = beets.config[self.format_config_key].get(unicode) + spec = beets.config[self._format_config_key].get(unicode) result = self.evaluate_template(spec) if isinstance(spec, bytes): # if spec is a byte string then we must return a one as well @@ -444,7 +444,7 @@ class Item(LibModel): _sorts = {'artist': SmartArtistSort} - format_config_key = 'format_item' + _format_config_key = 'format_item' @classmethod def _getters(cls): @@ -877,7 +877,7 @@ class Album(LibModel): """List of keys that are set on an album's items. """ - format_config_key = 'format_album' + _format_config_key = 'format_album' @classmethod def _getters(cls): diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 85c5e4824..335129af2 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -637,7 +637,7 @@ class CommonOptionsParser(optparse.OptionParser, object): value = fmt or value and unicode(value) or '' parser.values.format = value if target: - config[target.format_config_key].set(value) + config[target._format_config_key].set(value) else: if self._album_flags: if parser.values.album: @@ -648,10 +648,10 @@ class CommonOptionsParser(optparse.OptionParser, object): target = library.Album else: target = library.Item - config[target.format_config_key].set(value) + config[target._format_config_key].set(value) else: - config[library.Item.format_config_key].set(value) - config[library.Album.format_config_key].set(value) + config[library.Item._format_config_key].set(value) + config[library.Album._format_config_key].set(value) def add_path_option(self, flags=('-p', '--path')): """Add a -p/--path option to display the path instead of the default From 1722c26ffe9ea0b24a9257cf795cc7e7b71d28e0 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sat, 7 Mar 2015 14:00:17 +0100 Subject: [PATCH 090/129] Fix DiscogsPlugin.setup() arguments Session is optional. This fixes re-authorization. Improve #1347. --- beetsplug/discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index a7e67b922..1f0070d93 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -59,7 +59,7 @@ class DiscogsPlugin(BeetsPlugin): self.discogs_client = None self.register_listener('import_begin', self.setup) - def setup(self, session): + def setup(self, session=None): """Create the `discogs_client` field. Authenticate if necessary. """ c_key = self.config['apikey'].get(unicode) From ee6fba1e821792112ddaf9c891b05572c1012fc3 Mon Sep 17 00:00:00 2001 From: David Logie Date: Sun, 8 Mar 2015 12:59:00 +0000 Subject: [PATCH 091/129] Fix call to add_format_option() missing plugin. --- beetsplug/missing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/missing.py b/beetsplug/missing.py index 7f976f75c..816449cc4 100644 --- a/beetsplug/missing.py +++ b/beetsplug/missing.py @@ -100,7 +100,7 @@ class MissingPlugin(BeetsPlugin): self._command.parser.add_option('-t', '--total', dest='total', action='store_true', help='count total of missing tracks') - self._command.add_format_option() + self._command.parser.add_format_option() def commands(self): def _miss(lib, opts, args): From e71caded81851c585dcfd2326f8aa4342d9fbd8b Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 11 Mar 2015 11:08:40 +0100 Subject: [PATCH 092/129] =?UTF-8?q?Update=20docs'=20html=5Ftheme=20value:?= =?UTF-8?q?=20default=20=E2=86=92=20classic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New proposed default is 'alabaster', which looks nice but leaves less room to the core content. 'classic' replaces 'default'. Anyway readthedocs.org applies its own theme so this only impacts local builds. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 82fc15da8..4aeb66d33 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ extlinks = { } # Options for HTML output -html_theme = 'default' +html_theme = 'classic' htmlhelp_basename = 'beetsdoc' # Options for LaTeX output From 295ebc0bc672ff333a3e984a9ea8e83c818457c2 Mon Sep 17 00:00:00 2001 From: guibog Date: Wed, 11 Mar 2015 15:52:00 +0800 Subject: [PATCH 093/129] Add debug log for plugin paths --- beets/ui/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 335129af2..35ee69e01 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -921,6 +921,7 @@ def _load_plugins(config): """ paths = config['pluginpath'].get(confit.StrSeq(split=False)) paths = map(util.normpath, paths) + log.debug('plugin paths: {0}', util.displayable_path(paths)) import beetsplug beetsplug.__path__ = paths + beetsplug.__path__ From 02855a44bd9f591eb1d1406b59975b6ff02fc212 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Fri, 13 Mar 2015 11:51:26 +0100 Subject: [PATCH 094/129] Fix exception construction in util.command_output() `cmd` being a byte string array, it should be joined by b" " and not u" ". --- beets/util/__init__.py | 2 +- test/test_util.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 53156143f..a3b57eea6 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -664,7 +664,7 @@ def command_output(cmd, shell=False): if proc.returncode: raise subprocess.CalledProcessError( returncode=proc.returncode, - cmd=' '.join(cmd), + cmd=b' '.join(cmd), ) return stdout diff --git a/test/test_util.py b/test/test_util.py index 20c7708d5..3de8bfffc 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -19,8 +19,9 @@ from __future__ import (division, absolute_import, print_function, import sys import re import os +import subprocess -from mock import patch +from mock import patch, Mock from test._common import unittest from test import _common @@ -102,6 +103,19 @@ class UtilTest(unittest.TestCase): ]) self.assertEqual(p, u'foo/_/bar') + @patch('beets.util.subprocess.Popen') + def test_command_output(self, mock_popen): + def popen_fail(*args, **kwargs): + m = Mock(returncode=1) + m.communicate.return_value = None, None + return m + + mock_popen.side_effect = popen_fail + with self.assertRaises(subprocess.CalledProcessError) as exc_context: + util.command_output([b"taga", b"\xc3\xa9"]) + self.assertEquals(exc_context.exception.returncode, 1) + self.assertEquals(exc_context.exception.cmd, b"taga \xc3\xa9") + class PathConversionTest(_common.TestCase): def test_syspath_windows_format(self): From 96a565a61489ea03b418ec6e8cfda316d96f73ba Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Fri, 13 Mar 2015 17:55:09 +0100 Subject: [PATCH 095/129] info: error msg instead of traceback on IOError IOErrors were previously not caught, displaying full traceback to the user --- beetsplug/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/info.py b/beetsplug/info.py index 7a5a47b84..82bcbe965 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -145,7 +145,7 @@ class InfoPlugin(BeetsPlugin): for data_emitter in data_collector(lib, ui.decargs(args)): try: data = data_emitter() - except mediafile.UnreadableFileError as ex: + except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue From 736eab412c5352de26193037d51f1fbe9de3b0c3 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 14 Mar 2015 14:43:00 -0700 Subject: [PATCH 096/129] Bytes arguments to bs1770gain command --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 11f8c794c..5159816d4 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -148,7 +148,7 @@ class Bs1770gainBackend(Backend): # Construct shell command. cmd = [self.command] cmd = cmd + [self.method] - cmd = cmd + ['-it'] + cmd = cmd + [b'-it'] cmd = cmd + [syspath(i.path) for i in items] self._log.debug(u'analyzing {0} files', len(items)) From f6df14a7982eb0af913fbe4bdf28ed0c7d8ee233 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 14 Mar 2015 15:13:07 -0700 Subject: [PATCH 097/129] Isolate bugs in pylast Should fix crashes like this: http://hastebin.com/nizusukuli.log --- beetsplug/lastgenre/__init__.py | 20 +++++++++++++------- docs/changelog.rst | 2 ++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index e65593730..ab0e22be9 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -26,6 +26,7 @@ https://gist.github.com/1241307 import pylast import os import yaml +import traceback from beets import plugins from beets import ui @@ -391,17 +392,22 @@ class LastGenrePlugin(plugins.BeetsPlugin): If `min_weight` is specified, tags are filtered by weight. """ + # Work around an inconsistency in pylast where + # Album.get_top_tags() does not return TopItem instances. + # https://code.google.com/p/pylast/issues/detail?id=85 + if isinstance(obj, pylast.Album): + obj = super(pylast.Album, obj) + try: - # Work around an inconsistency in pylast where - # Album.get_top_tags() does not return TopItem instances. - # https://code.google.com/p/pylast/issues/detail?id=85 - if isinstance(obj, pylast.Album): - res = super(pylast.Album, obj).get_top_tags() - else: - res = obj.get_top_tags() + res = obj.get_top_tags() except PYLAST_EXCEPTIONS as exc: self._log.debug(u'last.fm error: {0}', exc) return [] + except Exception as exc: + # Isolate bugs in pylast. + self._log.debug(traceback.format_exc()) + self._log.error('error in pylast library: {0}', exc) + return [] # Filter by weight (optionally). if min_weight: diff --git a/docs/changelog.rst b/docs/changelog.rst index 693c4b1d8..b222e3ad8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -123,6 +123,8 @@ Fixes: Unicode filenames. :bug:`1297` * :doc:`/plugins/discogs`: Handle and log more kinds of communication errors. :bug:`1299` :bug:`1305` +* :doc:`/plugins/lastgenre`: Bugs in the `pylast` library can no longer crash + beets. For developers: From 18fc0cd3def38c9ac5a933b48b3048a339a1b268 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 14:00:47 +0100 Subject: [PATCH 098/129] summarize_items(): work even with 0 items It crashed on empty albums. Fix #1357 --- beets/ui/commands.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 5d29093c9..bf38c5c2c 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -426,8 +426,6 @@ def summarize_items(items, singleton): this is an album or single-item import (if the latter, them `items` should only have one element). """ - assert items, "summarizing zero items" - summary_parts = [] if not singleton: summary_parts.append("{0} items".format(len(items))) @@ -443,12 +441,13 @@ def summarize_items(items, singleton): for fmt, count in format_counts.iteritems(): summary_parts.append('{0} {1}'.format(fmt, count)) - average_bitrate = sum([item.bitrate for item in items]) / len(items) - total_duration = sum([item.length for item in items]) - total_filesize = sum([item.filesize for item in items]) - summary_parts.append('{0}kbps'.format(int(average_bitrate / 1000))) - summary_parts.append(ui.human_seconds_short(total_duration)) - summary_parts.append(ui.human_bytes(total_filesize)) + if items: + average_bitrate = sum([item.bitrate for item in items]) / len(items) + total_duration = sum([item.length for item in items]) + total_filesize = sum([item.filesize for item in items]) + summary_parts.append('{0}kbps'.format(int(average_bitrate / 1000))) + summary_parts.append(ui.human_seconds_short(total_duration)) + summary_parts.append(ui.human_bytes(total_filesize)) return ', '.join(summary_parts) From 1e2d481ac068584e5f76baad75d9666b83cffb34 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 14:25:53 +0100 Subject: [PATCH 099/129] Add tests for summarize_items() --- test/test_ui.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/test_ui.py b/test/test_ui.py index 0f8bb6306..181ee25f6 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -22,7 +22,9 @@ import shutil import re import subprocess import platform +from copy import deepcopy +from mock import patch from test import _common from test._common import unittest from test.helper import capture_stdout, has_program, TestHelper, control_stdin @@ -972,6 +974,40 @@ class ShowChangeTest(_common.TestCase): u'caf.mp3 ->' in msg) +class SummarizeItemsTest(_common.TestCase): + def setUp(self): + super(SummarizeItemsTest, self).setUp() + item = library.Item() + item.bitrate = 4321 + item.length = 10 * 60 + 54 + item.format = "F" + self.item = item + fsize_mock = patch('beets.library.Item.try_filesize').start() + fsize_mock.return_value = 987 + + def test_summarize_item(self): + summary = commands.summarize_items([], True) + self.assertEqual(summary, "") + + summary = commands.summarize_items([self.item], True) + self.assertEqual(summary, "F, 4kbps, 10:54, 987.0 B") + + def test_summarize_items(self): + summary = commands.summarize_items([], False) + self.assertEqual(summary, "0 items") + + summary = commands.summarize_items([self.item], False) + self.assertEqual(summary, "1 items, F, 4kbps, 10:54, 987.0 B") + + i2 = deepcopy(self.item) + summary = commands.summarize_items([self.item, i2], False) + self.assertEqual(summary, "2 items, F, 4kbps, 21:48, 1.9 KB") + + i2.format = "G" + summary = commands.summarize_items([self.item, i2], False) + self.assertEqual(summary, "2 items, G 1, F 1, 4kbps, 21:48, 1.9 KB") + + class PathFormatTest(_common.TestCase): def test_custom_paths_prepend(self): default_formats = ui.get_path_formats() From 4bfa439ee14decfb9d690260f1de435508b63711 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 15 Mar 2015 11:58:53 +0100 Subject: [PATCH 100/129] database_change: send model that changed --- beets/library.py | 6 +++--- beetsplug/mpdupdate.py | 2 +- beetsplug/plexupdate.py | 2 +- beetsplug/smartplaylist.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 139cdfec0..132df4f52 100644 --- a/beets/library.py +++ b/beets/library.py @@ -270,15 +270,15 @@ class LibModel(dbcore.Model): def store(self): super(LibModel, self).store() - plugins.send('database_change', lib=self._db) + plugins.send('database_change', lib=self._db, model=self) def remove(self): super(LibModel, self).remove() - plugins.send('database_change', lib=self._db) + plugins.send('database_change', lib=self._db, model=self) def add(self, lib=None): super(LibModel, self).add(lib) - plugins.send('database_change', lib=self._db) + plugins.send('database_change', lib=self._db, model=self) def __format__(self, spec): if not spec: diff --git a/beetsplug/mpdupdate.py b/beetsplug/mpdupdate.py index 96141c567..dfe402497 100644 --- a/beetsplug/mpdupdate.py +++ b/beetsplug/mpdupdate.py @@ -80,7 +80,7 @@ class MPDUpdatePlugin(BeetsPlugin): self.register_listener('database_change', self.db_change) - def db_change(self, lib): + def db_change(self, lib, model): self.register_listener('cli_exit', self.update) def update(self, lib): diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index 9781e300e..5aa096486 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -55,7 +55,7 @@ class PlexUpdate(BeetsPlugin): self.register_listener('database_change', self.listen_for_db_change) - def listen_for_db_change(self, lib): + def listen_for_db_change(self, lib, model): """Listens for beets db change and register the update for the end""" self.register_listener('cli_exit', self.update) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index e2c256b2b..a9c0a25aa 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -65,7 +65,7 @@ class SmartPlaylistPlugin(BeetsPlugin): spl_update.func = update return [spl_update] - def db_change(self, lib): + def db_change(self, lib, model): self.register_listener('cli_exit', self.update_playlists) def update_playlists(self, lib): From 8e25a70e40d8541adea3c01e07b79d46c8b1188e Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 14:41:43 +0100 Subject: [PATCH 101/129] summarize_items(): sort format by decr. freq Make the summary deterministic. --- beets/ui/commands.py | 5 +++-- test/test_ui.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index bf38c5c2c..7dc54b953 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -437,8 +437,9 @@ def summarize_items(items, singleton): # A single format. summary_parts.append(items[0].format) else: - # Enumerate all the formats. - for fmt, count in format_counts.iteritems(): + # Enumerate all the formats by decreasing frequencies: + for fmt, count in sorted(format_counts.items(), + key=lambda (f, c): (-c, f)): summary_parts.append('{0} {1}'.format(fmt, count)) if items: diff --git a/test/test_ui.py b/test/test_ui.py index 181ee25f6..14cb4081f 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1005,7 +1005,10 @@ class SummarizeItemsTest(_common.TestCase): i2.format = "G" summary = commands.summarize_items([self.item, i2], False) - self.assertEqual(summary, "2 items, G 1, F 1, 4kbps, 21:48, 1.9 KB") + self.assertEqual(summary, "2 items, F 1, G 1, 4kbps, 21:48, 1.9 KB") + + summary = commands.summarize_items([self.item, i2, i2], False) + self.assertEqual(summary, "3 items, G 2, F 1, 4kbps, 32:42, 2.9 KB") class PathFormatTest(_common.TestCase): From f06c33cb71f800ed60ffed41ef282a8413ddca4d Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 15 Mar 2015 12:02:21 +0100 Subject: [PATCH 102/129] Smartplaylist: update only if item changed --- beetsplug/smartplaylist.py | 101 +++++++++++++++++++++++++------------ test/test_smartplaylist.py | 41 +++++++++++++++ 2 files changed, 110 insertions(+), 32 deletions(-) create mode 100644 test/test_smartplaylist.py diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index a9c0a25aa..ab63c9de8 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -21,30 +21,13 @@ from __future__ import (division, absolute_import, print_function, from beets.plugins import BeetsPlugin from beets import ui from beets.util import mkdirall, normpath, syspath +from beets.library import Item, Album, parse_query_parts +from beets.dbcore import OrQuery import os -def _items_for_query(lib, queries, album): - """Get the matching items for a list of queries. - - `queries` can either be a single string or a list of strings. In the - latter case, the results from each query are concatenated. `album` - indicates whether the queries are item-level or album-level. - """ - if isinstance(queries, basestring): - queries = [queries] - if album: - for query in queries: - for album in lib.albums(query): - for item in album.items(): - yield item - else: - for query in queries: - for item in lib.items(query): - yield item - - class SmartPlaylistPlugin(BeetsPlugin): + def __init__(self): super(SmartPlaylistPlugin, self).__init__() self.config.add({ @@ -54,42 +37,96 @@ class SmartPlaylistPlugin(BeetsPlugin): 'playlists': [] }) + self._matched_playlists = None + self._unmatched_playlists = None + if self.config['auto']: self.register_listener('database_change', self.db_change) def commands(self): def update(lib, opts, args): + self.build_queries() + self._matched_playlists = self._unmatched_playlists self.update_playlists(lib) spl_update = ui.Subcommand('splupdate', help='update the smart playlists') spl_update.func = update return [spl_update] + def build_queries(self): + """ + Instanciate queries for the playlists. + + Each playlist has 2 queries: one or items one for albums. We must also + remember its name. _unmatched_playlists is a set of tuples + (name, q, album_q). + """ + self._unmatched_playlists = set() + self._matched_playlists = set() + + for playlist in self.config['playlists'].get(list): + playlist_data = (playlist['name'],) + for key, Model in (('query', Item), ('album_query', Album)): + qs = playlist.get(key) + # FIXME sort mgmt + if qs is None: + query = None + sort = None + elif isinstance(qs, basestring): + query, sort = parse_query_parts(qs, Model) + else: + query = OrQuery([parse_query_parts(q, Model)[0] + for q in qs]) + sort = None + playlist_data += (query,) + + self._unmatched_playlists.add(playlist_data) + def db_change(self, lib, model): - self.register_listener('cli_exit', self.update_playlists) + if self._unmatched_playlists is None: + self.build_queries() + + for playlist in self._unmatched_playlists: + n, q, a_q = playlist + if a_q and isinstance(model, Album): + matches = a_q.match(model) + elif q and isinstance(model, Item): + matches = q.match(model) or q.match(model.get_album()) + else: + matches = False + + if matches: + self._log.debug("{0} will be updated because of {1}", n, model) + self._matched_playlists.add(playlist) + self.register_listener('cli_exit', self.update_playlists) + + self._unmatched_playlists -= self._matched_playlists def update_playlists(self, lib): - self._log.info("Updating smart playlists...") - playlists = self.config['playlists'].get(list) + self._log.info("Updating {0} smart playlists...", + len(self._matched_playlists)) + playlist_dir = self.config['playlist_dir'].as_filename() relative_to = self.config['relative_to'].get() if relative_to: relative_to = normpath(relative_to) - for playlist in playlists: - self._log.debug(u"Creating playlist {0[name]}", playlist) + for playlist in self._matched_playlists: + name, query, album_query = playlist + self._log.debug(u"Creating playlist {0}", name) items = [] - if 'album_query' in playlist: - items.extend(_items_for_query(lib, playlist['album_query'], - True)) - if 'query' in playlist: - items.extend(_items_for_query(lib, playlist['query'], False)) + + if query: + items.extend(lib.items(query)) + if album_query: + for album in lib.albums(album_query): + items.extend(album.items()) m3us = {} # As we allow tags in the m3u names, we'll need to iterate through # the items and generate the correct m3u file names. for item in items: - m3u_name = item.evaluate_template(playlist['name'], True) + m3u_name = item.evaluate_template(name, True) if m3u_name not in m3us: m3us[m3u_name] = [] item_path = item.path @@ -104,4 +141,4 @@ class SmartPlaylistPlugin(BeetsPlugin): with open(syspath(m3u_path), 'w') as f: for path in m3us[m3u]: f.write(path + b'\n') - self._log.info("{0} playlists updated", len(playlists)) + self._log.info("{0} playlists updated", len(self._matched_playlists)) diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py new file mode 100644 index 000000000..90aca4e9b --- /dev/null +++ b/test/test_smartplaylist.py @@ -0,0 +1,41 @@ +# This file is part of beets. +# Copyright 2015, Bruno Cauet. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import (division, absolute_import, print_function, + unicode_literals) + +from test._common import unittest +from beetsplug import smartplaylist +from beets import config, ui + +from test.helper import TestHelper + + +class SmartPlaylistTest(unittest.TestCase): + def test_build_queries(self): + pass + + def test_db_changes(self): + pass + + def test_playlist_update(self): + pass + + +class SmartPlaylistCLITest(unittest.TestCase, TestHelper): + def test_import(self): + pass + + def test_splupdate(self): + pass From 2d9f665848b726589e93cd7bca3785a30d442652 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 16:03:14 +0100 Subject: [PATCH 103/129] Smartplaylist: offer "splupdate " splupdate command of the SmartPlaylistPlugin looks in "args" for matches of playlist names. --- beetsplug/smartplaylist.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index ab63c9de8..197257459 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -44,15 +44,33 @@ class SmartPlaylistPlugin(BeetsPlugin): self.register_listener('database_change', self.db_change) def commands(self): - def update(lib, opts, args): - self.build_queries() - self._matched_playlists = self._unmatched_playlists - self.update_playlists(lib) spl_update = ui.Subcommand('splupdate', - help='update the smart playlists') - spl_update.func = update + help='update the smart playlists. Playlist ' + 'names may be passed as arguments.') + spl_update.func = self.update_cmd return [spl_update] + def update_cmd(self, lib, opts, args): + self.build_queries() + if args: + args = set(ui.decargs(args)) + for a in list(args): + if not a.endswith(".m3u"): + args.add("{0}.m3u".format(a)) + + playlists = {(name, q, a_q) + for name, q, a_q in self._unmatched_playlists + if name in args} + if not playlists: + # raise UserError + pass + self._matched_playlists = playlists + self._unmatched_playlists -= playlists + else: + self._matched_playlists = self._unmatched_playlists + + self.update_playlists(lib) + def build_queries(self): """ Instanciate queries for the playlists. From 774decda7d120e8ad83c69c5f70ae66ccfaf7925 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 16:21:49 +0100 Subject: [PATCH 104/129] =?UTF-8?q?Smartplayist:=20parse=5Fquery=5Fparts()?= =?UTF-8?q?=20=E2=86=92=20...ry=5Fstring()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beetsplug/smartplaylist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 197257459..12e9c3fcd 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -21,7 +21,7 @@ from __future__ import (division, absolute_import, print_function, from beets.plugins import BeetsPlugin from beets import ui from beets.util import mkdirall, normpath, syspath -from beets.library import Item, Album, parse_query_parts +from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery import os @@ -91,9 +91,9 @@ class SmartPlaylistPlugin(BeetsPlugin): query = None sort = None elif isinstance(qs, basestring): - query, sort = parse_query_parts(qs, Model) + query, sort = parse_query_string(qs, Model) else: - query = OrQuery([parse_query_parts(q, Model)[0] + query = OrQuery([parse_query_string(q, Model)[0] for q in qs]) sort = None playlist_data += (query,) From 40e793cdb1e4d2081eab636fae728893befeefae Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 16:25:43 +0100 Subject: [PATCH 105/129] Fix flake8 errors --- beetsplug/smartplaylist.py | 1 + test/test_smartplaylist.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 12e9c3fcd..e89a43cf7 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -96,6 +96,7 @@ class SmartPlaylistPlugin(BeetsPlugin): query = OrQuery([parse_query_string(q, Model)[0] for q in qs]) sort = None + del sort # FIXME playlist_data += (query,) self._unmatched_playlists.add(playlist_data) diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index 90aca4e9b..5d1f894a9 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -16,9 +16,6 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) from test._common import unittest -from beetsplug import smartplaylist -from beets import config, ui - from test.helper import TestHelper From a1b048f5de5ce7cad89aea524245dfc1876867ef Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 16:45:09 +0100 Subject: [PATCH 106/129] =?UTF-8?q?RegexpQuery:=20fix=20typo:=20false=20?= =?UTF-8?q?=E2=86=92=20fast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beets/dbcore/query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index cd891148e..24020e94c 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -177,8 +177,8 @@ class RegexpQuery(StringFieldQuery): Raises InvalidQueryError when the pattern is not a valid regular expression. """ - def __init__(self, field, pattern, false=True): - super(RegexpQuery, self).__init__(field, pattern, false) + def __init__(self, field, pattern, fast=True): + super(RegexpQuery, self).__init__(field, pattern, fast) try: self.pattern = re.compile(self.pattern) except re.error as exc: From 4151e819691555a4240b007b170b211a57d89b73 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 16:56:25 +0100 Subject: [PATCH 107/129] Implement __eq__ for all Query subclasses Tests are a bit light. --- beets/dbcore/query.py | 18 ++++++++++++++++-- test/test_query.py | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 24020e94c..b1314d1f8 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -73,6 +73,9 @@ class Query(object): """ raise NotImplementedError + def __eq__(self, other): + return type(self) == type(other) + class FieldQuery(Query): """An abstract query that searches in a specific field for a @@ -106,6 +109,10 @@ class FieldQuery(Query): def match(self, item): return self.value_match(self.pattern, item.get(self.field)) + def __eq__(self, other): + return super(FieldQuery, self).__eq__(other) and \ + self.field == other.field and self.pattern == other.pattern + class MatchQuery(FieldQuery): """A query that looks for exact matches in an item field.""" @@ -120,8 +127,7 @@ class MatchQuery(FieldQuery): class NoneQuery(FieldQuery): def __init__(self, field, fast=True): - self.field = field - self.fast = fast + super(NoneQuery, self).__init__(field, None, fast) def col_clause(self): return self.field + " IS NULL", () @@ -337,6 +343,10 @@ class CollectionQuery(Query): clause = (' ' + joiner + ' ').join(clause_parts) return clause, subvals + def __eq__(self, other): + return super(CollectionQuery, self).__eq__(other) and \ + self.subqueries == other.subqueries + class AnyFieldQuery(CollectionQuery): """A query that matches if a given FieldQuery subclass matches in @@ -362,6 +372,10 @@ class AnyFieldQuery(CollectionQuery): return True return False + def __eq__(self, other): + return super(AnyFieldQuery, self).__eq__(other) and \ + self.query_class == other.query_class + class MutableCollectionQuery(CollectionQuery): """A collection query whose subqueries may be modified after the diff --git a/test/test_query.py b/test/test_query.py index d512e02b8..6d8d744fe 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -60,6 +60,16 @@ class AnyFieldQueryTest(_common.LibTestCase): dbcore.query.SubstringQuery) self.assertEqual(self.lib.items(q).get(), None) + def test_eq(self): + q1 = dbcore.query.AnyFieldQuery('foo', ['bar'], + dbcore.query.SubstringQuery) + q2 = dbcore.query.AnyFieldQuery('foo', ['bar'], + dbcore.query.SubstringQuery) + self.assertEqual(q1, q2) + + q2.query_class = None + self.assertNotEqual(q1, q2) + class AssertsMixin(object): def assert_items_matched(self, results, titles): @@ -344,6 +354,16 @@ class MatchTest(_common.TestCase): def test_open_range(self): dbcore.query.NumericQuery('bitrate', '100000..') + def test_eq(self): + q1 = dbcore.query.MatchQuery('foo', 'bar') + q2 = dbcore.query.MatchQuery('foo', 'bar') + q3 = dbcore.query.MatchQuery('foo', 'baz') + q4 = dbcore.query.StringFieldQuery('foo', 'bar') + self.assertEqual(q1, q2) + self.assertNotEqual(q1, q3) + self.assertNotEqual(q1, q4) + self.assertNotEqual(q3, q4) + class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): def setUp(self): From 64614ff57904bd4c10b0c422be2b1ef0c4295c23 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 17:27:42 +0100 Subject: [PATCH 108/129] Query.__eq__: fix indents --- beets/dbcore/query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index b1314d1f8..3a6f03041 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -111,7 +111,7 @@ class FieldQuery(Query): def __eq__(self, other): return super(FieldQuery, self).__eq__(other) and \ - self.field == other.field and self.pattern == other.pattern + self.field == other.field and self.pattern == other.pattern class MatchQuery(FieldQuery): @@ -345,7 +345,7 @@ class CollectionQuery(Query): def __eq__(self, other): return super(CollectionQuery, self).__eq__(other) and \ - self.subqueries == other.subqueries + self.subqueries == other.subqueries class AnyFieldQuery(CollectionQuery): @@ -374,7 +374,7 @@ class AnyFieldQuery(CollectionQuery): def __eq__(self, other): return super(AnyFieldQuery, self).__eq__(other) and \ - self.query_class == other.query_class + self.query_class == other.query_class class MutableCollectionQuery(CollectionQuery): From e00282b4fe74e3a83f16739a0709a23b227d9961 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 17:25:09 +0100 Subject: [PATCH 109/129] Implement hash() for query subclasses It is implemented on mutable queries as well, which is bad. IMPORTANT: don't mutate your queries if you want to put them in a set or in a dict (as keys). --- beets/dbcore/query.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 3a6f03041..ba7125634 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -18,7 +18,7 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) import re -from operator import attrgetter +from operator import attrgetter, mul from beets import util from datetime import datetime, timedelta @@ -76,6 +76,9 @@ class Query(object): def __eq__(self, other): return type(self) == type(other) + def __hash__(self): + return 0 + class FieldQuery(Query): """An abstract query that searches in a specific field for a @@ -113,6 +116,9 @@ class FieldQuery(Query): return super(FieldQuery, self).__eq__(other) and \ self.field == other.field and self.pattern == other.pattern + def __hash__(self): + return hash((self.field, hash(self.pattern))) + class MatchQuery(FieldQuery): """A query that looks for exact matches in an item field.""" @@ -347,6 +353,12 @@ class CollectionQuery(Query): return super(CollectionQuery, self).__eq__(other) and \ self.subqueries == other.subqueries + def __hash__(self): + """Since subqueries are mutable, this object should not be hashable. + However and for conveniencies purposes, it can be hashed. + """ + return reduce(mul, map(hash, self.subqueries), 1) + class AnyFieldQuery(CollectionQuery): """A query that matches if a given FieldQuery subclass matches in @@ -376,6 +388,9 @@ class AnyFieldQuery(CollectionQuery): return super(AnyFieldQuery, self).__eq__(other) and \ self.query_class == other.query_class + def __hash__(self): + return hash((self.pattern, tuple(self.fields), self.query_class)) + class MutableCollectionQuery(CollectionQuery): """A collection query whose subqueries may be modified after the From 8b6b938a37bcd096c5c247c88bed8ef1ed9a9eb6 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 16:56:49 +0100 Subject: [PATCH 110/129] SmartPlaylistPlugin: add unit tests They're not exhaustive but still quite heavy. --- test/test_smartplaylist.py | 115 ++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index 5d1f894a9..83bb4543f 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -15,19 +15,128 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +from os import path +from tempfile import mkdtemp +from shutil import rmtree + +from mock import Mock, MagicMock + +from beetsplug.smartplaylist import SmartPlaylistPlugin +from beets.library import Item, Album, parse_query_string +from beets.dbcore import OrQuery +from beets.util import syspath +from beets import config + from test._common import unittest from test.helper import TestHelper class SmartPlaylistTest(unittest.TestCase): def test_build_queries(self): - pass + spl = SmartPlaylistPlugin() + self.assertEqual(spl._matched_playlists, None) + self.assertEqual(spl._unmatched_playlists, None) + + config['smartplaylist']['playlists'].set([]) + spl.build_queries() + self.assertEqual(spl._matched_playlists, set()) + self.assertEqual(spl._unmatched_playlists, set()) + + config['smartplaylist']['playlists'].set([ + {'name': 'foo', + 'query': 'FOO foo'}, + {'name': 'bar', + 'album_query': ['BAR bar1', 'BAR bar2']}, + {'name': 'baz', + 'query': 'BAZ baz', + 'album_query': 'BAZ baz'} + ]) + spl.build_queries() + self.assertEqual(spl._matched_playlists, set()) + foo_foo, _ = parse_query_string('FOO foo', Item) + bar_bar = OrQuery([parse_query_string('BAR bar1', Album)[0], + parse_query_string('BAR bar2', Album)[0]]) + baz_baz, _ = parse_query_string('BAZ baz', Item) + baz_baz2, _ = parse_query_string('BAZ baz', Album) + self.assertEqual(spl._unmatched_playlists, { + ('foo', foo_foo, None), + ('bar', None, bar_bar), + ('baz', baz_baz, baz_baz2) + }) def test_db_changes(self): - pass + spl = SmartPlaylistPlugin() + + i1 = MagicMock(Item) + i2 = MagicMock(Item) + a = MagicMock(Album) + i1.get_album.return_value = a + + q1 = Mock() + q1.matches.side_effect = {i1: False, i2: False}.__getitem__ + a_q1 = Mock() + a_q1.matches.side_effect = {a: True}.__getitem__ + q2 = Mock() + q2.matches.side_effect = {i1: False, i2: True}.__getitem__ + + pl1 = ('1', q1, a_q1) + pl2 = ('2', None, a_q1) + pl3 = ('3', q2, None) + + spl._unmatched_playlists = {pl1, pl2, pl3} + spl._matched_playlists = set() + spl.db_change(None, i1) + self.assertEqual(spl._unmatched_playlists, {pl2}) + self.assertEqual(spl._matched_playlists, {pl1, pl3}) + + spl._unmatched_playlists = {pl1, pl2, pl3} + spl._matched_playlists = set() + spl.db_change(None, i2) + self.assertEqual(spl._unmatched_playlists, {pl2}) + self.assertEqual(spl._matched_playlists, {pl1, pl3}) + + spl._unmatched_playlists = {pl1, pl2, pl3} + spl._matched_playlists = set() + spl.db_change(None, a) + self.assertEqual(spl._unmatched_playlists, {pl3}) + self.assertEqual(spl._matched_playlists, {pl1, pl2}) + spl.db_change(None, i2) + self.assertEqual(spl._unmatched_playlists, set()) + self.assertEqual(spl._matched_playlists, {pl1, pl2, pl3}) def test_playlist_update(self): - pass + spl = SmartPlaylistPlugin() + + i = Mock(path='/tagada.mp3') + i.evaluate_template.side_effect = lambda x, _: x + q = Mock() + a_q = Mock() + lib = Mock() + lib.items.return_value = [i] + lib.albums.return_value = [] + pl = 'my_playlist.m3u', q, a_q + spl._matched_playlists = {pl} + + dir = mkdtemp() + config['smartplaylist']['relative_to'] = False + config['smartplaylist']['playlist_dir'] = dir + try: + spl.update_playlists(lib) + except Exception: + rmtree(dir) + raise + + lib.items.assert_called_once_with(q) + lib.albums.assert_called_once_with(a_q) + + m3u_filepath = path.join(dir, pl[0]) + self.assertTrue(path.exists(m3u_filepath), m3u_filepath) + + with open(syspath(m3u_filepath), 'r') as f: + content = f.readlines() + rmtree(dir) + + self.assertEqual(content, ["/tagada.mp3\n"]) class SmartPlaylistCLITest(unittest.TestCase, TestHelper): From b79c025142602e9be8d1d6682d4dde98326cc05f Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 18:36:08 +0100 Subject: [PATCH 111/129] CLI tests for smartplaylist plugin No import CLI test. --- beetsplug/smartplaylist.py | 6 +++-- test/test_smartplaylist.py | 48 +++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index e89a43cf7..3e23f8407 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -62,8 +62,10 @@ class SmartPlaylistPlugin(BeetsPlugin): for name, q, a_q in self._unmatched_playlists if name in args} if not playlists: - # raise UserError - pass + raise ui.UserError('No playlist matching any of {0} ' + 'found'.format([name for name, _, _ in + self._unmatched_playlists])) + self._matched_playlists = playlists self._unmatched_playlists -= playlists else: diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index 83bb4543f..ac6ff921a 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -15,7 +15,7 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) -from os import path +from os import path, remove from tempfile import mkdtemp from shutil import rmtree @@ -25,6 +25,7 @@ from beetsplug.smartplaylist import SmartPlaylistPlugin from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery from beets.util import syspath +from beets.ui import UserError from beets import config from test._common import unittest @@ -130,18 +131,49 @@ class SmartPlaylistTest(unittest.TestCase): lib.albums.assert_called_once_with(a_q) m3u_filepath = path.join(dir, pl[0]) - self.assertTrue(path.exists(m3u_filepath), m3u_filepath) - + self.assertTrue(path.exists(m3u_filepath)) with open(syspath(m3u_filepath), 'r') as f: - content = f.readlines() + content = f.read() rmtree(dir) - self.assertEqual(content, ["/tagada.mp3\n"]) + self.assertEqual(content, "/tagada.mp3\n") class SmartPlaylistCLITest(unittest.TestCase, TestHelper): - def test_import(self): - pass + def setUp(self): + self.setup_beets() + + self.item = self.add_item() + config['smartplaylist']['playlists'].set([ + {'name': 'my_playlist.m3u', + 'query': self.item.title}, + {'name': 'all.m3u', + 'query': ''} + ]) + config['smartplaylist']['playlist_dir'].set(self.temp_dir) + self.load_plugins('smartplaylist') + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() def test_splupdate(self): - pass + with self.assertRaises(UserError): + self.run_with_output('splupdate', 'tagada') + + self.run_with_output('splupdate', 'my_playlist') + m3u_path = path.join(self.temp_dir, 'my_playlist.m3u') + self.assertTrue(path.exists(m3u_path)) + with open(m3u_path, 'r') as f: + self.assertEqual(f.read(), self.item.path + b"\n") + remove(m3u_path) + + self.run_with_output('splupdate', 'my_playlist.m3u') + with open(m3u_path, 'r') as f: + self.assertEqual(f.read(), self.item.path + b"\n") + remove(m3u_path) + + self.run_with_output('splupdate') + for name in ('my_playlist.m3u', 'all.m3u'): + with open(path.join(self.temp_dir, name), 'r') as f: + self.assertEqual(f.read(), self.item.path + b"\n") From 191ff61c532b58a8721354a03454c7a52b4f0f10 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 18:38:15 +0100 Subject: [PATCH 112/129] Document the database_change parameter update --- docs/changelog.rst | 2 ++ docs/dev/plugins.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b222e3ad8..57802841e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -128,6 +128,8 @@ Fixes: For developers: +* The ``database_change`` event now sends the item or album that is subject to + a change in the db. * the ``OptionParser`` is now a ``CommonOptionsParser`` that offers facilities for adding usual options (``--album``, ``--path`` and ``--format``). See :ref:`add_subcommands`. :bug:`1271` diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 0c1f7017f..1d610f53b 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -203,7 +203,7 @@ The events currently available are: Library object. Parameter: ``lib``. * *database_change*: a modification has been made to the library database. The - change might not be committed yet. Parameter: ``lib``. + change might not be committed yet. Parameters: ``lib`` and ``model``. * *cli_exit*: called just before the ``beet`` command-line program exits. Parameter: ``lib``. From 5d1ee0457fe99b9c3a66cd675ca0e1f54e70421f Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 18:47:38 +0100 Subject: [PATCH 113/129] Document the smartplaylist plugin updates --- docs/changelog.rst | 5 +++++ docs/plugins/smartplaylist.rst | 11 +++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 57802841e..c2adb1126 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,11 @@ Changelog Features: +* :doc:`/plugins/smartplaylist`: detect for each playlist if it needs to be + regenated, instead of systematically regenerating all of them after a + database modification. +* :doc:`/plugins/smartplaylist`: the ``splupdate`` command can now take + additinal parameters: names of the playlists to regenerate. * Beets now accept top-level options ``--format-item`` and ``--format-album`` before any subcommand to control how items and albums are displayed. :bug:`1271`: diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index bc39e581e..b414e1eb0 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -53,13 +53,16 @@ to albums that have a ``for_travel`` extensible field set to 1:: album_query: 'for_travel:1' query: 'for_travel:1' -By default, all playlists are automatically regenerated at the end of the -session if the library database was changed. To force regeneration, you can -invoke it manually from the command line:: +By default, each playlist is automatically regenerated at the end of the +session if an item or album it matches changed in the library database. To +force regeneration, you can invoke it manually from the command line:: $ beet splupdate -which will generate your new smart playlists. +This will regenerate all smart playlists. You can also specify which ones you +want to regenerate:: + + $ beet splupdate BeatlesUniverse.m3u MyTravelPlaylist You can also use this plugin together with the :doc:`mpdupdate`, in order to automatically notify MPD of the playlist change, by adding ``mpdupdate`` to From 7bee2f093b90a5fb454e0b6f9fedeeb3fffc592e Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 18:48:14 +0100 Subject: [PATCH 114/129] changelog: fix an extra ':' after a bug # --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c2adb1126..f3e71e9dd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ Features: additinal parameters: names of the playlists to regenerate. * Beets now accept top-level options ``--format-item`` and ``--format-album`` before any subcommand to control how items and albums are displayed. - :bug:`1271`: + :bug:`1271` * :doc:`/plugins/replaygain`: There is a new backend for the `bs1770gain`_ tool. Thanks to :user:`jmwatte`. :bug:`1343` * There are now multiple levels of verbosity. On the command line, you can From 65b52b9c48a4fe63a97342c9d0e589a159432b17 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 16 Mar 2015 19:30:31 +0100 Subject: [PATCH 115/129] python 2.6 compat: don't use set literals In smartplaylist and test_smartplaylist. --- beetsplug/smartplaylist.py | 6 +++--- test/test_smartplaylist.py | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 3e23f8407..e7048a1cf 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -58,9 +58,9 @@ class SmartPlaylistPlugin(BeetsPlugin): if not a.endswith(".m3u"): args.add("{0}.m3u".format(a)) - playlists = {(name, q, a_q) - for name, q, a_q in self._unmatched_playlists - if name in args} + playlists = set((name, q, a_q) + for name, q, a_q in self._unmatched_playlists + if name in args) if not playlists: raise ui.UserError('No playlist matching any of {0} ' 'found'.format([name for name, _, _ in diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index ac6ff921a..fc9bba59a 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -59,11 +59,11 @@ class SmartPlaylistTest(unittest.TestCase): parse_query_string('BAR bar2', Album)[0]]) baz_baz, _ = parse_query_string('BAZ baz', Item) baz_baz2, _ = parse_query_string('BAZ baz', Album) - self.assertEqual(spl._unmatched_playlists, { + self.assertEqual(spl._unmatched_playlists, set([ ('foo', foo_foo, None), ('bar', None, bar_bar), ('baz', baz_baz, baz_baz2) - }) + ])) def test_db_changes(self): spl = SmartPlaylistPlugin() @@ -84,26 +84,26 @@ class SmartPlaylistTest(unittest.TestCase): pl2 = ('2', None, a_q1) pl3 = ('3', q2, None) - spl._unmatched_playlists = {pl1, pl2, pl3} + spl._unmatched_playlists = set([pl1, pl2, pl3]) spl._matched_playlists = set() spl.db_change(None, i1) - self.assertEqual(spl._unmatched_playlists, {pl2}) - self.assertEqual(spl._matched_playlists, {pl1, pl3}) + self.assertEqual(spl._unmatched_playlists, set([pl2])) + self.assertEqual(spl._matched_playlists, set([pl1, pl3])) - spl._unmatched_playlists = {pl1, pl2, pl3} + spl._unmatched_playlists = set([pl1, pl2, pl3]) spl._matched_playlists = set() spl.db_change(None, i2) - self.assertEqual(spl._unmatched_playlists, {pl2}) - self.assertEqual(spl._matched_playlists, {pl1, pl3}) + self.assertEqual(spl._unmatched_playlists, set([pl2])) + self.assertEqual(spl._matched_playlists, set([pl1, pl3])) - spl._unmatched_playlists = {pl1, pl2, pl3} + spl._unmatched_playlists = set([pl1, pl2, pl3]) spl._matched_playlists = set() spl.db_change(None, a) - self.assertEqual(spl._unmatched_playlists, {pl3}) - self.assertEqual(spl._matched_playlists, {pl1, pl2}) + self.assertEqual(spl._unmatched_playlists, set([pl3])) + self.assertEqual(spl._matched_playlists, set([pl1, pl2])) spl.db_change(None, i2) self.assertEqual(spl._unmatched_playlists, set()) - self.assertEqual(spl._matched_playlists, {pl1, pl2, pl3}) + self.assertEqual(spl._matched_playlists, set([pl1, pl2, pl3])) def test_playlist_update(self): spl = SmartPlaylistPlugin() @@ -116,7 +116,7 @@ class SmartPlaylistTest(unittest.TestCase): lib.items.return_value = [i] lib.albums.return_value = [] pl = 'my_playlist.m3u', q, a_q - spl._matched_playlists = {pl} + spl._matched_playlists = [pl] dir = mkdtemp() config['smartplaylist']['relative_to'] = False From bcfdcdc4b768c138c074a4ac599623f7e3f35066 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Mar 2015 17:20:40 +0100 Subject: [PATCH 116/129] Automatically remove dups when they're all empty --- beets/ui/commands.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 7dc54b953..4b1248add 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -778,6 +778,16 @@ class TerminalImportSession(importer.ImportSession): log.warn(u"This {0} is already in the library!", ("album" if task.is_album else "item")) + # skip empty albums (coming from a previous failed import session) + if task.is_album: + real_duplicates = filter(len, found_duplicates) + if not real_duplicates: + log.info("All duplicates are empty, we ignore them") + task.should_remove_duplicates = True + return + else: + real_duplicates = found_duplicates + if config['import']['quiet']: # In quiet mode, don't prompt -- just skip. log.info(u'Skipping.') @@ -785,11 +795,17 @@ class TerminalImportSession(importer.ImportSession): else: # Print some detail about the existing and new items so the # user can make an informed decision. - for duplicate in found_duplicates: + for duplicate in real_duplicates: print("Old: " + summarize_items( list(duplicate.items()) if task.is_album else [duplicate], not task.is_album, )) + + if real_duplicates != found_duplicates: # there's empty albums + count = len(found_duplicates) - len(real_duplicates) + print("Old: {0} empty album{1}".format( + count, "s" if count > 1 else "")) + print("New: " + summarize_items( task.imported_items(), not task.is_album, From 45c0c9b3cbcf4689eb22555f97e53c362f94c4fe Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Tue, 17 Mar 2015 18:43:55 +0100 Subject: [PATCH 117/129] Deal with sorting Try to follow any sort found & manage absence of sort. When there are multiple sort directives given, concatenate them. Tests not extended yet. --- beets/dbcore/query.py | 12 +++++++++ beetsplug/smartplaylist.py | 50 ++++++++++++++++++++++++++------------ test/test_smartplaylist.py | 37 +++++++++++++++++----------- 3 files changed, 69 insertions(+), 30 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index ba7125634..a51805bed 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -730,3 +730,15 @@ class NullSort(Sort): """No sorting. Leave results unsorted.""" def sort(items): return items + + def __nonzero__(self): + return self.__bool__() + + def __bool__(self): + return False + + def __eq__(self, other): + return type(self) == type(other) or other is None + + def __hash__(self): + return 0 diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index e7048a1cf..80e1faa89 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -23,6 +23,7 @@ from beets import ui from beets.util import mkdirall, normpath, syspath from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery +from beets.dbcore.query import MultipleSort import os @@ -77,9 +78,16 @@ class SmartPlaylistPlugin(BeetsPlugin): """ Instanciate queries for the playlists. - Each playlist has 2 queries: one or items one for albums. We must also - remember its name. _unmatched_playlists is a set of tuples - (name, q, album_q). + Each playlist has 2 queries: one or items one for albums, each with a + sort. We must also remember its name. _unmatched_playlists is a set of + tuples (name, (q, q_sort), (album_q, album_q_sort)). + + sort may be any sort, or NullSort, or None. None and NullSort are + equivalent and both eval to False. + More precisely + - it will be NullSort when a playlist query ('query' or 'album_query') + is a single item or a list with 1 element + - it will be None when there are multiple items i a query """ self._unmatched_playlists = set() self._matched_playlists = set() @@ -88,18 +96,28 @@ class SmartPlaylistPlugin(BeetsPlugin): playlist_data = (playlist['name'],) for key, Model in (('query', Item), ('album_query', Album)): qs = playlist.get(key) - # FIXME sort mgmt if qs is None: - query = None - sort = None + query_and_sort = None, None elif isinstance(qs, basestring): - query, sort = parse_query_string(qs, Model) + query_and_sort = parse_query_string(qs, Model) + elif len(qs) == 1: + query_and_sort = parse_query_string(qs[0], Model) else: - query = OrQuery([parse_query_string(q, Model)[0] - for q in qs]) - sort = None - del sort # FIXME - playlist_data += (query,) + # multiple queries and sorts + queries, sorts = zip(*(parse_query_string(q, Model) + for q in qs)) + query = OrQuery(queries) + sort = MultipleSort() + for s in sorts: + if s: + sort.add_sort(s) + if not sort.sorts: + sort = None + elif len(sort.sorts) == 1: + sort = sort.sorts[0] + query_and_sort = query, sort + + playlist_data += (query_and_sort,) self._unmatched_playlists.add(playlist_data) @@ -108,7 +126,7 @@ class SmartPlaylistPlugin(BeetsPlugin): self.build_queries() for playlist in self._unmatched_playlists: - n, q, a_q = playlist + n, (q, _), (a_q, _) = playlist if a_q and isinstance(model, Album): matches = a_q.match(model) elif q and isinstance(model, Item): @@ -133,14 +151,14 @@ class SmartPlaylistPlugin(BeetsPlugin): relative_to = normpath(relative_to) for playlist in self._matched_playlists: - name, query, album_query = playlist + name, (query, q_sort), (album_query, a_q_sort) = playlist self._log.debug(u"Creating playlist {0}", name) items = [] if query: - items.extend(lib.items(query)) + items.extend(lib.items(query, q_sort)) if album_query: - for album in lib.albums(album_query): + for album in lib.albums(album_query, a_q_sort): items.extend(album.items()) m3us = {} diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index fc9bba59a..42bffc8a5 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -24,6 +24,7 @@ from mock import Mock, MagicMock from beetsplug.smartplaylist import SmartPlaylistPlugin from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery +from beets.dbcore.query import NullSort from beets.util import syspath from beets.ui import UserError from beets import config @@ -54,15 +55,15 @@ class SmartPlaylistTest(unittest.TestCase): ]) spl.build_queries() self.assertEqual(spl._matched_playlists, set()) - foo_foo, _ = parse_query_string('FOO foo', Item) - bar_bar = OrQuery([parse_query_string('BAR bar1', Album)[0], - parse_query_string('BAR bar2', Album)[0]]) - baz_baz, _ = parse_query_string('BAZ baz', Item) - baz_baz2, _ = parse_query_string('BAZ baz', Album) + foo_foo = parse_query_string('FOO foo', Item) + baz_baz = parse_query_string('BAZ baz', Item) + baz_baz2 = parse_query_string('BAZ baz', Album) + bar_bar = OrQuery((parse_query_string('BAR bar1', Album)[0], + parse_query_string('BAR bar2', Album)[0])) self.assertEqual(spl._unmatched_playlists, set([ - ('foo', foo_foo, None), - ('bar', None, bar_bar), - ('baz', baz_baz, baz_baz2) + ('foo', foo_foo, (None, None)), + ('baz', baz_baz, baz_baz2), + ('bar', (None, None), (bar_bar, None)), ])) def test_db_changes(self): @@ -80,9 +81,9 @@ class SmartPlaylistTest(unittest.TestCase): q2 = Mock() q2.matches.side_effect = {i1: False, i2: True}.__getitem__ - pl1 = ('1', q1, a_q1) - pl2 = ('2', None, a_q1) - pl3 = ('3', q2, None) + pl1 = '1', (q1, None), (a_q1, None) + pl2 = '2', (None, None), (a_q1, None) + pl3 = '3', (q2, None), (None, None) spl._unmatched_playlists = set([pl1, pl2, pl3]) spl._matched_playlists = set() @@ -115,7 +116,7 @@ class SmartPlaylistTest(unittest.TestCase): lib = Mock() lib.items.return_value = [i] lib.albums.return_value = [] - pl = 'my_playlist.m3u', q, a_q + pl = 'my_playlist.m3u', (q, None), (a_q, None) spl._matched_playlists = [pl] dir = mkdtemp() @@ -127,8 +128,8 @@ class SmartPlaylistTest(unittest.TestCase): rmtree(dir) raise - lib.items.assert_called_once_with(q) - lib.albums.assert_called_once_with(a_q) + lib.items.assert_called_once_with(q, None) + lib.albums.assert_called_once_with(a_q, None) m3u_filepath = path.join(dir, pl[0]) self.assertTrue(path.exists(m3u_filepath)) @@ -177,3 +178,11 @@ class SmartPlaylistCLITest(unittest.TestCase, TestHelper): for name in ('my_playlist.m3u', 'all.m3u'): with open(path.join(self.temp_dir, name), 'r') as f: self.assertEqual(f.read(), self.item.path + b"\n") + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == b'__main__': + unittest.main(defaultTest='suite') From bcd57bd2b5f0c67b0ccfb650f96cbf60cf79328a Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Mar 2015 18:46:24 +0100 Subject: [PATCH 118/129] Test queries building sort management in smartplaylist Slighly modify Sort parsing: avoid building MultiplSort() instances comptised of a single sort, but return that sort instead, since it wraps things with any gain. --- beets/dbcore/query.py | 21 +++++++++++++++++++++ beets/dbcore/queryparse.py | 6 ++++-- beetsplug/smartplaylist.py | 15 ++++++++++----- test/test_dbcore.py | 11 ++++++----- test/test_smartplaylist.py | 27 ++++++++++++++++++++++++++- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index a51805bed..965c13868 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -625,6 +625,12 @@ class Sort(object): """ return False + def __hash__(self): + return 0 + + def __eq__(self, other): + return type(self) == type(other) + class MultipleSort(Sort): """Sort that encapsulates multiple sub-sorts. @@ -686,6 +692,13 @@ class MultipleSort(Sort): def __repr__(self): return u'MultipleSort({0})'.format(repr(self.sorts)) + def __hash__(self): + return hash(tuple(self.sorts)) + + def __eq__(self, other): + return super(MultipleSort, self).__eq__(other) and \ + self.sorts == other.sorts + class FieldSort(Sort): """An abstract sort criterion that orders by a specific field (of @@ -709,6 +722,14 @@ class FieldSort(Sort): '+' if self.ascending else '-', ) + def __hash__(self): + return hash((self.field, self.ascending)) + + def __eq__(self, other): + return super(FieldSort, self).__eq__(other) and \ + self.field == other.field and \ + self.ascending == other.ascending + class FixedFieldSort(FieldSort): """Sort object to sort on a fixed field. diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index 6628bebf0..1dcf9c4b3 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -152,12 +152,14 @@ def sort_from_strings(model_cls, sort_parts): """Create a `Sort` from a list of sort criteria (strings). """ if not sort_parts: - return query.NullSort() + sort = query.NullSort() + elif len(sort_parts) == 1: + sort = construct_sort_part(model_cls, sort_parts[0]) else: sort = query.MultipleSort() for part in sort_parts: sort.add_sort(construct_sort_part(model_cls, part)) - return sort + return sort def parse_sorted_query(model_cls, parts, prefixes={}, diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 80e1faa89..34fa94979 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -107,14 +107,19 @@ class SmartPlaylistPlugin(BeetsPlugin): queries, sorts = zip(*(parse_query_string(q, Model) for q in qs)) query = OrQuery(queries) - sort = MultipleSort() + final_sorts = [] for s in sorts: if s: - sort.add_sort(s) - if not sort.sorts: + if isinstance(s, MultipleSort): + final_sorts += s.sorts + else: + final_sorts.append(s) + if not final_sorts: sort = None - elif len(sort.sorts) == 1: - sort = sort.sorts[0] + elif len(final_sorts) == 1: + sort, = final_sorts + else: + sort = MultipleSort(final_sorts) query_and_sort = query, sort playlist_data += (query_and_sort,) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index dffe9ae75..39867ceb0 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -449,6 +449,7 @@ class SortFromStringsTest(unittest.TestCase): def test_zero_parts(self): s = self.sfs([]) self.assertIsInstance(s, dbcore.query.NullSort) + self.assertEqual(s, dbcore.query.NullSort()) def test_one_parts(self): s = self.sfs(['field+']) @@ -461,17 +462,17 @@ class SortFromStringsTest(unittest.TestCase): def test_fixed_field_sort(self): s = self.sfs(['field_one+']) - self.assertIsInstance(s, dbcore.query.MultipleSort) - self.assertIsInstance(s.sorts[0], dbcore.query.FixedFieldSort) + self.assertIsInstance(s, dbcore.query.FixedFieldSort) + self.assertEqual(s, dbcore.query.FixedFieldSort('field_one')) def test_flex_field_sort(self): s = self.sfs(['flex_field+']) - self.assertIsInstance(s, dbcore.query.MultipleSort) - self.assertIsInstance(s.sorts[0], dbcore.query.SlowFieldSort) + self.assertIsInstance(s, dbcore.query.SlowFieldSort) + self.assertEqual(s, dbcore.query.SlowFieldSort('flex_field')) def test_special_sort(self): s = self.sfs(['some_sort+']) - self.assertIsInstance(s.sorts[0], TestSort) + self.assertIsInstance(s, TestSort) class ResultsIteratorTest(unittest.TestCase): diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index 42bffc8a5..4ed2cb4ba 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -24,7 +24,7 @@ from mock import Mock, MagicMock from beetsplug.smartplaylist import SmartPlaylistPlugin from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery -from beets.dbcore.query import NullSort +from beets.dbcore.query import NullSort, MultipleSort, FixedFieldSort from beets.util import syspath from beets.ui import UserError from beets import config @@ -66,6 +66,31 @@ class SmartPlaylistTest(unittest.TestCase): ('bar', (None, None), (bar_bar, None)), ])) + def test_build_queries_with_sorts(self): + spl = SmartPlaylistPlugin() + config['smartplaylist']['playlists'].set([ + {'name': 'no_sort', 'query': 'foo'}, + {'name': 'one_sort', 'query': 'foo year+'}, + {'name': 'only_empty_sorts', 'query': ['foo', 'bar']}, + {'name': 'one_non_empty_sort', 'query': ['foo year+', 'bar']}, + {'name': 'multiple_sorts', 'query': ['foo year+', 'bar genre-']}, + {'name': 'mixed', 'query': ['foo year+', 'bar', 'baz genre+ id-']} + ]) + + spl.build_queries() + sorts = {name: sort for name, (_, sort), _ in spl._unmatched_playlists} + + asseq = self.assertEqual # less cluttered code + S = FixedFieldSort # short cut since we're only dealing with this + asseq(sorts["no_sort"], NullSort()) + asseq(sorts["one_sort"], S('year')) + asseq(sorts["only_empty_sorts"], None) + asseq(sorts["one_non_empty_sort"], S('year')) + asseq(sorts["multiple_sorts"], + MultipleSort([S('year'), S('genre', False)])) + asseq(sorts["mixed"], + MultipleSort([S('year'), S('genre'), S('id', False)])) + def test_db_changes(self): spl = SmartPlaylistPlugin() From 86443c076dd52b55a75411090ff84a668448d714 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 18 Mar 2015 19:00:44 +0100 Subject: [PATCH 119/129] Document smartplaylist sorting behavior. --- docs/conf.py | 2 +- docs/plugins/smartplaylist.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4aeb66d33..82fc15da8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ extlinks = { } # Options for HTML output -html_theme = 'classic' +html_theme = 'default' htmlhelp_basename = 'beetsdoc' # Options for LaTeX output diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index b414e1eb0..2f691c4fe 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -44,6 +44,18 @@ You can also gather the results of several queries by putting them in a list. - name: 'BeatlesUniverse.m3u' query: ['artist:beatles', 'genre:"beatles cover"'] +Note that since beets query syntax is in effect, you can also use sorting +directives:: + + - name: 'Chronological Beatles' + query: 'artist:Beatles year+' + - name: 'Mixed Rock' + query: ['artist:Beatles year+', 'artist:"Led Zeppelin" bitrate+'] + +The former case behaves as expected, however please note that in the latter the +sorts will be merged: ``year+ bitrate+`` will apply to both the Beatles and Led +Zeppelin. If that bothers you, please get in touch. + For querying albums instead of items (mainly useful with extensible fields), use the ``album_query`` field. ``query`` and ``album_query`` can be used at the same time. The following example gathers single items but also items belonging From 12c2511b1fe72a29ab7bfe47ff509897dfd6780b Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 19 Mar 2015 13:37:29 +0100 Subject: [PATCH 120/129] Smartplaylist tests: don't use a dict literal fucking py26! --- test/test_smartplaylist.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index 4ed2cb4ba..4af1cfeb6 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -78,7 +78,8 @@ class SmartPlaylistTest(unittest.TestCase): ]) spl.build_queries() - sorts = {name: sort for name, (_, sort), _ in spl._unmatched_playlists} + sorts = dict((name, sort) + for name, (_, sort), _ in spl._unmatched_playlists) asseq = self.assertEqual # less cluttered code S = FixedFieldSort # short cut since we're only dealing with this From 7f34c101d70126497afbf448c9d7ffb6faf6655a Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 19 Mar 2015 13:37:53 +0100 Subject: [PATCH 121/129] Plugin events: restore backwards compatibility An event listener that expects too few arguments won't crash, arguments will be cut off instead. This restores a backwards-compatibility hack that was removed in commit 327b62b6. --- beets/plugins.py | 28 +++++++++---- test/test_plugins.py | 94 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 10 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index bee4d9f32..c642e9318 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -17,6 +17,7 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) +import inspect import traceback import re from collections import defaultdict @@ -100,26 +101,37 @@ class BeetsPlugin(object): `self.import_stages`. Wrapping provides some bookkeeping for the plugin: specifically, the logging level is adjusted to WARNING. """ - return [self._set_log_level(logging.WARNING, import_stage) + return [self._set_log_level_and_params(logging.WARNING, import_stage) for import_stage in self.import_stages] - def _set_log_level(self, base_log_level, func): + def _set_log_level_and_params(self, base_log_level, func): """Wrap `func` to temporarily set this plugin's logger level to `base_log_level` + config options (and restore it to its previous - value after the function returns). + value after the function returns). Also determines which params may not + be sent for backwards-compatibility. - Note that that value may not be NOTSET, e.g. if a plugin import stage - triggers an event that is listened this very same plugin + Note that the log level value may not be NOTSET, e.g. if a plugin + import stage triggers an event that is listened this very same plugin. """ + argspec = inspect.getargspec(func) + @wraps(func) def wrapper(*args, **kwargs): old_log_level = self._log.level verbosity = beets.config['verbose'].get(int) log_level = max(logging.DEBUG, base_log_level - 10 * verbosity) self._log.setLevel(log_level) - try: - return func(*args, **kwargs) + try: + return func(*args, **kwargs) + except TypeError as exc: + if exc.args[0].startswith(func.__name__): + # caused by 'func' and not stuff internal to 'func' + kwargs = dict((arg, val) for arg, val in kwargs.items() + if arg in argspec.args) + return func(*args, **kwargs) + else: + raise finally: self._log.setLevel(old_log_level) return wrapper @@ -186,7 +198,7 @@ class BeetsPlugin(object): def register_listener(self, event, func): """Add a function as a listener for the specified event. """ - wrapped_func = self._set_log_level(logging.WARNING, func) + wrapped_func = self._set_log_level_and_params(logging.WARNING, func) cls = self.__class__ if cls.listeners is None or cls._raw_listeners is None: diff --git a/test/test_plugins.py b/test/test_plugins.py index c9c5be502..2e8bedca1 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -16,8 +16,9 @@ from __future__ import (division, absolute_import, print_function, unicode_literals) import os -from mock import patch +from mock import patch, Mock import shutil +import itertools from beets.importer import SingletonImportTask, SentinelImportTask, \ ArchiveImportTask @@ -57,7 +58,6 @@ class ItemTypesTest(unittest.TestCase, TestHelper): def setUp(self): self.setup_plugin_loader() - self.setup_beets() def tearDown(self): self.teardown_plugin_loader() @@ -309,6 +309,96 @@ class ListenersTest(unittest.TestCase, TestHelper): self.assertEqual(DummyPlugin._raw_listeners['cli_exit'], [d.dummy, d2.dummy]) + @patch('beets.plugins.find_plugins') + @patch('beets.plugins.inspect') + def test_events_called(self, mock_inspect, mock_find_plugins): + mock_inspect.getargspec.return_value = None + + class DummyPlugin(plugins.BeetsPlugin): + def __init__(self): + super(DummyPlugin, self).__init__() + self.foo = Mock(__name__=b'foo') + self.register_listener('event_foo', self.foo) + self.bar = Mock(__name__=b'bar') + self.register_listener('event_bar', self.bar) + + d = DummyPlugin() + mock_find_plugins.return_value = d, + + plugins.send('event') + d.foo.assert_has_calls([]) + d.bar.assert_has_calls([]) + + plugins.send('event_foo', var="tagada") + d.foo.assert_called_once_with(var="tagada") + d.bar.assert_has_calls([]) + + @patch('beets.plugins.find_plugins') + def test_listener_params(self, mock_find_plugins): + test = self + + class DummyPlugin(plugins.BeetsPlugin): + def __init__(self): + super(DummyPlugin, self).__init__() + for i in itertools.count(1): + try: + meth = getattr(self, 'dummy{0}'.format(i)) + except AttributeError: + break + self.register_listener('event{0}'.format(i), meth) + + def dummy1(self, foo): + test.assertEqual(foo, 5) + + def dummy2(self, foo=None): + test.assertEqual(foo, 5) + + def dummy3(self): + # argument cut off + pass + + def dummy4(self, bar=None): + # argument cut off + pass + + def dummy5(self, bar): + test.assertFalse(True) + + # more complex exmaples + + def dummy6(self, foo, bar=None): + test.assertEqual(foo, 5) + test.assertEqual(bar, None) + + def dummy7(self, foo, **kwargs): + test.assertEqual(foo, 5) + test.assertEqual(kwargs, {}) + + def dummy8(self, foo, bar, **kwargs): + test.assertFalse(True) + + def dummy9(self, **kwargs): + test.assertEqual(kwargs, {"foo": 5}) + + d = DummyPlugin() + mock_find_plugins.return_value = d, + + plugins.send('event1', foo=5) + plugins.send('event2', foo=5) + plugins.send('event3', foo=5) + plugins.send('event4', foo=5) + + with self.assertRaises(TypeError): + plugins.send('event5', foo=5) + + plugins.send('event6', foo=5) + plugins.send('event7', foo=5) + + with self.assertRaises(TypeError): + plugins.send('event8', foo=5) + + plugins.send('event9', foo=5) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 7be7a78d5a4d65566144a4d165ebc683e4b8cfdd Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 19 Mar 2015 13:41:55 +0100 Subject: [PATCH 122/129] Restore recent html_theme value in docs/conf.py The old value got restored in commit 86443c076 by mistake. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 82fc15da8..4aeb66d33 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ extlinks = { } # Options for HTML output -html_theme = 'default' +html_theme = 'classic' htmlhelp_basename = 'beetsdoc' # Options for LaTeX output From a96f2fd3c940366b6d5c0f722bc9ecea3689f6dc Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Thu, 19 Mar 2015 13:51:20 +0100 Subject: [PATCH 123/129] Fix pep8 --- beets/dbcore/query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 965c13868..330913706 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -697,7 +697,7 @@ class MultipleSort(Sort): def __eq__(self, other): return super(MultipleSort, self).__eq__(other) and \ - self.sorts == other.sorts + self.sorts == other.sorts class FieldSort(Sort): @@ -727,8 +727,8 @@ class FieldSort(Sort): def __eq__(self, other): return super(FieldSort, self).__eq__(other) and \ - self.field == other.field and \ - self.ascending == other.ascending + self.field == other.field and \ + self.ascending == other.ascending class FixedFieldSort(FieldSort): From a70f8bb91f3fc5489513bd06f325686f6a5d9b12 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 20 Mar 2015 19:52:36 -0400 Subject: [PATCH 124/129] Fix #1365: lastimport config --- beetsplug/lastimport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/lastimport.py b/beetsplug/lastimport.py index 66da6880d..0303db219 100644 --- a/beetsplug/lastimport.py +++ b/beetsplug/lastimport.py @@ -51,8 +51,8 @@ class LastImportPlugin(plugins.BeetsPlugin): def import_lastfm(lib, log): - user = config['lastfm']['user'] - per_page = config['lastimport']['per_page'] + user = config['lastfm']['user'].get(unicode) + per_page = config['lastimport']['per_page'].get(int) if not user: raise ui.UserError('You must specify a user name for lastimport') From c4d7dd0d6d506d90a0e3d1c759ce4b27bd50d9bf Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Mon, 23 Mar 2015 07:56:20 +0100 Subject: [PATCH 125/129] Add beets-setlister to community plugins docs (shameless self-promotion!) --- docs/plugins/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 3a79af4d7..2cda05e78 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -202,6 +202,8 @@ Here are a few of the plugins written by the beets community: * `beets-follow`_ lets you check for new albums from artists you like. +* `beets-setlister`_ generate playlists from the setlists of a given artist + .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts .. _dsedivec: https://github.com/dsedivec/beets-plugins @@ -215,3 +217,4 @@ Here are a few of the plugins written by the beets community: .. _beet-amazon: https://github.com/jmwatte/beet-amazon .. _beets-alternatives: https://github.com/geigerzaehler/beets-alternatives .. _beets-follow: https://github.com/nolsto/beets-follow +.. _beets-setlister: https://github.com/tomjaspers/beets-setlister From 9b5d78bad09c1e42bfa8f680b861a8c4e42f4caa Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 23 Mar 2015 17:24:30 +0100 Subject: [PATCH 126/129] Fix album emptiness test in duplicates resolution Fix #1367. Tests are missing for I've not found yet a satisfying way to build them. --- beets/ui/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 4b1248add..73bcc7b0f 100644 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -780,7 +780,7 @@ class TerminalImportSession(importer.ImportSession): # skip empty albums (coming from a previous failed import session) if task.is_album: - real_duplicates = filter(len, found_duplicates) + real_duplicates = [dup for dup in found_duplicates if dup.items()] if not real_duplicates: log.info("All duplicates are empty, we ignore them") task.should_remove_duplicates = True From adec09de96d1651df0a66a47f27d279805f71d33 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Mon, 23 Mar 2015 19:15:14 +0100 Subject: [PATCH 127/129] Fix outdated docstring in test.TestHelper --- test/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helper.py b/test/helper.py index bb32c1c87..3de31e2df 100644 --- a/test/helper.py +++ b/test/helper.py @@ -15,7 +15,7 @@ """This module includes various helpers that provide fixtures, capture information or mock the environment. -- The `control_stdin` and `capture_output` context managers allow one to +- The `control_stdin` and `capture_stdout` context managers allow one to interact with the user interface. - `has_program` checks the presence of a command on the system. From 8f5bae26fd557dd886f192e90fe0ba3c2c4b739b Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 25 Mar 2015 10:38:46 +0100 Subject: [PATCH 128/129] Smartplaylists: improve tests & code modularization --- beetsplug/cue.py | 54 +++++++++++++++++++++++++++++ beetsplug/smartplaylist.py | 16 ++++----- test/test_smartplaylist.py | 70 +++++++++++++++++++++----------------- 3 files changed, 101 insertions(+), 39 deletions(-) create mode 100644 beetsplug/cue.py diff --git a/beetsplug/cue.py b/beetsplug/cue.py new file mode 100644 index 000000000..25205f8e8 --- /dev/null +++ b/beetsplug/cue.py @@ -0,0 +1,54 @@ +# Copyright 2015 Bruno Cauet +# Split an album-file in tracks thanks a cue file + +import subprocess +from os import path +from glob import glob + +from beets.util import command_output, displayable_path +from beets.plugins import BeetsPlugin +from beets.autotag import TrackInfo + + +class CuePlugin(BeetsPlugin): + def __init__(self): + super(CuePlugin, self).__init__() + # this does not seem supported by shnsplit + self.config.add({ + 'keep_before': .1, + 'keep_after': .9, + }) + + # self.register_listener('import_task_start', self.look_for_cues) + + def candidates(self, items, artist, album, va_likely): + import pdb + pdb.set_trace() + + def item_candidates(self, item, artist, album): + dir = path.dirname(item.path) + cues = glob.glob(path.join(dir, "*.cue")) + if not cues: + return + if len(cues) > 1: + self._log.info(u"Found multiple cue files doing nothing: {0}", + map(displayable_path, cues)) + + cue_file = cues[0] + self._log.info("Found {} for {}", displayable_path(cue_file), item) + + try: + # careful: will ask for input in case of conflicts + command_output(['shnsplit', '-f', cue_file, item.path]) + except (subprocess.CalledProcessError, OSError): + self._log.exception(u'shnsplit execution failed') + return + + tracks = glob(path.join(dir, "*.wav")) + self._log.info("Generated {0} tracks", len(tracks)) + for t in tracks: + title = "dunno lol" + track_id = "wtf" + index = int(path.basename(t)[len("split-track"):-len(".wav")]) + yield TrackInfo(title, track_id, index=index, artist=artist) + # generate TrackInfo instances diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 34fa94979..8889a2534 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -126,20 +126,20 @@ class SmartPlaylistPlugin(BeetsPlugin): self._unmatched_playlists.add(playlist_data) + def matches(self, model, query, album_query): + if album_query and isinstance(model, Album): + return album_query.match(model) + if query and isinstance(model, Item): + return query.match(model) + return False + def db_change(self, lib, model): if self._unmatched_playlists is None: self.build_queries() for playlist in self._unmatched_playlists: n, (q, _), (a_q, _) = playlist - if a_q and isinstance(model, Album): - matches = a_q.match(model) - elif q and isinstance(model, Item): - matches = q.match(model) or q.match(model.get_album()) - else: - matches = False - - if matches: + if self.matches(model, q, a_q): self._log.debug("{0} will be updated because of {1}", n, model) self._matched_playlists.add(playlist) self.register_listener('cli_exit', self.update_playlists) diff --git a/test/test_smartplaylist.py b/test/test_smartplaylist.py index 4af1cfeb6..ddfa8a156 100644 --- a/test/test_smartplaylist.py +++ b/test/test_smartplaylist.py @@ -92,45 +92,53 @@ class SmartPlaylistTest(unittest.TestCase): asseq(sorts["mixed"], MultipleSort([S('year'), S('genre'), S('id', False)])) + def test_matches(self): + spl = SmartPlaylistPlugin() + + a = MagicMock(Album) + i = MagicMock(Item) + + self.assertFalse(spl.matches(i, None, None)) + self.assertFalse(spl.matches(a, None, None)) + + query = Mock() + query.match.side_effect = {i: True}.__getitem__ + self.assertTrue(spl.matches(i, query, None)) + self.assertFalse(spl.matches(a, query, None)) + + a_query = Mock() + a_query.match.side_effect = {a: True}.__getitem__ + self.assertFalse(spl.matches(i, None, a_query)) + self.assertTrue(spl.matches(a, None, a_query)) + + self.assertTrue(spl.matches(i, query, a_query)) + self.assertTrue(spl.matches(a, query, a_query)) + def test_db_changes(self): spl = SmartPlaylistPlugin() - i1 = MagicMock(Item) - i2 = MagicMock(Item) - a = MagicMock(Album) - i1.get_album.return_value = a - - q1 = Mock() - q1.matches.side_effect = {i1: False, i2: False}.__getitem__ - a_q1 = Mock() - a_q1.matches.side_effect = {a: True}.__getitem__ - q2 = Mock() - q2.matches.side_effect = {i1: False, i2: True}.__getitem__ - - pl1 = '1', (q1, None), (a_q1, None) - pl2 = '2', (None, None), (a_q1, None) - pl3 = '3', (q2, None), (None, None) + nones = None, None + pl1 = '1', ('q1', None), nones + pl2 = '2', ('q2', None), nones + pl3 = '3', ('q3', None), nones spl._unmatched_playlists = set([pl1, pl2, pl3]) spl._matched_playlists = set() - spl.db_change(None, i1) - self.assertEqual(spl._unmatched_playlists, set([pl2])) + + spl.matches = Mock(return_value=False) + spl.db_change(None, "nothing") + self.assertEqual(spl._unmatched_playlists, set([pl1, pl2, pl3])) + self.assertEqual(spl._matched_playlists, set()) + + spl.matches.side_effect = lambda _, q, __: q == 'q3' + spl.db_change(None, "matches 3") + self.assertEqual(spl._unmatched_playlists, set([pl1, pl2])) + self.assertEqual(spl._matched_playlists, set([pl3])) + + spl.matches.side_effect = lambda _, q, __: q == 'q1' + spl.db_change(None, "matches 3") self.assertEqual(spl._matched_playlists, set([pl1, pl3])) - - spl._unmatched_playlists = set([pl1, pl2, pl3]) - spl._matched_playlists = set() - spl.db_change(None, i2) self.assertEqual(spl._unmatched_playlists, set([pl2])) - self.assertEqual(spl._matched_playlists, set([pl1, pl3])) - - spl._unmatched_playlists = set([pl1, pl2, pl3]) - spl._matched_playlists = set() - spl.db_change(None, a) - self.assertEqual(spl._unmatched_playlists, set([pl3])) - self.assertEqual(spl._matched_playlists, set([pl1, pl2])) - spl.db_change(None, i2) - self.assertEqual(spl._unmatched_playlists, set()) - self.assertEqual(spl._matched_playlists, set([pl1, pl2, pl3])) def test_playlist_update(self): spl = SmartPlaylistPlugin() From 86559bcb1ad12717caacd48c0187bc4415d26a59 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Wed, 25 Mar 2015 10:56:17 +0100 Subject: [PATCH 129/129] Implement repr() on Query & its subclasses. This should help with #1359-style issues. --- beets/dbcore/query.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 330913706..7dc897412 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -73,6 +73,9 @@ class Query(object): """ raise NotImplementedError + def __repr__(self): + return "{0.__class__.__name__}()".format(self) + def __eq__(self, other): return type(self) == type(other) @@ -112,6 +115,10 @@ class FieldQuery(Query): def match(self, item): return self.value_match(self.pattern, item.get(self.field)) + def __repr__(self): + return ("{0.__class__.__name__}({0.field!r}, {0.pattern!r}, " + "{0.fast})".format(self)) + def __eq__(self, other): return super(FieldQuery, self).__eq__(other) and \ self.field == other.field and self.pattern == other.pattern @@ -145,6 +152,9 @@ class NoneQuery(FieldQuery): except KeyError: return True + def __repr__(self): + return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) + class StringFieldQuery(FieldQuery): """A FieldQuery that converts values to strings before matching @@ -349,6 +359,9 @@ class CollectionQuery(Query): clause = (' ' + joiner + ' ').join(clause_parts) return clause, subvals + def __repr__(self): + return "{0.__class__.__name__}({0.subqueries})".format(self) + def __eq__(self, other): return super(CollectionQuery, self).__eq__(other) and \ self.subqueries == other.subqueries @@ -384,6 +397,10 @@ class AnyFieldQuery(CollectionQuery): return True return False + def __repr__(self): + return ("{0.__class__.__name__}({0.pattern!r}, {0.fields}, " + "{0.query_class.__name__})".format(self)) + def __eq__(self, other): return super(AnyFieldQuery, self).__eq__(other) and \ self.query_class == other.query_class