diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index b87b41644..ae6151f34 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -52,15 +52,24 @@ class FormattedMapping(Mapping): The accessor `mapping[key]` returns the formatted version of `model[key]` as a unicode string. + The `included_keys` parameter allows filtering the fields that are + returned. By default all fields are returned. Limiting to specific keys can + avoid expensive per-item database queries. + If `for_path` is true, all path separators in the formatted values are replaced. """ - def __init__(self, model, for_path=False, compute_keys=True): + ALL_KEYS = '*' + + def __init__(self, model, included_keys=ALL_KEYS, for_path=False): self.for_path = for_path self.model = model - if compute_keys: - self.model_keys = model.keys(True) + if included_keys == self.ALL_KEYS: + # Performance note: this triggers a database query. + self.model_keys = self.model.keys(True) + else: + self.model_keys = included_keys def __getitem__(self, key): if key in self.model_keys: @@ -605,11 +614,11 @@ class Model(object): _formatter = FormattedMapping - def formatted(self, for_path=False): + def formatted(self, included_keys=_formatter.ALL_KEYS, for_path=False): """Get a mapping containing all values on this object formatted as human-readable unicode strings. """ - return self._formatter(self, for_path) + return self._formatter(self, included_keys, for_path) def evaluate_template(self, template, for_path=False): """Evaluate a template (a string or a `Template` object) using @@ -619,7 +628,7 @@ class Model(object): # Perform substitution. if isinstance(template, six.string_types): template = functemplate.template(template) - return template.substitute(self.formatted(for_path), + return template.substitute(self.formatted(for_path=for_path), self._template_funcs()) # Parsing. diff --git a/beets/library.py b/beets/library.py index df20c7a3d..9e459a3f2 100644 --- a/beets/library.py +++ b/beets/library.py @@ -374,12 +374,19 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): Album-level fields take precedence if `for_path` is true. """ - def __init__(self, item, for_path=False): + ALL_KEYS = '*' + + def __init__(self, item, included_keys=ALL_KEYS, for_path=False): # We treat album and item keys specially here, # so exclude transitive album keys from the model's keys. - super(FormattedItemMapping, self).__init__(item, for_path, - compute_keys=False) - self.model_keys = item.keys(computed=True, with_album=False) + super(FormattedItemMapping, self).__init__(item, included_keys=[], + for_path=for_path) + self.included_keys = included_keys + if included_keys == self.ALL_KEYS: + # Performance note: this triggers a database query. + self.model_keys = item.keys(computed=True, with_album=False) + else: + self.model_keys = included_keys self.item = item @lazy_property @@ -390,10 +397,14 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def album_keys(self): album_keys = [] if self.album: - for key in self.album.keys(computed=True): - if key in Album.item_keys \ - or key not in self.item._fields.keys(): - album_keys.append(key) + if self.included_keys == self.ALL_KEYS: + # Performance note: this triggers a database query. + for key in self.album.keys(computed=True): + if key in Album.item_keys \ + or key not in self.item._fields.keys(): + album_keys.append(key) + else: + album_keys = self.included_keys return album_keys @property @@ -422,12 +433,15 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # `artist` and `albumartist` fields fall back to one another. # This is helpful in path formats when the album artist is unset # on as-is imports. - if key == 'artist' and not value: - return self._get('albumartist') - elif key == 'albumartist' and not value: - return self._get('artist') - else: - return value + try: + if key == 'artist' and not value: + return self._get('albumartist') + elif key == 'albumartist' and not value: + return self._get('artist') + except KeyError: + pass + + return value def __iter__(self): return iter(self.all_keys) @@ -1703,7 +1717,7 @@ class DefaultTemplateFunctions(object): return res # Flatten disambiguation value into a string. - disam_value = album.formatted(True).get(disambiguator) + disam_value = album.formatted(for_path=True).get(disambiguator) # Return empty string if disambiguator is empty. if disam_value: diff --git a/beetsplug/export.py b/beetsplug/export.py index 3ce9ab887..9aabaea05 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -26,8 +26,9 @@ from xml.etree import ElementTree from datetime import datetime, date from beets.plugins import BeetsPlugin from beets import ui +from beets import util import mediafile -from beetsplug.info import make_key_filter, library_data, tag_data +from beetsplug.info import library_data, tag_data class ExportEncoder(json.JSONEncoder): @@ -129,16 +130,18 @@ class ExportPlugin(BeetsPlugin): for keys in opts.included_keys: included_keys.extend(keys.split(',')) - key_filter = make_key_filter(included_keys) - for data_emitter in data_collector(lib, ui.decargs(args)): try: - data, item = data_emitter() + data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue - data = key_filter(data) + for key, value in data.items(): + if isinstance(value, bytes): + data[key] = util.displayable_path(value) + + items += [data] if file_format_is_line_based: export_format.export(data, **format_options) diff --git a/beetsplug/info.py b/beetsplug/info.py index 05d215e7b..828b0a81a 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -19,7 +19,6 @@ from __future__ import division, absolute_import, print_function import os -import re from beets.plugins import BeetsPlugin from beets import ui @@ -42,15 +41,29 @@ def tag_data(lib, args): yield tag_data_emitter(item.path) +def tag_fields(): + fields = set(mediafile.MediaFile.readable_fields()) + fields.add('art') + return fields + + def tag_data_emitter(path): - def emitter(): - fields = list(mediafile.MediaFile.readable_fields()) - fields.remove('images') + def emitter(included_keys): + if included_keys == '*': + fields = tag_fields() + else: + fields = included_keys + if 'images' in fields: + # We can't serialize the image data. + fields.remove('images') mf = mediafile.MediaFile(syspath(path)) tags = {} for field in fields: - tags[field] = getattr(mf, field) - tags['art'] = mf.art is not None + if field == 'art': + tags[field] = mf.art is not None + else: + tags[field] = getattr(mf, field, None) + # create a temporary Item to take advantage of __format__ item = Item.from_path(syspath(path)) @@ -64,8 +77,8 @@ def library_data(lib, args): def library_data_emitter(item): - def emitter(): - data = dict(item.formatted()) + def emitter(included_keys): + data = dict(item.formatted(included_keys=included_keys)) return data, item return emitter @@ -185,18 +198,16 @@ class InfoPlugin(BeetsPlugin): included_keys.extend(keys.split(',')) # Drop path even if user provides it multiple times included_keys = [k for k in included_keys if k != 'path'] - key_filter = make_key_filter(included_keys) first = True summary = {} for data_emitter in data_collector(lib, ui.decargs(args)): try: - data, item = data_emitter() + data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue - data = key_filter(data) if opts.summarize: update_summary(summary, data) else: @@ -211,34 +222,3 @@ class InfoPlugin(BeetsPlugin): if opts.summarize: print_data(summary) - - -def make_key_filter(include): - """Return a function that filters a dictionary. - - The returned filter takes a dictionary and returns another - dictionary that only includes the key-value pairs where the key - glob-matches one of the keys in `include`. - """ - # By default, if no field inclusions are specified, include - # everything but `path`. - if not include: - def filter_(data): - return {k: v for k, v in data.items() - if k != 'path'} - return filter_ - - matchers = [] - for key in include: - key = re.escape(key) - key = key.replace(r'\*', '.*') - matchers.append(re.compile(key + '$')) - - def filter_(data): - filtered = {} - for key, value in data.items(): - if any([m.match(key) for m in matchers]): - filtered[key] = value - return filtered - - return filter_ diff --git a/docs/changelog.rst b/docs/changelog.rst index 88338f7a7..6caed75c6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -212,6 +212,8 @@ Other new things: used to target a maximum image filesize. * :doc:`/plugins/badfiles`: Checkers can now be run during import with the ``check_on_import`` config option. +* :doc:`/plugins/export`: big speedups when `--include-keys` option is used + Thanks to :user:`ssssam`. Fixes: diff --git a/docs/plugins/info.rst b/docs/plugins/info.rst index d628fb4ad..3950cf0aa 100644 --- a/docs/plugins/info.rst +++ b/docs/plugins/info.rst @@ -20,14 +20,12 @@ your library:: If you just want to see specific properties you can use the ``--include-keys`` option to filter them. The argument is a -comma-separated list of simple glob patterns where ``*`` matches any -string. For example:: +comma-separated list of field names. For example:: - $ beet info -i 'title,mb*' beatles + $ beet info -i 'title,mb_artistid' beatles -Will only show the ``title`` property and all properties starting with -``mb``. You can add the ``-i`` option multiple times to the command -line. +Will only show the ``title`` and ``mb_artistid`` properties. You can add the +``-i`` option multiple times to the command line. Additional command-line options include: diff --git a/test/test_info.py b/test/test_info.py index 814205527..0f29621d6 100644 --- a/test/test_info.py +++ b/test/test_info.py @@ -92,17 +92,6 @@ class InfoTest(unittest.TestCase, TestHelper): self.assertIn(u'title: [various]', out) self.remove_mediafile_fixtures() - def test_include_pattern(self): - item, = self.add_item_fixtures() - item.album = 'xxxx' - item.store() - - out = self.run_with_output('info', '--library', 'album:xxxx', - '--include-keys', '*lbu*') - self.assertIn(displayable_path(item.path), out) - self.assertNotIn(u'title:', out) - self.assertIn(u'album: xxxx', out) - def test_custom_format(self): self.add_item_fixtures() out = self.run_with_output('info', '--library', '--format',