From 9c7a313b0d6ce79529f21eba72386c3f60120828 Mon Sep 17 00:00:00 2001 From: Patrick Heneghan Date: Tue, 16 Oct 2018 18:51:39 +0100 Subject: [PATCH 001/339] Remove `default` arg for `lastgenre --force` flag --- beetsplug/lastgenre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index 2f660206e..c4b973620 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -373,7 +373,7 @@ class LastGenrePlugin(plugins.BeetsPlugin): lastgenre_cmd = ui.Subcommand('lastgenre', help=u'fetch genres') lastgenre_cmd.parser.add_option( u'-f', u'--force', dest='force', - action='store_true', default=False, + action='store_true', help=u're-download genre when already present' ) lastgenre_cmd.parser.add_option( From 3e0b2ad146a437aeb6d6df32cba5ca0f48bbe5b9 Mon Sep 17 00:00:00 2001 From: Vladimir Zhelezov Date: Mon, 10 Dec 2018 08:40:32 +0100 Subject: [PATCH 002/339] Fix #2826 Test for major Python version and use inspect.getargspec() or inspect.getfullargspec() respectively to silence deprecation warnings in Python 3 --- beets/plugins.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 6dec7ef2a..69784d269 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -127,7 +127,10 @@ class BeetsPlugin(object): value after the function returns). Also determines which params may not be sent for backwards-compatibility. """ - argspec = inspect.getargspec(func) + if six.PY2: + argspec = inspect.getargspec(func) + else: + argspec = inspect.getfullargspec(func) @wraps(func) def wrapper(*args, **kwargs): From 0df0dfe98660c7a73cba7085321cd728c79d0e43 Mon Sep 17 00:00:00 2001 From: Eric Fischer Date: Thu, 10 Jan 2019 21:01:03 -0500 Subject: [PATCH 003/339] Maintain python 2 compatibility Jellyfish is no longer python 2 compatible as of release 0.7.0. By pinning the previous release, beets is still able to be installed and run on python 2 systems without issue. --- docs/changelog.rst | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c1382b61c..d6b265fba 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,6 +67,7 @@ Changes: Fixes: +* Pin jellyfish requirement to version 0.6.0 to maintain python 2 compatibility. * A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks contained in data files :bug:`3021` * Restore iTunes Store album art source, and remove the dependency on diff --git a/setup.py b/setup.py index 19c03041a..c674db5a0 100755 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', - 'jellyfish', + 'jellyfish==0.6.0', ] + (['colorama'] if (sys.platform == 'win32') else []) + (['enum34>=1.0.4'] if sys.version_info < (3, 4, 0) else []), From 77fd5ee5488813654cf9881b460865eff55bf2f0 Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 8 Feb 2019 00:05:07 +0100 Subject: [PATCH 004/339] keep discogs requests below rate limit --- beetsplug/discogs.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 5b11b9617..4d25fca50 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -61,6 +61,8 @@ class DiscogsPlugin(BeetsPlugin): self.config['user_token'].redact = True self.discogs_client = None self.register_listener('import_begin', self.setup) + self.rate_limit_per_minute = 25 + self.last_request_timestamp = 0 def setup(self, session=None): """Create the `discogs_client` field. Authenticate if necessary. @@ -71,6 +73,7 @@ class DiscogsPlugin(BeetsPlugin): # Try using a configured user token (bypassing OAuth login). user_token = self.config['user_token'].as_str() if user_token: + self.rate_limit_per_minute = 60 self.discogs_client = Client(USER_AGENT, user_token=user_token) return @@ -88,6 +91,20 @@ class DiscogsPlugin(BeetsPlugin): self.discogs_client = Client(USER_AGENT, c_key, c_secret, token, secret) + def _time_to_next_request(self): + seconds_between_requests = 60 / self.rate_limit_per_minute + seconds_since_last_request = time.time() - self.last_request_timestamp + seconds_to_wait = seconds_between_requests - seconds_since_last_request + if seconds_to_wait > 0: + return seconds_to_wait + return 0 + + def wait_for_rate_limiter(self): + time_to_next_request = self._time_to_next_request() + if time_to_next_request > 0: + self._log.debug('hit rate limit, waiting for {0} seconds', time_to_next_request) + time.sleep(time_to_next_request) + def reset_auth(self): """Delete token file & redo the auth steps. """ @@ -206,9 +223,13 @@ class DiscogsPlugin(BeetsPlugin): # Strip medium information from query, Things like "CD1" and "disk 1" # can also negate an otherwise positive result. query = re.sub(br'(?i)\b(CD|disc)\s*\d+', b'', query) + + self.wait_for_rate_limiter() try: releases = self.discogs_client.search(query, type='release').page(1) + self.last_request_timestamp = time.time() + except CONNECTION_ERRORS: self._log.debug(u"Communication error while searching for {0!r}", query, exc_info=True) @@ -222,8 +243,11 @@ class DiscogsPlugin(BeetsPlugin): """ self._log.debug(u'Searching for master release {0}', master_id) result = Master(self.discogs_client, {'id': master_id}) + + self.wait_for_rate_limiter() try: year = result.fetch('year') + self.last_request_timestamp = time.time() return year except DiscogsAPIError as e: if e.status_code != 404: From 9bc3898951886a42f57f03802fc238e6b599a26d Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 8 Feb 2019 01:02:33 +0100 Subject: [PATCH 005/339] add request_finished function, rename wait_for_rate_limiter to request_start, add doc and changelog --- beetsplug/discogs.py | 24 +++++++++++++++--------- docs/changelog.rst | 3 ++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 4d25fca50..b9a832c82 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -73,6 +73,7 @@ class DiscogsPlugin(BeetsPlugin): # Try using a configured user token (bypassing OAuth login). user_token = self.config['user_token'].as_str() if user_token: + # rate limit for authenticated users is 60 per minute self.rate_limit_per_minute = 60 self.discogs_client = Client(USER_AGENT, user_token=user_token) return @@ -95,16 +96,22 @@ class DiscogsPlugin(BeetsPlugin): seconds_between_requests = 60 / self.rate_limit_per_minute seconds_since_last_request = time.time() - self.last_request_timestamp seconds_to_wait = seconds_between_requests - seconds_since_last_request - if seconds_to_wait > 0: - return seconds_to_wait - return 0 + return seconds_to_wait - def wait_for_rate_limiter(self): + def request_start(self): + """wait for rate limit if needed + """ time_to_next_request = self._time_to_next_request() if time_to_next_request > 0: - self._log.debug('hit rate limit, waiting for {0} seconds', time_to_next_request) + self._log.debug('hit rate limit, waiting for {0} seconds', + time_to_next_request) time.sleep(time_to_next_request) + def request_finished(self): + """update timestamp for rate limiting + """ + self.last_request_timestamp = time.time() + def reset_auth(self): """Delete token file & redo the auth steps. """ @@ -224,11 +231,10 @@ class DiscogsPlugin(BeetsPlugin): # can also negate an otherwise positive result. query = re.sub(br'(?i)\b(CD|disc)\s*\d+', b'', query) - self.wait_for_rate_limiter() + self.request_start() try: releases = self.discogs_client.search(query, type='release').page(1) - self.last_request_timestamp = time.time() except CONNECTION_ERRORS: self._log.debug(u"Communication error while searching for {0!r}", @@ -244,10 +250,10 @@ class DiscogsPlugin(BeetsPlugin): self._log.debug(u'Searching for master release {0}', master_id) result = Master(self.discogs_client, {'id': master_id}) - self.wait_for_rate_limiter() + self.request_start() try: year = result.fetch('year') - self.last_request_timestamp = time.time() + self.request_finished() return year except DiscogsAPIError as e: if e.status_code != 404: diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cab4a1e0..19a9aea32 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -65,7 +65,8 @@ New features: :bug:`3123` * :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which passes that flag to ipfs. Thanks to :user:`wildthyme`. - +* :doc:`/plugins/discogs`: The plugin has rate limiting for the discogs API now. + :bug:`3081` Changes: From 5ace66775714c4d50cd04826b7a3bbb380731d5a Mon Sep 17 00:00:00 2001 From: jan Date: Fri, 8 Feb 2019 01:09:07 +0100 Subject: [PATCH 006/339] add forgotten request_finished --- beetsplug/discogs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index b9a832c82..a5208f4f8 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -235,6 +235,7 @@ class DiscogsPlugin(BeetsPlugin): try: releases = self.discogs_client.search(query, type='release').page(1) + self.request_finished() except CONNECTION_ERRORS: self._log.debug(u"Communication error while searching for {0!r}", From f54042f194a91b0590d798338a6e43b8a3c4d20d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 8 Feb 2019 18:18:30 -0800 Subject: [PATCH 007/339] Make a comment into a full sentence --- beetsplug/discogs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index a5208f4f8..d7797e409 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -73,7 +73,8 @@ class DiscogsPlugin(BeetsPlugin): # Try using a configured user token (bypassing OAuth login). user_token = self.config['user_token'].as_str() if user_token: - # rate limit for authenticated users is 60 per minute + # The rate limit for authenticated users goes up to 60 + # requests per minute. self.rate_limit_per_minute = 60 self.discogs_client = Client(USER_AGENT, user_token=user_token) return From 90904014890521fd6ebfaf83a083627c359f171e Mon Sep 17 00:00:00 2001 From: Vinicius Massuchetto Date: Wed, 13 Feb 2019 07:48:14 -0200 Subject: [PATCH 008/339] added beets-ydl plugin --- docs/plugins/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 6bf50e227..773217891 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -254,6 +254,8 @@ Here are a few of the plugins written by the beets community: * `beets-barcode`_ lets you scan or enter barcodes for physical media to search for their metadata. +* `beets-ydl`_ download audio from youtube-dl sources and import into beets + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -273,3 +275,4 @@ Here are a few of the plugins written by the beets community: .. _whatlastgenre: https://github.com/YetAnotherNerd/whatlastgenre/tree/master/plugin/beets .. _beets-usertag: https://github.com/igordertigor/beets-usertag .. _beets-popularity: https://github.com/abba23/beets-popularity +.. _beets-ydl: https://github.com/vmassuchetto/beets-ydl From bc5b15f27757c92a410956e7e4f388aeabd82900 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Thu, 14 Feb 2019 15:52:55 +0100 Subject: [PATCH 009/339] library: Pass try_write() kwargs directly to write() This avoids duplication of the kwargs and their default values. --- beets/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index 1e46fe5ef..82ba4141d 100644 --- a/beets/library.py +++ b/beets/library.py @@ -657,14 +657,14 @@ class Item(LibModel): self.mtime = self.current_mtime() plugins.send('after_write', item=self, path=path) - def try_write(self, path=None, tags=None): + def try_write(self, *args, **kwargs): """Calls `write()` but catches and logs `FileOperationError` exceptions. Returns `False` an exception was caught and `True` otherwise. """ try: - self.write(path, tags) + self.write(*args, **kwargs) return True except FileOperationError as exc: log.error(u"{0}", exc) From 305bb640862d978fdeee0aa09c768ac95acd14c7 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Thu, 14 Feb 2019 15:53:32 +0100 Subject: [PATCH 010/339] library: Allow overriding global id3v23 option in write() --- beets/library.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/beets/library.py b/beets/library.py index 82ba4141d..16db1e974 100644 --- a/beets/library.py +++ b/beets/library.py @@ -611,7 +611,7 @@ class Item(LibModel): self.path = read_path - def write(self, path=None, tags=None): + def write(self, path=None, tags=None, id3v23=None): """Write the item's metadata to a media file. All fields in `_media_fields` are written to disk according to @@ -623,6 +623,9 @@ class Item(LibModel): `tags` is a dictionary of additional metadata the should be written to the file. (These tags need not be in `_media_fields`.) + `id3v23` will override the global `id3v23` config option if it is + set to something other than `None`. + Can raise either a `ReadError` or a `WriteError`. """ if path is None: @@ -630,6 +633,9 @@ class Item(LibModel): else: path = normpath(path) + if id3v23 is None: + id3v23 = beets.config['id3v23'].get(bool) + # Get the data to write to the file. item_tags = dict(self) item_tags = {k: v for k, v in item_tags.items() @@ -640,8 +646,7 @@ class Item(LibModel): # Open the file. try: - mediafile = MediaFile(syspath(path), - id3v23=beets.config['id3v23'].get(bool)) + mediafile = MediaFile(syspath(path), id3v23=id3v23) except UnreadableFileError as exc: raise ReadError(path, exc) From 53b63443fbb1a48341025c941b479e401f2c13d4 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Thu, 14 Feb 2019 23:32:40 +0100 Subject: [PATCH 011/339] art: Allow overriding id3v23 in embed_item() --- beets/art.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/art.py b/beets/art.py index 979a6f722..4a9ea58c7 100644 --- a/beets/art.py +++ b/beets/art.py @@ -51,7 +51,8 @@ def get_art(log, item): def embed_item(log, item, imagepath, maxwidth=None, itempath=None, - compare_threshold=0, ifempty=False, as_album=False): + compare_threshold=0, ifempty=False, as_album=False, + id3v23=None): """Embed an image into the item's media file. """ # Conditions and filters. @@ -80,7 +81,7 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, image.mime_type) return - item.try_write(path=itempath, tags={'images': [image]}) + item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23) def embed_album(log, album, maxwidth=None, quiet=False, From 7afeb9b2ace75c3d9d00e5f6858989fa763b755a Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Thu, 14 Feb 2019 19:18:02 +0100 Subject: [PATCH 012/339] convert: Add id3v23 config option to convert plugin --- beetsplug/convert.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 3c9080d1f..303563a7a 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -116,6 +116,7 @@ class ConvertPlugin(BeetsPlugin): u'pretend': False, u'threads': util.cpu_count(), u'format': u'mp3', + u'id3v23': u'inherit', u'formats': { u'aac': { u'command': u'ffmpeg -i $source -y -vn -acodec aac ' @@ -316,8 +317,12 @@ class ConvertPlugin(BeetsPlugin): if pretend: continue + id3v23 = self.config['id3v23'].as_choice([True, False, 'inherit']) + if id3v23 == 'inherit': + id3v23 = None + # Write tags from the database to the converted file. - item.try_write(path=converted) + item.try_write(path=converted, id3v23=id3v23) if keep_new: # If we're keeping the transcoded file, read it again (after @@ -332,7 +337,7 @@ class ConvertPlugin(BeetsPlugin): self._log.debug(u'embedding album art from {}', util.displayable_path(album.artpath)) art.embed_item(self._log, item, album.artpath, - itempath=converted) + itempath=converted, id3v23=id3v23) if keep_new: plugins.send('after_convert', item=item, From 057904648732fb949886b63ddf4bd03d5e9d56ad Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Thu, 14 Feb 2019 23:37:35 +0100 Subject: [PATCH 013/339] docs: Add new id3v23 config option to convert plugin documentation --- docs/plugins/convert.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index a631f7891..1a487cdee 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -68,6 +68,8 @@ file. The available options are: - **dest**: The directory where the files will be converted (or copied) to. Default: none. - **embed**: Embed album art in converted items. Default: ``yes``. +- **id3v23**: Can be used to override the global ``id3v23`` option. Default: + ``inherit``. - **max_bitrate**: All lossy files with a higher bitrate will be transcoded and those with a lower bitrate will simply be copied. Note that this does not guarantee that all converted files will have a lower From 72f837b0cc3ca3ceacaa3e41203db80e40f76de2 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 15 Feb 2019 13:35:26 +0100 Subject: [PATCH 014/339] docs: Add changelog entry regarding convert plugin's id3v23 option --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2e9b751fe..b44c0e817 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,10 @@ New features: relevant releases according to the :ref:`preferred` configuration options. Thanks to :user:`archer4499`. :bug:`3017` +* :doc:`/plugins/convert`: The plugin now has a ``id3v23`` option that allows + to override the global ``id3v23`` option. + Thanks to :user:`Holzhaus`. + :bug:`3104` * A new ``aunique`` configuration option allows setting default options for the :ref:`aunique` template function. * The ``albumdisambig`` field no longer includes the MusicBrainz release group From 9ca80dd3fd4199f14c571ae16a1a8a3eb57c6510 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 15 Feb 2019 23:56:21 +0000 Subject: [PATCH 015/339] Lock munkres to 1.0.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 19c03041a..cda52f360 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ setup( install_requires=[ 'six>=1.9', 'mutagen>=1.33', - 'munkres', + 'munkres~=1.0.0', 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', From a80a07f093c3a5f77446bd1b6e34bbc4d6d581e3 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Tue, 10 Jan 2017 14:42:15 +0000 Subject: [PATCH 016/339] playlist: Add playlist plugin Adds M3U playlist support as a query to beets and thus partially resolves issue #123. The implementation is heavily based on #2380 by Robin McCorkell. It supports referencing playlists by absolute path: $ beet ls playlist:/path/to/someplaylist.m3u It also supports referencing playlists by name. The playlist is then seached in the playlist_dir and the ".m3u" extension is appended to the name: $ beet ls playlist:anotherplaylist The configuration for the plugin looks like this: playlist: relative_to: library playlist_dir: /path/to/playlists The relative_to option specifies how relative paths in playlists are handled. By default, paths are relative to the "library" directory. It also possible to make them relative to the "playlist" or set the option or set it to a fixed path. --- beetsplug/playlist.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 beetsplug/playlist.py diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py new file mode 100644 index 000000000..624791ee4 --- /dev/null +++ b/beetsplug/playlist.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +import os +import beets + + +class PlaylistQuery(beets.dbcore.FieldQuery): + """Matches files listed by a playlist file. + """ + def __init__(self, field, pattern, fast=False): + super(PlaylistQuery, self).__init__(field, pattern, fast) + config = beets.config['playlist'] + + # Get the full path to the playlist + if os.path.isabs(beets.util.syspath(pattern)): + playlist_path = pattern + else: + playlist_path = os.path.abspath(os.path.join( + config['playlist_dir'].as_filename(), + '{0}.m3u'.format(pattern), + )) + + if config['relative_to'].get() == 'library': + relative_to = beets.config['directory'].as_filename() + elif config['relative_to'].get() == 'playlist': + relative_to = os.path.dirname(playlist_path) + else: + relative_to = config['relative_to'].as_filename() + relative_to = beets.util.bytestring_path(relative_to) + + self.paths = [] + with open(beets.util.syspath(playlist_path), 'rb') as f: + for line in f: + if line[0] == '#': + # ignore comments, and extm3u extension + continue + + self.paths.append(beets.util.normpath( + os.path.join(relative_to, line.rstrip()) + )) + + def match(self, item): + return item.path in self.paths + + +class PlaylistType(beets.dbcore.types.String): + """Custom type for playlist query. + """ + query = PlaylistQuery + + +class PlaylistPlugin(beets.plugins.BeetsPlugin): + item_types = {'playlist': PlaylistType()} + + def __init__(self): + super(PlaylistPlugin, self).__init__() + self.config.add({ + 'playlist_dir': '.', + 'relative_to': 'library', + }) From 19b92e1199439017e05bf79408a863a91c34972a Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 15 Feb 2019 18:44:58 +0100 Subject: [PATCH 017/339] playlist: Improve speed in PlaylistQuery class Implement the col_clause method for faster, sqlite-based querying. This will only make a difference if the "fast" kwarg is set to True. --- beetsplug/playlist.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 624791ee4..654e3e1d0 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -19,7 +19,7 @@ import beets class PlaylistQuery(beets.dbcore.FieldQuery): """Matches files listed by a playlist file. """ - def __init__(self, field, pattern, fast=False): + def __init__(self, field, pattern, fast=True): super(PlaylistQuery, self).__init__(field, pattern, fast) config = beets.config['playlist'] @@ -51,6 +51,14 @@ class PlaylistQuery(beets.dbcore.FieldQuery): os.path.join(relative_to, line.rstrip()) )) + def col_clause(self): + if not self.paths: + # Playlist is empty + return '0', () + clause = 'BYTELOWER(path) IN ({0})'.format( + ', '.join('BYTELOWER(?)' for path in self.paths)) + return clause, (beets.library.BLOB_TYPE(p) for p in self.paths) + def match(self, item): return item.path in self.paths From cc501be2d9c9442eeaffb7e9681b11b58671abc7 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 15 Feb 2019 23:06:36 +0100 Subject: [PATCH 018/339] docs: Add documentation for the playlist plugin --- docs/plugins/index.rst | 2 ++ docs/plugins/playlist.rst | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 docs/plugins/playlist.rst diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 6bf50e227..e51354dac 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -81,6 +81,7 @@ like this:: mpdupdate permissions play + playlist plexupdate random replaygain @@ -158,6 +159,7 @@ Interoperability * :doc:`mpdupdate`: Automatically notifies `MPD`_ whenever the beets library changes. * :doc:`play`: Play beets queries in your music player. +* :doc:`playlist`: Use M3U playlists tp query the beets library. * :doc:`plexupdate`: Automatically notifies `Plex`_ whenever the beets library changes. * :doc:`smartplaylist`: Generate smart playlists based on beets queries. diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst new file mode 100644 index 000000000..0a4e797c3 --- /dev/null +++ b/docs/plugins/playlist.rst @@ -0,0 +1,37 @@ +Smart Playlist Plugin +===================== + +``playlist`` is a plugin to use playlists in m3u format. + +To use it, enable the ``playlist`` plugin in your configuration +(see :ref:`using-plugins`). +Then configure your playlists like this:: + + playlist: + relative_to: ~/Music + playlist_dir: ~/.mpd/playlists + +It is possible to query the library based on a playlist by speicifying its +absolute path:: + + $ beet ls playlist:/path/to/someplaylist.m3u + +The plugin also supports referencing playlists by name. The playlist is then +seached in the playlist_dir and the ".m3u" extension is appended to the +name:: + + $ beet ls playlist:anotherplaylist + +Configuration +------------- + +To configure the plugin, make a ``smartplaylist:`` section in your +configuration file. In addition to the ``playlists`` described above, the +other configuration options are: + +- **playlist_dir**: Where to read playlist files from. + Default: The current working directory (i.e., ``'.'``). +- **relative_to**: Interpret paths in the playlist files relative to a base + directory. It is also possible to set it to ``playlist`` to use the + playlist's parent directory as base directory. + Default: ``library`` From d78bade30cfe39dcc0207330a31fa34195ad11b8 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 15 Feb 2019 23:07:19 +0100 Subject: [PATCH 019/339] docs: Add playlist plugin to the changelog --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2e9b751fe..a3b50af05 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,6 +14,10 @@ New features: issues with foobar2000 and Winamp. Thanks to :user:`mz2212`. :bug:`2944` +* :doc:`/plugins/playlist`: Add a plugin that can query the beets library using + M3U playlists. + Thanks to :user:`Holzhaus` and :user:`Xenopathic`. + :bug:`123` * Added whitespace padding to missing tracks dialog to improve readability. Thanks to :user:`jams2`. :bug:`2962` From 0988a2a18688a8b8e07d94e1609405c17bbe717d Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 15 Feb 2019 19:51:00 +0100 Subject: [PATCH 020/339] test: Add test suite for the playlist plugin --- test/test_playlist.py | 90 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 test/test_playlist.py diff --git a/test/test_playlist.py b/test/test_playlist.py new file mode 100644 index 000000000..176249134 --- /dev/null +++ b/test/test_playlist.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Thomas Scholtes. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import division, absolute_import, print_function + +import os +import tempfile +import unittest + +from test import _common +from test import helper + +import beets + + +class PlaylistTest(unittest.TestCase, helper.TestHelper): + def setUp(self): + self.setup_beets() + self.lib = beets.library.Library(':memory:') + + i1 = _common.item() + i1.path = beets.util.normpath('/a/b/c.mp3') + i1.title = u'some item' + i1.album = u'some album' + self.lib.add(i1) + self.lib.add_album([i1]) + + i2 = _common.item() + i2.path = beets.util.normpath('/d/e/f.mp3') + i2.title = 'another item' + i2.album = 'another album' + self.lib.add(i2) + self.lib.add_album([i2]) + + i3 = _common.item() + i3.path = beets.util.normpath('/x/y/z.mp3') + i3.title = 'yet another item' + i3.album = 'yet another album' + self.lib.add(i3) + self.lib.add_album([i3]) + + self.playlist_dir = tempfile.TemporaryDirectory() + with open(os.path.join(self.playlist_dir.name, 'test.m3u'), 'w') as f: + f.write('{0}\n'.format(beets.util.displayable_path(i1.path))) + f.write('{0}\n'.format(beets.util.displayable_path(i2.path))) + + self.config['directory'] = '/' + self.config['playlist']['relative_to'] = 'library' + self.config['playlist']['playlist_dir'] = self.playlist_dir.name + self.load_plugins('playlist') + + def tearDown(self): + self.unload_plugins() + self.playlist_dir.cleanup() + self.teardown_beets() + + def test_query_name(self): + q = u'playlist:test' + results = self.lib.items(q) + self.assertEqual(set([i.title for i in results]), set([ + u'some item', + u'another item', + ])) + + def test_query_path(self): + q = u'playlist:{0}/test.m3u'.format(self.playlist_dir.name) + results = self.lib.items(q) + self.assertEqual(set([i.title for i in results]), set([ + u'some item', + u'another item', + ])) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From f9f2fa0e266adb146b79542195c0761f59f0292f Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 15:13:55 +0100 Subject: [PATCH 021/339] playlist: Restructure playlist reading code and add error handling --- beetsplug/playlist.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 654e3e1d0..a6ab8d18b 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -13,6 +13,7 @@ # included in all copies or substantial portions of the Software. import os +import fnmatch import beets @@ -24,24 +25,33 @@ class PlaylistQuery(beets.dbcore.FieldQuery): config = beets.config['playlist'] # Get the full path to the playlist - if os.path.isabs(beets.util.syspath(pattern)): - playlist_path = pattern - else: - playlist_path = os.path.abspath(os.path.join( + playlist_paths = ( + pattern, + os.path.abspath(os.path.join( config['playlist_dir'].as_filename(), '{0}.m3u'.format(pattern), - )) - - if config['relative_to'].get() == 'library': - relative_to = beets.config['directory'].as_filename() - elif config['relative_to'].get() == 'playlist': - relative_to = os.path.dirname(playlist_path) - else: - relative_to = config['relative_to'].as_filename() - relative_to = beets.util.bytestring_path(relative_to) + )), + ) self.paths = [] - with open(beets.util.syspath(playlist_path), 'rb') as f: + for playlist_path in playlist_paths: + if not fnmatch.fnmatch(playlist_path, '*.[mM]3[uU]'): + # This is not am M3U playlist, skip this candidate + continue + + try: + f = open(beets.util.syspath(playlist_path), mode='rb') + except OSError: + continue + + if config['relative_to'].get() == 'library': + relative_to = beets.config['directory'].as_filename() + elif config['relative_to'].get() == 'playlist': + relative_to = os.path.dirname(playlist_path) + else: + relative_to = config['relative_to'].as_filename() + relative_to = beets.util.bytestring_path(relative_to) + for line in f: if line[0] == '#': # ignore comments, and extm3u extension @@ -50,6 +60,8 @@ class PlaylistQuery(beets.dbcore.FieldQuery): self.paths.append(beets.util.normpath( os.path.join(relative_to, line.rstrip()) )) + f.close() + break def col_clause(self): if not self.paths: From d52dcdd48ff10422cf9734d2980156a2832c8d7f Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 15:16:44 +0100 Subject: [PATCH 022/339] test: Add playlist testcases for nonexisting playlists --- test/test_playlist.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_playlist.py b/test/test_playlist.py index 176249134..2c7c89805 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -82,6 +82,16 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): u'another item', ])) + def test_query_name_nonexisting(self): + q = u'playlist:nonexisting'.format(self.playlist_dir.name) + results = self.lib.items(q) + self.assertEqual(set(results), set()) + + def test_query_path_nonexisting(self): + q = u'playlist:{0}/nonexisting.m3u'.format(self.playlist_dir.name) + results = self.lib.items(q) + self.assertEqual(set(results), set()) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 34cdeeefb72b23dc00e291ecf1934030089417b6 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 15:35:30 +0100 Subject: [PATCH 023/339] docs: Reword documentation of playlist plugin's relative_to option --- docs/plugins/playlist.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 0a4e797c3..1156e7f77 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -32,6 +32,7 @@ other configuration options are: - **playlist_dir**: Where to read playlist files from. Default: The current working directory (i.e., ``'.'``). - **relative_to**: Interpret paths in the playlist files relative to a base - directory. It is also possible to set it to ``playlist`` to use the - playlist's parent directory as base directory. + directory. Instead of setting it to a fixed path, it is also possible to + set it to ``playlist`` to use the playlist's parent directory or to + ``library`` to use the library directory. Default: ``library`` From d4039be9c07b118b619d0353fe844b7ea867be01 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 15:39:47 +0100 Subject: [PATCH 024/339] test: Get rid of TemporaryDirectory to restore Python 2.7 compatibility --- test/test_playlist.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 2c7c89805..abae5d969 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -16,6 +16,7 @@ from __future__ import division, absolute_import, print_function import os +import shutil import tempfile import unittest @@ -51,19 +52,19 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.lib.add(i3) self.lib.add_album([i3]) - self.playlist_dir = tempfile.TemporaryDirectory() - with open(os.path.join(self.playlist_dir.name, 'test.m3u'), 'w') as f: + self.playlist_dir = tempfile.mkdtemp() + with open(os.path.join(self.playlist_dir, 'test.m3u'), 'w') as f: f.write('{0}\n'.format(beets.util.displayable_path(i1.path))) f.write('{0}\n'.format(beets.util.displayable_path(i2.path))) self.config['directory'] = '/' self.config['playlist']['relative_to'] = 'library' - self.config['playlist']['playlist_dir'] = self.playlist_dir.name + self.config['playlist']['playlist_dir'] = self.playlist_dir self.load_plugins('playlist') def tearDown(self): self.unload_plugins() - self.playlist_dir.cleanup() + shutil.rmtree(self.playlist_dir) self.teardown_beets() def test_query_name(self): @@ -75,7 +76,7 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): ])) def test_query_path(self): - q = u'playlist:{0}/test.m3u'.format(self.playlist_dir.name) + q = u'playlist:{0}/test.m3u'.format(self.playlist_dir) results = self.lib.items(q) self.assertEqual(set([i.title for i in results]), set([ u'some item', @@ -83,12 +84,12 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): ])) def test_query_name_nonexisting(self): - q = u'playlist:nonexisting'.format(self.playlist_dir.name) + q = u'playlist:nonexisting'.format(self.playlist_dir) results = self.lib.items(q) self.assertEqual(set(results), set()) def test_query_path_nonexisting(self): - q = u'playlist:{0}/nonexisting.m3u'.format(self.playlist_dir.name) + q = u'playlist:{0}/nonexisting.m3u'.format(self.playlist_dir) results = self.lib.items(q) self.assertEqual(set(results), set()) From 32b6df046e242c437ddbebb71be1398b68c21293 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 15:57:40 +0100 Subject: [PATCH 025/339] test: Don't use unix-only paths in playlist plugin testcase --- test/test_playlist.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index abae5d969..f10076220 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -31,22 +31,33 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.setup_beets() self.lib = beets.library.Library(':memory:') + self.music_dir = os.path.expanduser('~/Music') + i1 = _common.item() - i1.path = beets.util.normpath('/a/b/c.mp3') + i1.path = beets.util.normpath(os.path.join( + self.music_dir, + 'a/b/c.mp3', + )) i1.title = u'some item' i1.album = u'some album' self.lib.add(i1) self.lib.add_album([i1]) i2 = _common.item() - i2.path = beets.util.normpath('/d/e/f.mp3') + i2.path = beets.util.normpath(os.path.join( + self.music_dir, + 'd/e/f.mp3', + )) i2.title = 'another item' i2.album = 'another album' self.lib.add(i2) self.lib.add_album([i2]) i3 = _common.item() - i3.path = beets.util.normpath('/x/y/z.mp3') + i3.path = beets.util.normpath(os.path.join( + self.music_dir, + 'x/y/z.mp3', + )) i3.title = 'yet another item' i3.album = 'yet another album' self.lib.add(i3) @@ -57,7 +68,7 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): f.write('{0}\n'.format(beets.util.displayable_path(i1.path))) f.write('{0}\n'.format(beets.util.displayable_path(i2.path))) - self.config['directory'] = '/' + self.config['directory'] = self.music_dir self.config['playlist']['relative_to'] = 'library' self.config['playlist']['playlist_dir'] = self.playlist_dir self.load_plugins('playlist') From 055f2d3702e40a62dff9ed9469af03e487b2d548 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 16:00:04 +0100 Subject: [PATCH 026/339] playlist: Also catch IOErrors to restore Python 2.7 compatiblity --- beetsplug/playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index a6ab8d18b..759eaa51b 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -41,7 +41,7 @@ class PlaylistQuery(beets.dbcore.FieldQuery): try: f = open(beets.util.syspath(playlist_path), mode='rb') - except OSError: + except (OSError, IOError): continue if config['relative_to'].get() == 'library': From 31c687c853670ab5b58d51f372ae4b0bc2fbe74f Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 16:17:47 +0100 Subject: [PATCH 027/339] test: Fix playlist plugin path handling for Windows compatibility --- test/test_playlist.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index f10076220..62528dac1 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -87,7 +87,10 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): ])) def test_query_path(self): - q = u'playlist:{0}/test.m3u'.format(self.playlist_dir) + q = u'playlist:{0}'.format(os.path.join( + self.playlist_dir, + 'test.m3u', + )) results = self.lib.items(q) self.assertEqual(set([i.title for i in results]), set([ u'some item', @@ -100,7 +103,10 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.assertEqual(set(results), set()) def test_query_path_nonexisting(self): - q = u'playlist:{0}/nonexisting.m3u'.format(self.playlist_dir) + q = u'playlist:{0}'.format(os.path.join( + self.playlist_dir, + 'nonexisting.m3u', + )) results = self.lib.items(q) self.assertEqual(set(results), set()) From d6022e28d72fa1dc5521fbde34d22b307235ad8d Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 16:43:36 +0100 Subject: [PATCH 028/339] test: Ensure path quoting in playlist tests --- test/test_playlist.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 62528dac1..529f3631c 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -14,6 +14,7 @@ # included in all copies or substantial portions of the Software. from __future__ import division, absolute_import, print_function +from six.moves import shlex_quote import os import shutil @@ -87,10 +88,10 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): ])) def test_query_path(self): - q = u'playlist:{0}'.format(os.path.join( + q = u'playlist:{0}'.format(shlex_quote(os.path.join( self.playlist_dir, 'test.m3u', - )) + ))) results = self.lib.items(q) self.assertEqual(set([i.title for i in results]), set([ u'some item', @@ -103,10 +104,11 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.assertEqual(set(results), set()) def test_query_path_nonexisting(self): - q = u'playlist:{0}'.format(os.path.join( + q = u'playlist:{0}'.format(shlex_quote(os.path.join( + self.playlist_dir, self.playlist_dir, 'nonexisting.m3u', - )) + ))) results = self.lib.items(q) self.assertEqual(set(results), set()) From 4f1a468aa944b37aadcef3c7164bf7f4576bb957 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 17:34:36 +0100 Subject: [PATCH 029/339] playlist: Restore case sensitivity in col_clause method --- beetsplug/playlist.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 759eaa51b..e5c80f129 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -67,8 +67,7 @@ class PlaylistQuery(beets.dbcore.FieldQuery): if not self.paths: # Playlist is empty return '0', () - clause = 'BYTELOWER(path) IN ({0})'.format( - ', '.join('BYTELOWER(?)' for path in self.paths)) + clause = 'path IN ({0})'.format(', '.join('?' for path in self.paths)) return clause, (beets.library.BLOB_TYPE(p) for p in self.paths) def match(self, item): From 7360bbc1526b1da664527fcc871f9a50cb5db21b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 13:06:55 -0500 Subject: [PATCH 030/339] Only pin Jellyfish version on py2 --- docs/changelog.rst | 3 ++- setup.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1298090aa..028049ed7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -86,7 +86,8 @@ Changes: Fixes: -* Pin jellyfish requirement to version 0.6.0 to maintain python 2 compatibility. +* On Python 2, pin the Jellyfish requirement to version 0.6.0 for + compatibility. * A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks contained in data files :bug:`3021` * Restore iTunes Store album art source, and remove the dependency on diff --git a/setup.py b/setup.py index 19efb9451..648e6d4d4 100755 --- a/setup.py +++ b/setup.py @@ -92,9 +92,16 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', - 'jellyfish==0.6.0', - ] + (['colorama'] if (sys.platform == 'win32') else []) + - (['enum34>=1.0.4'] if sys.version_info < (3, 4, 0) else []), + ] + ( + # Use the backport of Python 3.4's `enum` module. + ['enum34>=1.0.4'] if sys.version_info < (3, 4, 0) else [] + ) + ( + # Pin a Python 2-compatible version of Jellyfish. + ['jellyfish==0.6.0'] if sys.version_info < (3, 4, 0) else ['jellyfish'] + ) + ( + # Support for ANSI console colors on Windows. + ['colorama'] if (sys.platform == 'win32') else [] + ), tests_require=[ 'beautifulsoup4', From 3633c1e27f892c8076cb863f236c12d53547b7e0 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 13:17:22 -0500 Subject: [PATCH 031/339] Tiny doc refinements for #3145 --- docs/changelog.rst | 4 ++-- docs/plugins/index.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6c7b36b72..a254d5efc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,10 +14,10 @@ New features: issues with foobar2000 and Winamp. Thanks to :user:`mz2212`. :bug:`2944` -* :doc:`/plugins/playlist`: Add a plugin that can query the beets library using +* A new :doc:`/plugins/playlist` can query the beets library using M3U playlists. Thanks to :user:`Holzhaus` and :user:`Xenopathic`. - :bug:`123` + :bug:`123` :bug:`3145` * Added whitespace padding to missing tracks dialog to improve readability. Thanks to :user:`jams2`. :bug:`2962` diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 502775095..173aab5db 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -159,7 +159,7 @@ Interoperability * :doc:`mpdupdate`: Automatically notifies `MPD`_ whenever the beets library changes. * :doc:`play`: Play beets queries in your music player. -* :doc:`playlist`: Use M3U playlists tp query the beets library. +* :doc:`playlist`: Use M3U playlists to query the beets library. * :doc:`plexupdate`: Automatically notifies `Plex`_ whenever the beets library changes. * :doc:`smartplaylist`: Generate smart playlists based on beets queries. From 14cad04d35c6e99ea72c17b61b2c6f42812363d1 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 19:07:21 +0100 Subject: [PATCH 032/339] test: Further improve Windows compatibility in playlist plugin test --- test/test_playlist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 529f3631c..3dd80c35f 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -37,7 +37,7 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): i1 = _common.item() i1.path = beets.util.normpath(os.path.join( self.music_dir, - 'a/b/c.mp3', + 'a', 'b', 'c.mp3', )) i1.title = u'some item' i1.album = u'some album' @@ -47,7 +47,7 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): i2 = _common.item() i2.path = beets.util.normpath(os.path.join( self.music_dir, - 'd/e/f.mp3', + 'd', 'e', 'f.mp3', )) i2.title = 'another item' i2.album = 'another album' @@ -57,7 +57,7 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): i3 = _common.item() i3.path = beets.util.normpath(os.path.join( self.music_dir, - 'x/y/z.mp3', + 'x', 'y', 'z.mp3', )) i3.title = 'yet another item' i3.album = 'yet another album' From b00b38dab6fdea37e2fc7fe201388dea84768a7b Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 19:09:46 +0100 Subject: [PATCH 033/339] test: Add test for relative playlists --- test/test_playlist.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 3dd80c35f..b01a36d07 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -65,9 +65,12 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.lib.add_album([i3]) self.playlist_dir = tempfile.mkdtemp() - with open(os.path.join(self.playlist_dir, 'test.m3u'), 'w') as f: + with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: f.write('{0}\n'.format(beets.util.displayable_path(i1.path))) f.write('{0}\n'.format(beets.util.displayable_path(i2.path))) + with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join('a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join('d', 'e', 'f.mp3'))) self.config['directory'] = self.music_dir self.config['playlist']['relative_to'] = 'library' @@ -79,18 +82,18 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): shutil.rmtree(self.playlist_dir) self.teardown_beets() - def test_query_name(self): - q = u'playlist:test' + def test_name_query_with_absolute_paths_in_playlist(self): + q = u'playlist:absolute' results = self.lib.items(q) self.assertEqual(set([i.title for i in results]), set([ u'some item', u'another item', ])) - def test_query_path(self): + def test_path_query_with_absolute_paths_in_playlist(self): q = u'playlist:{0}'.format(shlex_quote(os.path.join( self.playlist_dir, - 'test.m3u', + 'absolute.m3u', ))) results = self.lib.items(q) self.assertEqual(set([i.title for i in results]), set([ @@ -98,12 +101,31 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): u'another item', ])) - def test_query_name_nonexisting(self): + def test_name_query_with_relative_paths_in_playlist(self): + q = u'playlist:relative' + results = self.lib.items(q) + self.assertEqual(set([i.title for i in results]), set([ + u'some item', + u'another item', + ])) + + def test_path_query_with_relative_paths_in_playlist(self): + q = u'playlist:{0}'.format(shlex_quote(os.path.join( + self.playlist_dir, + 'relative.m3u', + ))) + results = self.lib.items(q) + self.assertEqual(set([i.title for i in results]), set([ + u'some item', + u'another item', + ])) + + def test_name_query_with_nonexisting_playlist(self): q = u'playlist:nonexisting'.format(self.playlist_dir) results = self.lib.items(q) self.assertEqual(set(results), set()) - def test_query_path_nonexisting(self): + def test_path_query_with_nonexisting_playlist(self): q = u'playlist:{0}'.format(shlex_quote(os.path.join( self.playlist_dir, self.playlist_dir, From 9f3acce2aef55595e67e0368e07aa9bec91e5472 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 19:14:37 +0100 Subject: [PATCH 034/339] test: Add non-existing item to playlist tests --- test/test_playlist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_playlist.py b/test/test_playlist.py index b01a36d07..4408e69d7 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -68,9 +68,12 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: f.write('{0}\n'.format(beets.util.displayable_path(i1.path))) f.write('{0}\n'.format(beets.util.displayable_path(i2.path))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'nonexisting.mp3'))) with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: f.write('{0}\n'.format(os.path.join('a', 'b', 'c.mp3'))) f.write('{0}\n'.format(os.path.join('d', 'e', 'f.mp3'))) + f.write('{0}\n'.format('nonexisting.mp3')) self.config['directory'] = self.music_dir self.config['playlist']['relative_to'] = 'library' From 5b68d883466509a5736de0dd8aecf671fcdefd76 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 19:27:56 +0100 Subject: [PATCH 035/339] test: Add more playlist tests for the different relative_to settings --- test/test_playlist.py | 82 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 4408e69d7..e05e61550 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -27,7 +27,7 @@ from test import helper import beets -class PlaylistTest(unittest.TestCase, helper.TestHelper): +class PlaylistTestHelper(helper.TestHelper): def setUp(self): self.setup_beets() self.lib = beets.library.Library(':memory:') @@ -65,21 +65,15 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.lib.add_album([i3]) self.playlist_dir = tempfile.mkdtemp() - with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: - f.write('{0}\n'.format(beets.util.displayable_path(i1.path))) - f.write('{0}\n'.format(beets.util.displayable_path(i2.path))) - f.write('{0}\n'.format(os.path.join( - self.music_dir, 'nonexisting.mp3'))) - with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: - f.write('{0}\n'.format(os.path.join('a', 'b', 'c.mp3'))) - f.write('{0}\n'.format(os.path.join('d', 'e', 'f.mp3'))) - f.write('{0}\n'.format('nonexisting.mp3')) - self.config['directory'] = self.music_dir - self.config['playlist']['relative_to'] = 'library' self.config['playlist']['playlist_dir'] = self.playlist_dir + + self.setup_test() self.load_plugins('playlist') + def setup_test(self): + raise NotImplementedError + def tearDown(self): self.unload_plugins() shutil.rmtree(self.playlist_dir) @@ -138,6 +132,70 @@ class PlaylistTest(unittest.TestCase, helper.TestHelper): self.assertEqual(set(results), set()) +class PlaylistTestRelativeToLib(PlaylistTestHelper, unittest.TestCase): + def setup_test(self): + with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'd', 'e', 'f.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'nonexisting.mp3'))) + + with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join('a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join('d', 'e', 'f.mp3'))) + f.write('{0}\n'.format('nonexisting.mp3')) + + self.config['playlist']['relative_to'] = 'library' + + +class PlaylistTestRelativeToDir(PlaylistTestHelper, unittest.TestCase): + def setup_test(self): + with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'd', 'e', 'f.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'nonexisting.mp3'))) + + with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join('a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join('d', 'e', 'f.mp3'))) + f.write('{0}\n'.format('nonexisting.mp3')) + + self.config['playlist']['relative_to'] = self.music_dir + + +class PlaylistTestRelativeToPls(PlaylistTestHelper, unittest.TestCase): + def setup_test(self): + with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'd', 'e', 'f.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'nonexisting.mp3'))) + + with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.relpath( + os.path.join(self.music_dir, 'a', 'b', 'c.mp3'), + start=self.playlist_dir, + ))) + f.write('{0}\n'.format(os.path.relpath( + os.path.join(self.music_dir, 'd', 'e', 'f.mp3'), + start=self.playlist_dir, + ))) + f.write('{0}\n'.format(os.path.relpath( + os.path.join(self.music_dir, 'nonexisting.mp3'), + start=self.playlist_dir, + ))) + + self.config['playlist']['relative_to'] = 'playlist' + self.config['playlist']['playlist_dir'] = self.playlist_dir + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 6d420280571cc6ad4f4cf422013f0f036b800200 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 15 Feb 2019 19:50:26 +0100 Subject: [PATCH 036/339] playlist: Add playlist auto-update functionality --- beetsplug/playlist.py | 93 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index e5c80f129..393449217 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -14,6 +14,7 @@ import os import fnmatch +import tempfile import beets @@ -86,6 +87,98 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): def __init__(self): super(PlaylistPlugin, self).__init__() self.config.add({ + 'auto': False, 'playlist_dir': '.', 'relative_to': 'library', }) + + self.playlist_dir = self.config['playlist_dir'].as_filename() + self.changes = {} + + if self.config['relative_to'].get() == 'library': + self.relative_to = beets.util.bytestring_path( + beets.config['directory'].as_filename()) + elif self.config['relative_to'].get() != 'playlist': + print(repr(self.config['relative_to'].get())) + self.relative_to = beets.util.bytestring_path( + self.config['relative_to'].as_filename()) + else: + self.relative_to = None + + if self.config['auto'].get(bool): + self.register_listener('item_moved', self.item_moved) + self.register_listener('item_removed', self.item_removed) + self.register_listener('cli_exit', self.cli_exit) + + def item_moved(self, item, source, destination): + self.changes[source] = destination + + def item_removed(self, item): + if not os.path.exists(beets.util.syspath(item.path)): + self.changes[item.path] = None + + def cli_exit(self, lib): + for playlist in self.find_playlists(): + self._log.info('Updating playlist: {0}'.format(playlist)) + base_dir = beets.util.bytestring_path( + self.relative_to if self.relative_to + else os.path.dirname(playlist) + ) + + try: + self.update_playlist(playlist, base_dir) + except beets.util.FilesystemError: + self._log.error('Failed to update playlist: {0}'.format( + beets.util.displayable_path(playlist))) + + def find_playlists(self): + """Find M3U playlists in the playlist directory.""" + try: + dir_contents = os.listdir(beets.util.syspath(self.playlist_dir)) + except OSError: + self._log.warning('Unable to open playlist directory {0}'.format( + beets.util.displayable_path(self.playlist_dir))) + return + + for filename in dir_contents: + if fnmatch.fnmatch(filename, '*.[mM]3[uU]'): + yield os.path.join(self.playlist_dir, filename) + + def update_playlist(self, filename, base_dir): + """Find M3U playlists in the specified directory.""" + changes = 0 + deletions = 0 + + with tempfile.NamedTemporaryFile(mode='w+b') as tempfp: + with open(filename, mode='rb') as fp: + for line in fp: + original_path = line.rstrip(b'\r\n') + + # Ensure that path from playlist is absolute + is_relative = not os.path.isabs(beets.util.syspath(line)) + if is_relative: + lookup = os.path.join(base_dir, original_path) + else: + lookup = original_path + + try: + new_path = self.changes[lookup] + except KeyError: + tempfp.write(line) + else: + if new_path is None: + # Item has been deleted + deletions += 1 + continue + + changes += 1 + if is_relative: + new_path = os.path.relpath(new_path, base_dir) + + tempfp.write(line.replace(original_path, new_path)) + if changes or deletions: + self._log.info( + 'Updated playlist {0} ({1} changes, {2} deletions)'.format( + filename, changes, deletions)) + tempfp.flush() + beets.util.copy(tempfp.name, filename, replace=True) From d8e167637ec3eb025621d5b50aacca8526877872 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 13:41:05 -0500 Subject: [PATCH 037/339] Prototype support for named (pseudo-field) queries As discussed here: https://github.com/beetbox/beets/pull/3145#pullrequestreview-204523870 This would replace the need for #3149. --- beets/dbcore/db.py | 5 +++++ beets/dbcore/queryparse.py | 38 ++++++++++++++++++++------------------ test/test_dbcore.py | 18 ++++++++++++++++++ 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index e92cba40c..71810ead2 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -143,6 +143,11 @@ class Model(object): are subclasses of `Sort`. """ + _queries = {} + """Named queries that use a field-like `name:value` syntax but which + do not relate to any specific field. + """ + _always_dirty = False """By default, fields only become "dirty" when their value actually changes. Enabling this flag marks fields as dirty even when the new diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index ce88fa3bd..1cb25a8c7 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -119,12 +119,13 @@ def construct_query_part(model_cls, prefixes, query_part): if not query_part: return query.TrueQuery() - # Use `model_cls` to build up a map from field names to `Query` - # classes. + # Use `model_cls` to build up a map from field (or query) names to + # `Query` classes. query_classes = {} for k, t in itertools.chain(model_cls._fields.items(), model_cls._types.items()): query_classes[k] = t.query + query_classes.update(model_cls._queries) # Non-field queries. # Parse the string. key, pattern, query_class, negate = \ @@ -137,26 +138,27 @@ def construct_query_part(model_cls, prefixes, query_part): # The query type matches a specific field, but none was # specified. So we use a version of the query that matches # any field. - q = query.AnyFieldQuery(pattern, model_cls._search_fields, - query_class) - if negate: - return query.NotQuery(q) - else: - return q + out_query = query.AnyFieldQuery(pattern, model_cls._search_fields, + query_class) else: # Non-field query type. - if negate: - return query.NotQuery(query_class(pattern)) - else: - return query_class(pattern) + out_query = query_class(pattern) - # Otherwise, this must be a `FieldQuery`. Use the field name to - # construct the query object. - key = key.lower() - q = query_class(key.lower(), pattern, key in model_cls._fields) + # Field queries get constructed according to the name of the field + # they are querying. + elif issubclass(query_class, query.FieldQuery): + key = key.lower() + out_query = query_class(key.lower(), pattern, key in model_cls._fields) + + # Non-field (named) query. + else: + out_query = query_class(pattern) + + # Apply negation. if negate: - return query.NotQuery(q) - return q + return query.NotQuery(out_query) + else: + return out_query def query_from_strings(query_cls, model_cls, prefixes, query_parts): diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 89aca442b..34994e3b3 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -36,6 +36,17 @@ class TestSort(dbcore.query.FieldSort): pass +class TestQuery(dbcore.query.Query): + def __init__(self, pattern): + self.pattern = pattern + + def clause(self): + return None, () + + def match(self): + return True + + class TestModel1(dbcore.Model): _table = 'test' _flex_table = 'testflex' @@ -49,6 +60,9 @@ class TestModel1(dbcore.Model): _sorts = { 'some_sort': TestSort, } + _queries = { + 'some_query': TestQuery, + } @classmethod def _getters(cls): @@ -519,6 +533,10 @@ class QueryFromStringsTest(unittest.TestCase): q = self.qfs(['']) self.assertIsInstance(q.subqueries[0], dbcore.query.TrueQuery) + def test_parse_named_query(self): + q = self.qfs(['some_query:foo']) + self.assertIsInstance(q.subqueries[0], TestQuery) + class SortFromStringsTest(unittest.TestCase): def sfs(self, strings): From 1af82cc450929713edfd9d8da1cc7e26d8dd0f21 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sat, 16 Feb 2019 13:59:25 +0100 Subject: [PATCH 038/339] test: Split up playlist test helper class --- test/test_playlist.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index e05e61550..1fcdb0071 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -79,6 +79,8 @@ class PlaylistTestHelper(helper.TestHelper): shutil.rmtree(self.playlist_dir) self.teardown_beets() + +class PlaylistQueryTestHelper(PlaylistTestHelper): def test_name_query_with_absolute_paths_in_playlist(self): q = u'playlist:absolute' results = self.lib.items(q) @@ -132,7 +134,7 @@ class PlaylistTestHelper(helper.TestHelper): self.assertEqual(set(results), set()) -class PlaylistTestRelativeToLib(PlaylistTestHelper, unittest.TestCase): +class PlaylistTestRelativeToLib(PlaylistQueryTestHelper, unittest.TestCase): def setup_test(self): with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: f.write('{0}\n'.format(os.path.join( @@ -150,7 +152,7 @@ class PlaylistTestRelativeToLib(PlaylistTestHelper, unittest.TestCase): self.config['playlist']['relative_to'] = 'library' -class PlaylistTestRelativeToDir(PlaylistTestHelper, unittest.TestCase): +class PlaylistTestRelativeToDir(PlaylistQueryTestHelper, unittest.TestCase): def setup_test(self): with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: f.write('{0}\n'.format(os.path.join( @@ -168,7 +170,7 @@ class PlaylistTestRelativeToDir(PlaylistTestHelper, unittest.TestCase): self.config['playlist']['relative_to'] = self.music_dir -class PlaylistTestRelativeToPls(PlaylistTestHelper, unittest.TestCase): +class PlaylistTestRelativeToPls(PlaylistQueryTestHelper, unittest.TestCase): def setup_test(self): with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: f.write('{0}\n'.format(os.path.join( From 22c8289269df005ae2440f2540d314b6791ad5d1 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 13:49:54 -0500 Subject: [PATCH 039/339] Support plugin-provided named queries --- beets/plugins.py | 10 ++++++++++ beets/ui/__init__.py | 4 ++++ docs/dev/plugins.rst | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 6dec7ef2a..f10dc5849 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -344,6 +344,16 @@ def types(model_cls): return types +def named_queries(model_cls): + # Gather `item_queries` and `album_queries` from the plugins. + attr_name = '{0}_queries'.format(model_cls.__name__.lower()) + queries = {} + for plugin in find_plugins(): + plugin_queries = getattr(plugin, attr_name, {}) + queries.update(plugin_queries) + return queries + + def track_distance(item, info): """Gets the track distance calculated by all loaded plugins. Returns a Distance object. diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 1abce2e67..327db6b04 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1143,8 +1143,12 @@ def _setup(options, lib=None): if lib is None: lib = _open_library(config) plugins.send("library_opened", lib=lib) + + # Add types and queries defined by plugins. library.Item._types.update(plugins.types(library.Item)) library.Album._types.update(plugins.types(library.Album)) + library.Item._queries.update(plugins.named_queries(library.Item)) + library.Album._queries.update(plugins.named_queries(library.Album)) return subcommands, plugins, lib diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index bab0e604d..c9018c394 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -443,15 +443,24 @@ Extend the Query Syntax ^^^^^^^^^^^^^^^^^^^^^^^ You can add new kinds of queries to beets' :doc:`query syntax -` indicated by a prefix. As an example, beets already +`. There are two ways to add custom queries: using a prefix +and using a name. Prefix-based query extension can apply to *any* field, while +named queries are not associated with any field. For example, beets already supports regular expression queries, which are indicated by a colon prefix---plugins can do the same. -To do so, define a subclass of the ``Query`` type from the -``beets.dbcore.query`` module. Then, in the ``queries`` method of your plugin -class, return a dictionary mapping prefix strings to query classes. +For either kind of query extension, define a subclass of the ``Query`` type +from the ``beets.dbcore.query`` module. Then: -One simple kind of query you can extend is the ``FieldQuery``, which +- To define a prefix-based query, define a ``queries`` method in your plugin + class. Return from this method a dictionary mapping prefix strings to query + classes. +- To define a named query, defined dictionaries named either ``item_queries`` + or ``album_queries``. These should map names to query types. So if you + use ``{ "foo": FooQuery }``, then the query ``foo:bar`` will construct a + query like ``FooQuery("bar")``. + +For prefix-based queries, you will want to extend ``FieldQuery``, which implements string comparisons on fields. To use it, create a subclass inheriting from that class and override the ``value_match`` class method. (Remember the ``@classmethod`` decorator!) The following example plugin From 7efc67eb0361ff731f5c756e43c99107c49058f9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 13:52:00 -0500 Subject: [PATCH 040/339] playlist: Use new "named query" functionality --- beetsplug/playlist.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index e5c80f129..8b58b03be 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -17,11 +17,11 @@ import fnmatch import beets -class PlaylistQuery(beets.dbcore.FieldQuery): +class PlaylistQuery(beets.dbcore.Query): """Matches files listed by a playlist file. """ - def __init__(self, field, pattern, fast=True): - super(PlaylistQuery, self).__init__(field, pattern, fast) + def __init__(self, pattern): + self.pattern = pattern config = beets.config['playlist'] # Get the full path to the playlist @@ -74,14 +74,8 @@ class PlaylistQuery(beets.dbcore.FieldQuery): return item.path in self.paths -class PlaylistType(beets.dbcore.types.String): - """Custom type for playlist query. - """ - query = PlaylistQuery - - class PlaylistPlugin(beets.plugins.BeetsPlugin): - item_types = {'playlist': PlaylistType()} + item_queries = {'playlist': PlaylistQuery} def __init__(self): super(PlaylistPlugin, self).__init__() From 420772ea497872b99ac61dc242d9f7c75ef06ea9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 13:54:18 -0500 Subject: [PATCH 041/339] Changelog entry for pseudo-field queries --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a254d5efc..fc6b4cd93 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -144,6 +144,14 @@ Fixes: .. _python-itunes: https://github.com/ocelma/python-itunes +For developers: + +* In addition to prefix-based field queries, plugins can now define *named + queries* that are not associated with any specific field. + For example, the new :doc:`/plugins/playlist` supports queries like + ``playlist:name`` although there is no field named ``playlist``. + See :ref:`extend-query` for details. + 1.4.7 (May 29, 2018) -------------------- From 55ef2ffd39e73f15674a3e5b0438428d5ef50c29 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 14:02:26 -0500 Subject: [PATCH 042/339] Add future imports to playlist plugin --- beetsplug/playlist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index e5c80f129..e8683ff93 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -12,6 +12,8 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. +from __future__ import division, absolute_import, print_function + import os import fnmatch import beets From 7edba6e9eaeca98faf6b73c17d0bd9fd2bff1649 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 Feb 2019 14:11:40 -0500 Subject: [PATCH 043/339] Fix test harness for named queries --- test/helper.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/helper.py b/test/helper.py index 92128f511..392d01a55 100644 --- a/test/helper.py +++ b/test/helper.py @@ -222,12 +222,19 @@ class TestHelper(object): beets.config['plugins'] = plugins beets.plugins.load_plugins(plugins) beets.plugins.find_plugins() - # Take a backup of the original _types to restore when unloading + + # Take a backup of the original _types and _queries to restore + # when unloading. Item._original_types = dict(Item._types) Album._original_types = dict(Album._types) Item._types.update(beets.plugins.types(Item)) Album._types.update(beets.plugins.types(Album)) + Item._original_queries = dict(Item._queries) + Album._original_queries = dict(Album._queries) + Item._queries.update(beets.plugins.named_queries(Item)) + Album._queries.update(beets.plugins.named_queries(Album)) + def unload_plugins(self): """Unload all plugins and remove the from the configuration. """ @@ -237,6 +244,8 @@ class TestHelper(object): beets.plugins._instances = {} Item._types = Item._original_types Album._types = Album._original_types + Item._queries = Item._original_queries + Album._queries = Album._original_queries def create_importer(self, item_count=1, album_count=1): """Create files to import and return corresponding session. From a9dd5a7cdc72102fb85b85731542fd203161d589 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sat, 16 Feb 2019 15:37:26 +0100 Subject: [PATCH 044/339] test: Add testcase for playlist plugin's update functionality --- test/test_playlist.py | 103 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/test/test_playlist.py b/test/test_playlist.py index 1fcdb0071..2bc461f76 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -198,6 +198,109 @@ class PlaylistTestRelativeToPls(PlaylistQueryTestHelper, unittest.TestCase): self.config['playlist']['playlist_dir'] = self.playlist_dir +class PlaylistUpdateTestHelper(PlaylistTestHelper): + def setup_test(self): + with open(os.path.join(self.playlist_dir, 'absolute.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'd', 'e', 'f.mp3'))) + f.write('{0}\n'.format(os.path.join( + self.music_dir, 'nonexisting.mp3'))) + + with open(os.path.join(self.playlist_dir, 'relative.m3u'), 'w') as f: + f.write('{0}\n'.format(os.path.join('a', 'b', 'c.mp3'))) + f.write('{0}\n'.format(os.path.join('d', 'e', 'f.mp3'))) + f.write('{0}\n'.format('nonexisting.mp3')) + + self.config['playlist']['auto'] = True + self.config['playlist']['relative_to'] = 'library' + + +class PlaylistTestItemMoved(PlaylistUpdateTestHelper, unittest.TestCase): + def test_item_moved(self): + # Emit item_moved event for an item that is in a playlist + results = self.lib.items('path:{0}'.format(shlex_quote( + os.path.join(self.music_dir, 'd', 'e', 'f.mp3')))) + item = results[0] + beets.plugins.send( + 'item_moved', item=item, source=item.path, + destination=beets.util.bytestring_path( + os.path.join(self.music_dir, 'g', 'h', 'i.mp3'))) + + # Emit item_moved event for an item that is not in a playlist + results = self.lib.items('path:{0}'.format(shlex_quote( + os.path.join(self.music_dir, 'x', 'y', 'z.mp3')))) + item = results[0] + beets.plugins.send( + 'item_moved', item=item, source=item.path, + destination=beets.util.bytestring_path( + os.path.join(self.music_dir, 'u', 'v', 'w.mp3'))) + + # Emit cli_exit event + beets.plugins.send('cli_exit', lib=self.lib) + + # Check playlist with absolute paths + playlist_path = os.path.join(self.playlist_dir, 'absolute.m3u') + with open(playlist_path, 'r') as f: + lines = [line.strip() for line in f.readlines()] + + self.assertEqual(lines, [ + os.path.join(self.music_dir, 'a', 'b', 'c.mp3'), + os.path.join(self.music_dir, 'g', 'h', 'i.mp3'), + os.path.join(self.music_dir, 'nonexisting.mp3'), + ]) + + # Check playlist with relative paths + playlist_path = os.path.join(self.playlist_dir, 'relative.m3u') + with open(playlist_path, 'r') as f: + lines = [line.strip() for line in f.readlines()] + + self.assertEqual(lines, [ + os.path.join('a', 'b', 'c.mp3'), + os.path.join('g', 'h', 'i.mp3'), + 'nonexisting.mp3', + ]) + + +class PlaylistTestItemRemoved(PlaylistUpdateTestHelper, unittest.TestCase): + def test_item_removed(self): + # Emit item_removed event for an item that is in a playlist + results = self.lib.items('path:{0}'.format(shlex_quote( + os.path.join(self.music_dir, 'd', 'e', 'f.mp3')))) + item = results[0] + beets.plugins.send('item_removed', item=item) + + # Emit item_removed event for an item that is not in a playlist + results = self.lib.items('path:{0}'.format(shlex_quote( + os.path.join(self.music_dir, 'x', 'y', 'z.mp3')))) + item = results[0] + beets.plugins.send('item_removed', item=item) + + # Emit cli_exit event + beets.plugins.send('cli_exit', lib=self.lib) + + # Check playlist with absolute paths + playlist_path = os.path.join(self.playlist_dir, 'absolute.m3u') + with open(playlist_path, 'r') as f: + lines = [line.strip() for line in f.readlines()] + + self.assertEqual(lines, [ + os.path.join(self.music_dir, 'a', 'b', 'c.mp3'), + os.path.join(self.music_dir, 'nonexisting.mp3'), + ]) + + # Check playlist with relative paths + playlist_path = os.path.join(self.playlist_dir, 'relative.m3u') + with open(playlist_path, 'r') as f: + lines = [line.strip() for line in f.readlines()] + + self.assertEqual(lines, [ + os.path.join('a', 'b', 'c.mp3'), + 'nonexisting.mp3', + ]) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From fdd41b301d272604ded7b980532ee4275e46a71e Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sat, 16 Feb 2019 15:45:04 +0100 Subject: [PATCH 045/339] docs: Update documentation regarding playlist plugin --- docs/plugins/playlist.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 1156e7f77..d9b400987 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -8,6 +8,7 @@ To use it, enable the ``playlist`` plugin in your configuration Then configure your playlists like this:: playlist: + auto: no relative_to: ~/Music playlist_dir: ~/.mpd/playlists @@ -22,6 +23,10 @@ name:: $ beet ls playlist:anotherplaylist +The plugin can also update playlists in the playlist directory automatically +every time an item is moved or deleted. This can be controlled by the ``auto`` +configuration option. + Configuration ------------- @@ -29,6 +34,10 @@ To configure the plugin, make a ``smartplaylist:`` section in your configuration file. In addition to the ``playlists`` described above, the other configuration options are: +- **auto**: If this is set to ``yes``, then anytime an item in the library is + moved or removed, the plugin will update all playlists in the + ``playlist_dir`` directory that contain that item to reflect the change. + Default: ``no`` - **playlist_dir**: Where to read playlist files from. Default: The current working directory (i.e., ``'.'``). - **relative_to**: Interpret paths in the playlist files relative to a base From 7ec55a5f3be4a768d5889dbe8eaf712c27f96700 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 21:27:09 +0100 Subject: [PATCH 046/339] test: Use unicode literals for library queries in playlist tests --- test/test_playlist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 2bc461f76..1e800804e 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -220,7 +220,7 @@ class PlaylistUpdateTestHelper(PlaylistTestHelper): class PlaylistTestItemMoved(PlaylistUpdateTestHelper, unittest.TestCase): def test_item_moved(self): # Emit item_moved event for an item that is in a playlist - results = self.lib.items('path:{0}'.format(shlex_quote( + results = self.lib.items(u'path:{0}'.format(shlex_quote( os.path.join(self.music_dir, 'd', 'e', 'f.mp3')))) item = results[0] beets.plugins.send( @@ -229,7 +229,7 @@ class PlaylistTestItemMoved(PlaylistUpdateTestHelper, unittest.TestCase): os.path.join(self.music_dir, 'g', 'h', 'i.mp3'))) # Emit item_moved event for an item that is not in a playlist - results = self.lib.items('path:{0}'.format(shlex_quote( + results = self.lib.items(u'path:{0}'.format(shlex_quote( os.path.join(self.music_dir, 'x', 'y', 'z.mp3')))) item = results[0] beets.plugins.send( @@ -266,13 +266,13 @@ class PlaylistTestItemMoved(PlaylistUpdateTestHelper, unittest.TestCase): class PlaylistTestItemRemoved(PlaylistUpdateTestHelper, unittest.TestCase): def test_item_removed(self): # Emit item_removed event for an item that is in a playlist - results = self.lib.items('path:{0}'.format(shlex_quote( + results = self.lib.items(u'path:{0}'.format(shlex_quote( os.path.join(self.music_dir, 'd', 'e', 'f.mp3')))) item = results[0] beets.plugins.send('item_removed', item=item) # Emit item_removed event for an item that is not in a playlist - results = self.lib.items('path:{0}'.format(shlex_quote( + results = self.lib.items(u'path:{0}'.format(shlex_quote( os.path.join(self.music_dir, 'x', 'y', 'z.mp3')))) item = results[0] beets.plugins.send('item_removed', item=item) From 76a3e44aaddbd1aa23ffe8e3b6d8d2878d769738 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 21:27:37 +0100 Subject: [PATCH 047/339] test: Make music dir of playlist tests Windows-compatible --- test/test_playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_playlist.py b/test/test_playlist.py index 1e800804e..edd98e711 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -32,7 +32,7 @@ class PlaylistTestHelper(helper.TestHelper): self.setup_beets() self.lib = beets.library.Library(':memory:') - self.music_dir = os.path.expanduser('~/Music') + self.music_dir = os.path.expanduser(os.path.join('~', 'Music')) i1 = _common.item() i1.path = beets.util.normpath(os.path.join( From d991e2a7d8cf3a55dbad4797f084fe24f4d8016d Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 21:51:09 +0100 Subject: [PATCH 048/339] playlist: Normalize path before lookup in changes dict --- beetsplug/playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 393449217..68b2adb49 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -162,7 +162,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): lookup = original_path try: - new_path = self.changes[lookup] + new_path = self.changes[beets.util.normpath(lookup)] except KeyError: tempfp.write(line) else: From ee2cce4280b14c8ade99fd838bea330ebd1360f8 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 22:05:54 +0100 Subject: [PATCH 049/339] playlist: Work around Windows' Mandatory File Locking on playlist updates --- beetsplug/playlist.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 68b2adb49..ae530aaec 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -149,7 +149,8 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): changes = 0 deletions = 0 - with tempfile.NamedTemporaryFile(mode='w+b') as tempfp: + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as tempfp: + new_playlist = tempfp.name with open(filename, mode='rb') as fp: for line in fp: original_path = line.rstrip(b'\r\n') @@ -176,9 +177,10 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): new_path = os.path.relpath(new_path, base_dir) tempfp.write(line.replace(original_path, new_path)) - if changes or deletions: - self._log.info( - 'Updated playlist {0} ({1} changes, {2} deletions)'.format( - filename, changes, deletions)) - tempfp.flush() - beets.util.copy(tempfp.name, filename, replace=True) + + if changes or deletions: + self._log.info( + 'Updated playlist {0} ({1} changes, {2} deletions)'.format( + filename, changes, deletions)) + beets.util.copy(new_playlist, filename, replace=True) + beets.util.remove(new_playlist) From 7bca5cf549c9f4569c8f2616e67b4f52afb0bd82 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 17 Feb 2019 22:28:39 +0100 Subject: [PATCH 050/339] playlist: Don't use syspath() when checking if path is absolute --- beetsplug/playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index ae530aaec..08c4bc8af 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -156,7 +156,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): original_path = line.rstrip(b'\r\n') # Ensure that path from playlist is absolute - is_relative = not os.path.isabs(beets.util.syspath(line)) + is_relative = not os.path.isabs(line) if is_relative: lookup = os.path.join(base_dir, original_path) else: From 4ba5dfaa43bffd3470b10e857388516e6590b19e Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Mon, 18 Feb 2019 09:13:39 +0100 Subject: [PATCH 051/339] playlist: Remove leftover print call and fix 'auto' option access style --- beetsplug/playlist.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 08c4bc8af..04959c431 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -99,13 +99,12 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): self.relative_to = beets.util.bytestring_path( beets.config['directory'].as_filename()) elif self.config['relative_to'].get() != 'playlist': - print(repr(self.config['relative_to'].get())) self.relative_to = beets.util.bytestring_path( self.config['relative_to'].as_filename()) else: self.relative_to = None - if self.config['auto'].get(bool): + if self.config['auto']: self.register_listener('item_moved', self.item_moved) self.register_listener('item_removed', self.item_removed) self.register_listener('cli_exit', self.cli_exit) From c4506558f552e96dcd893483c3ad92829a363961 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 18 Feb 2019 22:19:41 -0800 Subject: [PATCH 052/339] Added par_map utility --- beets/util/__init__.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 69870edf2..ba7393975 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -24,6 +24,7 @@ import re import shutil import fnmatch from collections import Counter +from multiprocessing.pool import ThreadPool import traceback import subprocess import platform @@ -1009,3 +1010,22 @@ def asciify_path(path, sep_replace): sep_replace ) return os.sep.join(path_components) + + +def par_map(transform, items): + """ + This module implements a simple utility to either: + a) Perform a parallel map when running under Python >=3 + b) Perform a sequential map otherwise + + This is useful whenever there is some operation `do_something()` which we + want to efficiently apply to our music library. + """ + if sys.version_info[0] < 3: + for item in items: + transform(item) + else: + pool = ThreadPool() + pool.map(transform, items) + pool.close() + pool.join() From 1dad5ded039451023fa281481f605d95793c044e Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 18 Feb 2019 22:26:59 -0800 Subject: [PATCH 053/339] Move absubmit plugin parallelization to util.par_map --- beetsplug/absubmit.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 5cce11bc0..9d26ac5db 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -24,9 +24,7 @@ import json import os import subprocess import tempfile -import sys -from multiprocessing.pool import ThreadPool from distutils.spawn import find_executable import requests @@ -106,15 +104,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) - if sys.version_info[0] < 3: - for item in items: - self.analyze_submit(item) - else: - # Analyze in parallel using a thread pool. - pool = ThreadPool() - pool.map(self.analyze_submit, items) - pool.close() - pool.join() + util.par_map(self.analyze_submit, items) def analyze_submit(self, item): analysis = self._get_analysis(item) From c07903ed666e6182dc7842bf79bae6cc05bfe79e Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Tue, 19 Feb 2019 16:16:56 +0100 Subject: [PATCH 054/339] fetchart: Add some error handling to prevent crashes Today I had some network problems regarding dbpedia.org, which made beets crash because a requests.exceptions.ConnectionError was raised ("[Errno 113] No route to host"). This commits adds some error handling around network requests to prevent further crashes in the future. --- beetsplug/fetchart.py | 109 +++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 43 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index d7a885315..bfda94670 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -365,12 +365,17 @@ class GoogleImages(RemoteArtSource): if not (album.albumartist and album.album): return search_string = (album.albumartist + ',' + album.album).encode('utf-8') - response = self.request(self.URL, params={ - 'key': self.key, - 'cx': self.cx, - 'q': search_string, - 'searchType': 'image' - }) + + try: + response = self.request(self.URL, params={ + 'key': self.key, + 'cx': self.cx, + 'q': search_string, + 'searchType': 'image' + }) + except requests.RequestException: + self._log.debug(u'google: error receiving response') + return # Get results using JSON. try: @@ -406,10 +411,14 @@ class FanartTV(RemoteArtSource): if not album.mb_releasegroupid: return - response = self.request( - self.API_ALBUMS + album.mb_releasegroupid, - headers={'api-key': self.PROJECT_KEY, - 'client-key': self.client_key}) + try: + response = self.request( + self.API_ALBUMS + album.mb_releasegroupid, + headers={'api-key': self.PROJECT_KEY, + 'client-key': self.client_key}) + except requests.RequestException: + self._log.debug(u'fanart.tv: error receiving response') + return try: data = response.json() @@ -545,16 +554,22 @@ class Wikipedia(RemoteArtSource): # Find the name of the cover art filename on DBpedia cover_filename, page_id = None, None - dbpedia_response = self.request( - self.DBPEDIA_URL, - params={ - 'format': 'application/sparql-results+json', - 'timeout': 2500, - 'query': self.SPARQL_QUERY.format( - artist=album.albumartist.title(), album=album.album) - }, - headers={'content-type': 'application/json'}, - ) + + try: + dbpedia_response = self.request( + self.DBPEDIA_URL, + params={ + 'format': 'application/sparql-results+json', + 'timeout': 2500, + 'query': self.SPARQL_QUERY.format( + artist=album.albumartist.title(), album=album.album) + }, + headers={'content-type': 'application/json'}, + ) + except requests.RequestException: + self._log.debug(u'dbpedia: error receiving response') + return + try: data = dbpedia_response.json() results = data['results']['bindings'] @@ -584,17 +599,21 @@ class Wikipedia(RemoteArtSource): lpart, rpart = cover_filename.rsplit(' .', 1) # Query all the images in the page - wikipedia_response = self.request( - self.WIKIPEDIA_URL, - params={ - 'format': 'json', - 'action': 'query', - 'continue': '', - 'prop': 'images', - 'pageids': page_id, - }, - headers={'content-type': 'application/json'}, - ) + try: + wikipedia_response = self.request( + self.WIKIPEDIA_URL, + params={ + 'format': 'json', + 'action': 'query', + 'continue': '', + 'prop': 'images', + 'pageids': page_id, + }, + headers={'content-type': 'application/json'}, + ) + except requests.RequestException: + self._log.debug(u'wikipedia: error receiving response') + return # Try to see if one of the images on the pages matches our # incomplete cover_filename @@ -613,18 +632,22 @@ class Wikipedia(RemoteArtSource): return # Find the absolute url of the cover art on Wikipedia - wikipedia_response = self.request( - self.WIKIPEDIA_URL, - params={ - 'format': 'json', - 'action': 'query', - 'continue': '', - 'prop': 'imageinfo', - 'iiprop': 'url', - 'titles': cover_filename.encode('utf-8'), - }, - headers={'content-type': 'application/json'}, - ) + try: + wikipedia_response = self.request( + self.WIKIPEDIA_URL, + params={ + 'format': 'json', + 'action': 'query', + 'continue': '', + 'prop': 'imageinfo', + 'iiprop': 'url', + 'titles': cover_filename.encode('utf-8'), + }, + headers={'content-type': 'application/json'}, + ) + except requests.RequestException: + self._log.debug(u'wikipedia: error receiving response') + return try: data = wikipedia_response.json() From 10bc4665732cdda2502de16a11dd28cd7aba6404 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 19 Feb 2019 18:36:19 -0500 Subject: [PATCH 055/339] Refine docs for #3153 --- beets/util/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index ba7393975..f3dedcb41 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -1013,15 +1013,17 @@ def asciify_path(path, sep_replace): def par_map(transform, items): - """ - This module implements a simple utility to either: - a) Perform a parallel map when running under Python >=3 - b) Perform a sequential map otherwise + """Apply the function `transform` to all the elements in the + iterable `items`, like `map(transform, items)` but with no return + value. The map *might* happen in parallel: it's parallel on Python 3 + and sequential on Python 2. - This is useful whenever there is some operation `do_something()` which we - want to efficiently apply to our music library. + The parallelism uses threads (not processes), so this is only useful + for IO-bound `transform`s. """ if sys.version_info[0] < 3: + # multiprocessing.pool.ThreadPool does not seem to work on + # Python 2. We could consider switching to futures instead. for item in items: transform(item) else: From e209fe5886129ba17fa52d7dfeeb14a059a121d4 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Sat, 18 Aug 2018 12:12:20 -0300 Subject: [PATCH 056/339] Parallelized `beet bad` --- beetsplug/badfiles.py | 101 ++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 62c6d8af5..23a9c446e 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -20,7 +20,7 @@ from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets.ui import Subcommand -from beets.util import displayable_path, confit +from beets.util import displayable_path, confit, par_map from beets import ui from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT import shlex @@ -48,6 +48,9 @@ class CheckerCommandException(Exception): class BadFiles(BeetsPlugin): + def __init__(self): + self.verbose = False + def run_command(self, cmd): self._log.debug(u"running command: {}", displayable_path(list2cmdline(cmd))) @@ -89,56 +92,60 @@ class BadFiles(BeetsPlugin): command = None if command: return self.check_custom(command) - elif ext == "mp3": + if ext == "mp3": return self.check_mp3val - elif ext == "flac": + if ext == "flac": return self.check_flac - def check_bad(self, lib, opts, args): - for item in lib.items(ui.decargs(args)): + def check_item(self, item): + # First, check whether the path exists. If not, the user + # should probably run `beet update` to cleanup your library. + dpath = displayable_path(item.path) + self._log.debug(u"checking path: {}", dpath) + if not os.path.exists(item.path): + ui.print_(u"{}: file does not exist".format( + ui.colorize('text_error', dpath))) - # First, check whether the path exists. If not, the user - # should probably run `beet update` to cleanup your library. - dpath = displayable_path(item.path) - self._log.debug(u"checking path: {}", dpath) - if not os.path.exists(item.path): - ui.print_(u"{}: file does not exist".format( - ui.colorize('text_error', dpath))) + # Run the checker against the file if one is found + ext = os.path.splitext(item.path)[1][1:].decode('utf8', 'ignore') + checker = self.get_checker(ext) + if not checker: + self._log.error(u"no checker specified in the config for {}", + ext) + return + path = item.path + if not isinstance(path, six.text_type): + path = item.path.decode(sys.getfilesystemencoding()) + try: + status, errors, output = checker(path) + except CheckerCommandException as e: + if e.errno == errno.ENOENT: + self._log.error( + u"command not found: {} when validating file: {}", + e.checker, + e.path + ) + else: + self._log.error(u"error invoking {}: {}", e.checker, e.msg) + return + if status > 0: + ui.print_(u"{}: checker exited with status {}" + .format(ui.colorize('text_error', dpath), status)) + for line in output: + ui.print_(u" {}".format(displayable_path(line))) + elif errors > 0: + ui.print_(u"{}: checker found {} errors or warnings" + .format(ui.colorize('text_warning', dpath), errors)) + for line in output: + ui.print_(u" {}".format(displayable_path(line))) + elif self.verbose: + ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) - # Run the checker against the file if one is found - ext = os.path.splitext(item.path)[1][1:].decode('utf8', 'ignore') - checker = self.get_checker(ext) - if not checker: - self._log.error(u"no checker specified in the config for {}", - ext) - continue - path = item.path - if not isinstance(path, six.text_type): - path = item.path.decode(sys.getfilesystemencoding()) - try: - status, errors, output = checker(path) - except CheckerCommandException as e: - if e.errno == errno.ENOENT: - self._log.error( - u"command not found: {} when validating file: {}", - e.checker, - e.path - ) - else: - self._log.error(u"error invoking {}: {}", e.checker, e.msg) - continue - if status > 0: - ui.print_(u"{}: checker exited with status {}" - .format(ui.colorize('text_error', dpath), status)) - for line in output: - ui.print_(u" {}".format(displayable_path(line))) - elif errors > 0: - ui.print_(u"{}: checker found {} errors or warnings" - .format(ui.colorize('text_warning', dpath), errors)) - for line in output: - ui.print_(u" {}".format(displayable_path(line))) - elif opts.verbose: - ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) + def command(self, lib, opts, args): + # Get items from arguments + items = lib.items(ui.decargs(args)) + self.verbose = opts.verbose + par_map(self.check_item, items) def commands(self): bad_command = Subcommand('bad', @@ -148,5 +155,5 @@ class BadFiles(BeetsPlugin): action='store_true', default=False, dest='verbose', help=u'view results for both the bad and uncorrupted files' ) - bad_command.func = self.check_bad + bad_command.func = self.command return [bad_command] From 9374983e9dbb9a13d338417c1ee530a777c81757 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Sat, 18 Aug 2018 12:25:05 -0300 Subject: [PATCH 057/339] Fixed import order --- beetsplug/badfiles.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 23a9c446e..52caa9994 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -18,16 +18,17 @@ from __future__ import division, absolute_import, print_function -from beets.plugins import BeetsPlugin -from beets.ui import Subcommand -from beets.util import displayable_path, confit, par_map -from beets import ui from subprocess import check_output, CalledProcessError, list2cmdline, STDOUT + import shlex import os import errno import sys import six +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand +from beets.util import displayable_path, confit, par_map +from beets import ui class CheckerCommandException(Exception): From c3c7aa619d6fd4230c79d8f755b802e42f2087db Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Sat, 18 Aug 2018 12:13:28 -0300 Subject: [PATCH 058/339] Updated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index fc6b4cd93..bb95c46ae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -75,6 +75,8 @@ New features: Thanks to :user:`wildthyme`. * :doc:`/plugins/discogs`: The plugin has rate limiting for the discogs API now. :bug:`3081` +* The `badfiles` plugin now works in parallel (on Python 3 only). + Thanks to :user:`bemeurer`. Changes: From 9b5e681f86b8c273f874b21fb83f13e1dacdec8b Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Wed, 20 Feb 2019 07:50:22 +0100 Subject: [PATCH 059/339] docs: Add fetchart error handling fix to changelog The changelog entry also mentions that this fixes #1579. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index fc6b4cd93..bdb8b1603 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -140,6 +140,10 @@ Fixes: * The ``%title`` template function now works correctly with apostrophes. Thanks to :user:`GuilhermeHideki`. :bug:`3033` +* :doc:`/plugins/fetchart`: Added network connection error handling to backends + so that beets won't crash if a request fails. + Thanks to :user:`Holzhaus`. + :bug:`1579` * Fetchart now respects the ``ignore`` and ``ignore_hidden`` settings. :bug:`1632` .. _python-itunes: https://github.com/ocelma/python-itunes From 3e10d5d39f9f0ab12fa51b5ed6de69ab1bb1ad00 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Thu, 21 Feb 2019 12:40:54 +0100 Subject: [PATCH 060/339] badfiles: Fix #3158 by calling superclass __init__ method --- beetsplug/badfiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 52caa9994..0be08bae5 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -50,6 +50,7 @@ class CheckerCommandException(Exception): class BadFiles(BeetsPlugin): def __init__(self): + super(BadFiles, self).__init__() self.verbose = False def run_command(self, cmd): From 80f4f0a0f235b9764f516990c174f6b73695175b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 24 Feb 2019 16:06:36 -0500 Subject: [PATCH 061/339] badfiles: Fix decoding for command output Probably fixes #3165. There were several things going wrong here: 1. For some reason, this was using the *filesystem* encoding, which is what you use to decode filenames. But this was general command output, not filenames. 2. Errors in decoding threw exceptions, even though all we do with this output is show it to the user. 3. The prints were using `displayable_path`, even though the lines are *already* Unicode strings. Hopefully this cleans up that mess. --- beetsplug/badfiles.py | 6 +++--- docs/changelog.rst | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 0be08bae5..fdfbf204a 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -66,7 +66,7 @@ class BadFiles(BeetsPlugin): status = e.returncode except OSError as e: raise CheckerCommandException(cmd, e) - output = output.decode(sys.getfilesystemencoding()) + output = output.decode(sys.getdefaultencoding(), 'replace') return status, errors, [line for line in output.split("\n") if line] def check_mp3val(self, path): @@ -134,12 +134,12 @@ class BadFiles(BeetsPlugin): ui.print_(u"{}: checker exited with status {}" .format(ui.colorize('text_error', dpath), status)) for line in output: - ui.print_(u" {}".format(displayable_path(line))) + ui.print_(u" {}".format(line)) elif errors > 0: ui.print_(u"{}: checker found {} errors or warnings" .format(ui.colorize('text_warning', dpath), errors)) for line in output: - ui.print_(u" {}".format(displayable_path(line))) + ui.print_(u" {}".format(line)) elif self.verbose: ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8e02c974e..f311571d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -147,6 +147,9 @@ Fixes: Thanks to :user:`Holzhaus`. :bug:`1579` * Fetchart now respects the ``ignore`` and ``ignore_hidden`` settings. :bug:`1632` +* :doc:`/plugins/badfiles`: Avoid a crash when the underlying tool emits + undecodable output. + :bug:`3165` .. _python-itunes: https://github.com/ocelma/python-itunes From 9bb6c29d22cfa3b01c9152600a214951528b5703 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 25 Feb 2019 09:52:36 +0000 Subject: [PATCH 062/339] Always use custom formatter for formatting hook commands --- beetsplug/hook.py | 10 ++-------- docs/changelog.rst | 2 ++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/beetsplug/hook.py b/beetsplug/hook.py index de44c1b81..ac0c4acad 100644 --- a/beetsplug/hook.py +++ b/beetsplug/hook.py @@ -18,7 +18,6 @@ from __future__ import division, absolute_import, print_function import string import subprocess -import six from beets.plugins import BeetsPlugin from beets.util import shlex_split, arg_encoding @@ -46,10 +45,8 @@ class CodingFormatter(string.Formatter): See str.format and string.Formatter.format. """ - try: + if isinstance(format_string, bytes): format_string = format_string.decode(self._coding) - except UnicodeEncodeError: - pass return super(CodingFormatter, self).format(format_string, *args, **kwargs) @@ -96,10 +93,7 @@ class HookPlugin(BeetsPlugin): return # Use a string formatter that works on Unicode strings. - if six.PY2: - formatter = CodingFormatter(arg_encoding()) - else: - formatter = string.Formatter() + formatter = CodingFormatter(arg_encoding()) command_pieces = shlex_split(command) diff --git a/docs/changelog.rst b/docs/changelog.rst index f311571d5..7e98a8360 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -150,6 +150,8 @@ Fixes: * :doc:`/plugins/badfiles`: Avoid a crash when the underlying tool emits undecodable output. :bug:`3165` +* :doc:`/plugins/hook`: Fix byte string interpolation in hook commands. + :bug:`2967` :bug:`3167` .. _python-itunes: https://github.com/ocelma/python-itunes From 25549a656f47b1cd383b850c2220e3b911f067f7 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 25 Feb 2019 09:39:33 +0000 Subject: [PATCH 063/339] Add test for interpolating byte strings in hook plugin --- test/test_hook.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_hook.py b/test/test_hook.py index 39fd08959..81363c73c 100644 --- a/test/test_hook.py +++ b/test/test_hook.py @@ -110,6 +110,25 @@ class HookTest(_common.TestCase, TestHelper): self.assertTrue(os.path.isfile(path)) os.remove(path) + def test_hook_bytes_interpolation(self): + temporary_paths = [ + get_temporary_path().encode('utf-8') + for i in range(self.TEST_HOOK_COUNT) + ] + + for index, path in enumerate(temporary_paths): + self._add_hook('test_bytes_event_{0}'.format(index), + 'touch "{path}"') + + self.load_plugins('hook') + + for index, path in enumerate(temporary_paths): + plugins.send('test_bytes_event_{0}'.format(index), path=path) + + for path in temporary_paths: + self.assertTrue(os.path.isfile(path)) + os.remove(path) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From f312b1f0b778bcc4b140f41022b2549047d48684 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 25 Feb 2019 10:06:14 -0500 Subject: [PATCH 064/339] Fix #3168: several versions of munkres Require different version constraints for Pythons <3.5, =3.5, and >3.5. --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 648e6d4d4..ae8f76ff8 100755 --- a/setup.py +++ b/setup.py @@ -88,10 +88,14 @@ setup( install_requires=[ 'six>=1.9', 'mutagen>=1.33', - 'munkres~=1.0.0', 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', + ] + [ + # Avoid a version of munkres incompatible with Python 3. + 'munkres~=1.0.0' if sys.version_info < (3, 5, 0) else + 'munkres!=1.1.0,!=1.1.1' if sys.version_info < (3, 6, 0) else + 'munkres>=1.0.0', ] + ( # Use the backport of Python 3.4's `enum` module. ['enum34>=1.0.4'] if sys.version_info < (3, 4, 0) else [] From 520befb30ae424c9966e150efc0ad1609bafd2ee Mon Sep 17 00:00:00 2001 From: Vladimir Zhelezov Date: Tue, 5 Mar 2019 08:31:44 +0100 Subject: [PATCH 065/339] Fix #2826: introduce beets.util.inspect wrapper --- beets/plugins.py | 7 ++----- beets/util/inspect.py | 49 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 beets/util/inspect.py diff --git a/beets/plugins.py b/beets/plugins.py index 3fd0bec17..7019b70a0 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -17,7 +17,6 @@ from __future__ import division, absolute_import, print_function -import inspect import traceback import re from collections import defaultdict @@ -27,6 +26,7 @@ from functools import wraps import beets from beets import logging from beets import mediafile +from beets.util import inspect import six PLUGIN_NAMESPACE = 'beetsplug' @@ -127,10 +127,7 @@ class BeetsPlugin(object): value after the function returns). Also determines which params may not be sent for backwards-compatibility. """ - if six.PY2: - argspec = inspect.getargspec(func) - else: - argspec = inspect.getfullargspec(func) + argspec = inspect.getargspec(func) @wraps(func) def wrapper(*args, **kwargs): diff --git a/beets/util/inspect.py b/beets/util/inspect.py new file mode 100644 index 000000000..d114cfba2 --- /dev/null +++ b/beets/util/inspect.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Vladimir Zhelezov. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import absolute_import + +import inspect +from collections import namedtuple + +from six import PY2 + + +def getargspec(func): + if PY2: + return inspect.getargspec(func) + + ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') + + sig = inspect.signature(func) + args = [ + p.name for p in sig.parameters.values() + if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + ] + varargs = [ + p.name for p in sig.parameters.values() + if p.kind == inspect.Parameter.VAR_POSITIONAL + ] + varargs = varargs[0] if varargs else None + varkw = [ + p.name for p in sig.parameters.values() + if p.kind == inspect.Parameter.VAR_KEYWORD + ] + varkw = varkw[0] if varkw else None + defaults = tuple(p.default for p in sig.parameters.values() + if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + and p.default is not p.empty) or None + + return ArgSpec(args, varargs, varkw, defaults) From 8ae2b474cb254c0964841cdf2eb3c2bc7ea337e3 Mon Sep 17 00:00:00 2001 From: Vladimir Zhelezov Date: Wed, 6 Mar 2019 08:18:28 +0100 Subject: [PATCH 066/339] Move ArgSpec declaration to module level --- beets/util/inspect.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/util/inspect.py b/beets/util/inspect.py index d114cfba2..4090030f1 100644 --- a/beets/util/inspect.py +++ b/beets/util/inspect.py @@ -21,12 +21,13 @@ from collections import namedtuple from six import PY2 +ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') + + def getargspec(func): if PY2: return inspect.getargspec(func) - ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - sig = inspect.signature(func) args = [ p.name for p in sig.parameters.values() From 2cef0796039a09e2ef1a4454f4612c62a2159cb4 Mon Sep 17 00:00:00 2001 From: malan88 <42282289+malan88@users.noreply.github.com> Date: Thu, 7 Mar 2019 09:39:34 -0500 Subject: [PATCH 067/339] Improve specifications in the matches.ignored setting The setting is quite confusing. After having difficulty getting the autotagger to accept an id for a specific release that I knew was correct, I investigated and count [this closed issue][0] that totally explains the problem. To clarify the issue, I quoted Adrian from the issue and elaborated a bit, providing the solution. Unfortunately, the solution is undocumented. I'd like to document it, but I am unsure what the actual weighting system in `distance_weights` is. Adrian says 0.9 to ignore missing_tracks, which implies it's similar to the distance weighting in the other settings, but other settings in the default config are set to multiples of 1, implying it is not. [0]: https://github.com/beetbox/beets/issues/3064 --- docs/reference/config.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 684dea20c..b3cf84d40 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -802,6 +802,14 @@ the penalty name to the ``ignored`` setting:: The available penalties are the same as those for the :ref:`max_rec` setting. +Please note: the ignored setting will ignore any *matches* that have the specified penalties. In a sense, it maximizes the listed penalties. It will not ignore *the fact* that those matches have the penalties. For instance, with the above setting, it ignores all matches with missing tracks. It doesn't ignore the fact that there are missing tracks, which would sort of be the opposite. + +To ignore *the penalties themselves* in match selection, add something like the following to your matches settings: + + match: + distance_weights: + missing_tracks: 0.9 + .. _required: required From d1ba56e246e3e8cab0dd458b4a48a6f0c0a26697 Mon Sep 17 00:00:00 2001 From: malan88 <42282289+malan88@users.noreply.github.com> Date: Thu, 7 Mar 2019 10:41:37 -0500 Subject: [PATCH 068/339] Simplify wording, exclude ref to distance_weights Per conversation with Adrian, simplify the wording of the change. --- docs/reference/config.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index b3cf84d40..1a461bae7 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -802,13 +802,7 @@ the penalty name to the ``ignored`` setting:: The available penalties are the same as those for the :ref:`max_rec` setting. -Please note: the ignored setting will ignore any *matches* that have the specified penalties. In a sense, it maximizes the listed penalties. It will not ignore *the fact* that those matches have the penalties. For instance, with the above setting, it ignores all matches with missing tracks. It doesn't ignore the fact that there are missing tracks, which would sort of be the opposite. - -To ignore *the penalties themselves* in match selection, add something like the following to your matches settings: - - match: - distance_weights: - missing_tracks: 0.9 +For example, setting ``ignored: missing_tracks`` will skip any album matches where your audio files are missing some of the tracks. The importer will not attempt to display these matches. It does not ignore the fact that the album is missing tracks, which would allow these matches to apply more easily. To do that, you'll want to adjust the penalty for missing tracks. .. _required: From e09e6976074f9167173c50e3443e3cf00aa5f48c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 13 Mar 2019 20:54:42 -0400 Subject: [PATCH 069/339] Changelog/thanks for #3092 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7e98a8360..d925a194a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -152,6 +152,10 @@ Fixes: :bug:`3165` * :doc:`/plugins/hook`: Fix byte string interpolation in hook commands. :bug:`2967` :bug:`3167` +* Avoid some deprecation warnings with certain versions of the MusicBrainz + library. + Thanks to :user:`zhelezov`. + :bug:`2826` :bug:`3092` .. _python-itunes: https://github.com/ocelma/python-itunes From 6ee824fb0f3419a502b580b171cf9530a1a65b34 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 14 Mar 2019 09:53:32 -0400 Subject: [PATCH 070/339] Fix #3184: AttributeError in error handler --- beetsplug/beatport.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index fc412d998..0c25912b2 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -191,7 +191,7 @@ class BeatportClient(object): response = self.api.get(self._make_url(endpoint), params=kwargs) except Exception as e: raise BeatportAPIError("Error connecting to Beatport API: {}" - .format(e.message)) + .format(e)) if not response: raise BeatportAPIError( "Error {0.status_code} for '{0.request.path_url}" diff --git a/docs/changelog.rst b/docs/changelog.rst index d925a194a..eb2a32117 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -156,6 +156,8 @@ Fixes: library. Thanks to :user:`zhelezov`. :bug:`2826` :bug:`3092` +* :doc:`/plugins/beatport`: Avoid a crash when the server produces an error. + :bug:`3184` .. _python-itunes: https://github.com/ocelma/python-itunes From a46d5282bee9a7464b74d53c39266766b81d1076 Mon Sep 17 00:00:00 2001 From: jroitgrund Date: Sat, 23 Mar 2019 14:57:56 +0000 Subject: [PATCH 071/339] Clarify description of incremental_skip_later --- docs/reference/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 1a461bae7..58f92a5c1 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -509,7 +509,7 @@ incremental_skip_later Either ``yes`` or ``no``, controlling whether skipped directories are recorded in the incremental list. Set this option to ``yes`` if you would -like to revisit skipped directories later whilst using incremental +like to ignore skipped directories later whilst using incremental mode. Defaults to ``no``. .. _from_scratch: From 6a8444a3fdd669c4940fc5d450f5950bbc5f28da Mon Sep 17 00:00:00 2001 From: jroitgrund Date: Sat, 23 Mar 2019 15:03:57 +0000 Subject: [PATCH 072/339] Further clarify --- docs/reference/config.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 58f92a5c1..cc5b39c21 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -508,9 +508,10 @@ incremental_skip_later ~~~~~~~~~~~~~~~~~~~~~~ Either ``yes`` or ``no``, controlling whether skipped directories are -recorded in the incremental list. Set this option to ``yes`` if you would -like to ignore skipped directories later whilst using incremental -mode. Defaults to ``no``. +recorded in the incremental list. When set to ``yes``, skipped directories +will be recorded, and skipped later. When set to ``no``, skipped +directories won't be recorded, and beets will try to import them again +later. Defaults to ``no``. .. _from_scratch: From 1e1ddd276e0c5d9a6fa9db3cd1aeee7103495b4c Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Sat, 27 Jan 2018 18:16:09 +0100 Subject: [PATCH 073/339] Small performance optimization when fetching items from library. Fetch flexible attributes once for all items instead of once per item. For queries with larger result sets this drastically cuts down the number of issued sqlite queries. --- beets/dbcore/db.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 71810ead2..cc619b29b 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -565,6 +565,13 @@ class Results(object): a `Results` object a second time should be much faster than the first. """ + + # First fetch all flexible attributes for all items in the result. + # Doing the per-item filtering in python is faster than issuing + # one query per item to sqlite. + item_ids = [row['id'] for row in self._rows] + flex_attrs = self._get_flex_attrs(item_ids) + index = 0 # Position in the materialized objects. while index < len(self._objects) or self._rows: # Are there previously-materialized objects to produce? @@ -577,7 +584,10 @@ class Results(object): else: while self._rows: row = self._rows.pop(0) - obj = self._make_model(row) + if flex_attrs: + obj = self._make_model(row, flex_attrs[row['id']]) + else: + obj = self._make_model(row) # If there is a slow-query predicate, ensurer that the # object passes it. if not self.query or self.query.match(obj): @@ -599,20 +609,31 @@ class Results(object): # Objects are pre-sorted (i.e., by the database). return self._get_objects() - def _make_model(self, row): - # Get the flexible attributes for the object. + def _get_flex_attrs(self, ids): + # Get the flexible attributes for all ids. with self.db.transaction() as tx: + id_list = ', '.join(str(id) for id in ids) flex_rows = tx.query( - 'SELECT * FROM {0} WHERE entity_id=?'.format( - self.model_class._flex_table - ), - (row['id'],) + 'SELECT * FROM {0} WHERE entity_id IN ({1})'.format( + self.model_class._flex_table, + id_list + ) ) + # Index flexible attributes by the entity id they belong to + flex_values = dict() + for row in flex_rows: + if row['entity_id'] not in flex_values: + flex_values[row['entity_id']] = dict() + + flex_values[row['entity_id']][row['key']] = row['value'] + + return flex_values + + def _make_model(self, row, flex_values={}): cols = dict(row) values = dict((k, v) for (k, v) in cols.items() if not k[:4] == 'flex') - flex_values = dict((row['key'], row['value']) for row in flex_rows) # Construct the Python object obj = self.model_class._awaken(self.db, values, flex_values) From 31ec222e0e205dc1956a13529a3a3f382aa3ab39 Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Tue, 27 Nov 2018 17:39:09 +0100 Subject: [PATCH 074/339] Query flexible attributes already in _fetch() This resolves the query length and potential security problems, while keeping the performance benefits. --- beets/dbcore/db.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index cc619b29b..87587d057 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -32,7 +32,6 @@ from beets.dbcore import types from .query import MatchQuery, NullSort, TrueQuery import six - class DBAccessError(Exception): """The SQLite database became inaccessible. @@ -524,7 +523,8 @@ class Results(object): """An item query result set. Iterating over the collection lazily constructs LibModel objects that reflect database rows. """ - def __init__(self, model_class, rows, db, query=None, sort=None): + def __init__(self, model_class, rows, db, flex_rows, + query=None, sort=None): """Create a result set that will construct objects of type `model_class`. @@ -544,6 +544,7 @@ class Results(object): self.db = db self.query = query self.sort = sort + self.flex_rows = flex_rows # We keep a queue of rows we haven't yet consumed for # materialization. We preserve the original total number of @@ -569,8 +570,7 @@ class Results(object): # First fetch all flexible attributes for all items in the result. # Doing the per-item filtering in python is faster than issuing # one query per item to sqlite. - item_ids = [row['id'] for row in self._rows] - flex_attrs = self._get_flex_attrs(item_ids) + flex_attrs = self._get_indexed_flex_attrs() index = 0 # Position in the materialized objects. while index < len(self._objects) or self._rows: @@ -609,20 +609,11 @@ class Results(object): # Objects are pre-sorted (i.e., by the database). return self._get_objects() - def _get_flex_attrs(self, ids): - # Get the flexible attributes for all ids. - with self.db.transaction() as tx: - id_list = ', '.join(str(id) for id in ids) - flex_rows = tx.query( - 'SELECT * FROM {0} WHERE entity_id IN ({1})'.format( - self.model_class._flex_table, - id_list - ) - ) - - # Index flexible attributes by the entity id they belong to + def _get_indexed_flex_attrs(self): + """ Index flexible attributes by the entity id they belong to + """ flex_values = dict() - for row in flex_rows: + for row in self.flex_rows: if row['entity_id'] not in flex_values: flex_values[row['entity_id']] = dict() @@ -631,6 +622,8 @@ class Results(object): return flex_values def _make_model(self, row, flex_values={}): + """ Create a Model object for the given row + """ cols = dict(row) values = dict((k, v) for (k, v) in cols.items() if not k[:4] == 'flex') @@ -920,11 +913,23 @@ class Database(object): "ORDER BY {0}".format(order_by) if order_by else '', ) + # Fetch flexible attributes for items matching the main query + flex_sql = (""" + SELECT * FROM {0} WHERE entity_id IN + (SELECT id FROM {1} WHERE {2}); + """.format( + model_cls._flex_table, + model_cls._table, + where or '1', + ) + ) + with self.transaction() as tx: rows = tx.query(sql, subvals) + flex_rows = tx.query(flex_sql, subvals) return Results( - model_cls, rows, self, + model_cls, rows, self, flex_rows, None if where else query, # Slow query component. sort if sort.is_slow() else None, # Slow sort component. ) From 5494c8b2609de2443c12b2da1fff1c67bfc7bf2b Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Wed, 28 Nov 2018 11:06:21 +0100 Subject: [PATCH 075/339] Improve documentation --- beets/dbcore/db.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 87587d057..14b0553f1 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -32,6 +32,7 @@ from beets.dbcore import types from .query import MatchQuery, NullSort, TrueQuery import six + class DBAccessError(Exception): """The SQLite database became inaccessible. @@ -567,9 +568,7 @@ class Results(object): first. """ - # First fetch all flexible attributes for all items in the result. - # Doing the per-item filtering in python is faster than issuing - # one query per item to sqlite. + # Index flexible attributes by the item ID, so we have easier access flex_attrs = self._get_indexed_flex_attrs() index = 0 # Position in the materialized objects. @@ -913,7 +912,9 @@ class Database(object): "ORDER BY {0}".format(order_by) if order_by else '', ) - # Fetch flexible attributes for items matching the main query + # Fetch flexible attributes for items matching the main query. + # Doing the per-item filtering in python is faster than issuing + # one query per item to sqlite. flex_sql = (""" SELECT * FROM {0} WHERE entity_id IN (SELECT id FROM {1} WHERE {2}); From 7c568aa52894eb78c987ecbc0a72e68ce06d435f Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Sun, 2 Dec 2018 11:50:12 +0100 Subject: [PATCH 076/339] Account for items that do not have flexible attributes --- beets/dbcore/db.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 14b0553f1..24c20ef1a 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -583,10 +583,7 @@ class Results(object): else: while self._rows: row = self._rows.pop(0) - if flex_attrs: - obj = self._make_model(row, flex_attrs[row['id']]) - else: - obj = self._make_model(row) + obj = self._make_model(row, flex_attrs.get(row['id'], {})) # If there is a slow-query predicate, ensurer that the # object passes it. if not self.query or self.query.match(obj): From 439d4c1061567280f4be1ebe61ab57d7df50b521 Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Wed, 28 Nov 2018 14:06:27 +0100 Subject: [PATCH 077/339] Performance improvement: Type cast properties on demand --- beets/dbcore/db.py | 32 ++++++++++++++++++++++++++++---- beets/importer.py | 8 ++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 24c20ef1a..112b8ee93 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -177,6 +177,8 @@ class Model(object): """ self._db = db self._dirty = set() + self._raw_values_fixed = {} + self._raw_values_flex = {} self._values_fixed = {} self._values_flex = {} @@ -193,9 +195,9 @@ class Model(object): """ obj = cls(db) for key, value in fixed_values.items(): - obj._values_fixed[key] = cls._type(key).from_sql(value) + obj._raw_values_fixed[key] = value for key, value in flex_values.items(): - obj._values_flex[key] = cls._type(key).from_sql(value) + obj._raw_values_flex[key] = value return obj def __repr__(self): @@ -232,7 +234,9 @@ class Model(object): """ new = self.__class__() new._db = self._db + new._raw_values_fixed = self._raw_values_fixed.copy() new._values_fixed = self._values_fixed.copy() + new._raw_values_flex = self._raw_values_flex.copy() new._values_flex = self._values_flex.copy() new._dirty = self._dirty.copy() return new @@ -256,9 +260,18 @@ class Model(object): if key in getters: # Computed. return getters[key](self) elif key in self._fields: # Fixed. - return self._values_fixed.get(key, self._type(key).null) + if key in self._values_fixed: + return self._values_fixed[key] + elif key in self._raw_values_fixed: + self._values_fixed[key] = self._type(key).from_sql(self._raw_values_fixed[key]) + return self._values_fixed[key] + else: + return self._type(key).null elif key in self._values_flex: # Flexible. return self._values_flex[key] + elif key in self._raw_values_flex: # Flexible. + self._values_flex[key] = self._type(key).from_sql(self._raw_values_flex[key]) + return self._values_flex[key] else: raise KeyError(key) @@ -268,8 +281,12 @@ class Model(object): """ # Choose where to place the value. if key in self._fields: + if not key in self._values_fixed and key in self._raw_values_fixed: + self._values_fixed[key] = self._type(key).from_sql(self._raw_values_fixed[key]) source = self._values_fixed else: + if not key in self._values_flex and key in self._raw_values_flex: + self._values_flex[key] = self._type(key).from_sql(self._raw_values_flex[key]) source = self._values_flex # If the field has a type, filter the value. @@ -294,6 +311,11 @@ class Model(object): """ if key in self._values_flex: # Flexible. del self._values_flex[key] + if key in self._raw_values_flex: + del self._raw_values_flex[key] + self._dirty.add(key) # Mark for dropping on store. + elif key in self._raw_values_flex: # Flexible + del self._raw_values_flex[key] self._dirty.add(key) # Mark for dropping on store. elif key in self._fields: # Fixed setattr(self, key, self._type(key).null) @@ -307,7 +329,7 @@ class Model(object): `computed` parameter controls whether computed (plugin-provided) fields are included in the key list. """ - base_keys = list(self._fields) + list(self._values_flex.keys()) + base_keys = list(self._fields) + list(self._values_flex.keys()) + list(self._raw_values_flex.keys()) if computed: return base_keys + list(self._getters().keys()) else: @@ -436,7 +458,9 @@ class Model(object): self._check_db() stored_obj = self._db._get(type(self), self.id) assert stored_obj is not None, u"object {0} not in DB".format(self.id) + self._raw_values_fixed = {} self._values_fixed = {} + self._raw_values_flex = {} self._values_flex = {} self.update(dict(stored_obj)) self.clear_dirty() diff --git a/beets/importer.py b/beets/importer.py index 889f1297e..50e4545c5 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -785,14 +785,14 @@ class ImportTask(BaseImportTask): replaced_album = self.replaced_albums.get(self.album.path) if replaced_album: self.album.added = replaced_album.added - self.album.update(replaced_album._values_flex) + self.album.update(replaced_album._raw_values_flex) self.album.artpath = replaced_album.artpath self.album.store() log.debug( u'Reimported album: added {0}, flexible ' u'attributes {1} from album {2} for {3}', self.album.added, - replaced_album._values_flex.keys(), + replaced_album._raw_values_flex.keys(), replaced_album.id, displayable_path(self.album.path) ) @@ -809,11 +809,11 @@ class ImportTask(BaseImportTask): dup_item.id, displayable_path(item.path) ) - item.update(dup_item._values_flex) + item.update(dup_item._raw_values_flex) log.debug( u'Reimported item flexible attributes {0} ' u'from item {1} for {2}', - dup_item._values_flex.keys(), + dup_item._raw_values_flex.keys(), dup_item.id, displayable_path(item.path) ) From ea307f62d1803b29ea03982f8e733e17156d7236 Mon Sep 17 00:00:00 2001 From: "jroitgrund@gmail.com" Date: Wed, 27 Mar 2019 01:41:42 +0000 Subject: [PATCH 078/339] Don't move album art twice --- .gitignore | 1 + beets/ui/commands.py | 2 +- test/test_ui.py | 10 +++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b93d93305..64f08abe5 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ ENV/ /.project /.pydevproject /.settings +.vscode diff --git a/beets/ui/commands.py b/beets/ui/commands.py index a38be7a15..c89dbb6db 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1177,7 +1177,7 @@ def update_items(lib, query, album, move, pretend, fields): # Manually moving and storing the album. items = list(album.items()) for item in items: - item.move(store=False) + item.move(store=False, with_album=False) item.store(fields=fields) album.move(store=False) album.store(fields=fields) diff --git a/test/test_ui.py b/test/test_ui.py index b9039d236..bc9bb4829 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -507,10 +507,14 @@ class UpdateTest(_common.TestCase): # Copy a file into the library. self.lib = library.Library(':memory:', self.libdir) item_path = os.path.join(_common.RSRC, b'full.mp3') + item_path_two = os.path.join(_common.RSRC, b'full.flac') self.i = library.Item.from_path(item_path) + self.i2 = library.Item.from_path(item_path_two) self.lib.add(self.i) + self.lib.add(self.i2) self.i.move(operation=MoveOperation.COPY) - self.album = self.lib.add_album([self.i]) + self.i2.move(operation=MoveOperation.COPY) + self.album = self.lib.add_album([self.i, self.i2]) # Album art. artfile = os.path.join(self.temp_dir, b'testart.jpg') @@ -531,12 +535,14 @@ class UpdateTest(_common.TestCase): def test_delete_removes_item(self): self.assertTrue(list(self.lib.items())) os.remove(self.i.path) + os.remove(self.i2.path) self._update() self.assertFalse(list(self.lib.items())) def test_delete_removes_album(self): self.assertTrue(self.lib.albums()) os.remove(self.i.path) + os.remove(self.i2.path) self._update() self.assertFalse(self.lib.albums()) @@ -544,6 +550,7 @@ class UpdateTest(_common.TestCase): artpath = self.album.artpath self.assertExists(artpath) os.remove(self.i.path) + os.remove(self.i2.path) self._update() self.assertNotExists(artpath) @@ -607,6 +614,7 @@ class UpdateTest(_common.TestCase): self._update(move=True) album = self.lib.albums()[0] self.assertNotEqual(artpath, album.artpath) + self.assertIsNotNone(album.artpath) def test_selective_modified_album_metadata_moved(self): mf = MediaFile(syspath(self.i.path)) From 31b393586a8a8d1252b424a324ab1c897907a674 Mon Sep 17 00:00:00 2001 From: "jroitgrund@gmail.com" Date: Wed, 27 Mar 2019 01:43:42 +0000 Subject: [PATCH 079/339] Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index eb2a32117..a77cb4ceb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -142,6 +142,7 @@ Fixes: * The ``%title`` template function now works correctly with apostrophes. Thanks to :user:`GuilhermeHideki`. :bug:`3033` +* When updating the database, beets no longer tries to move album art twice. * :doc:`/plugins/fetchart`: Added network connection error handling to backends so that beets won't crash if a request fails. Thanks to :user:`Holzhaus`. From 9abfc45adc5dd864840c9b72f35239c6204d39fa Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 26 Mar 2019 22:11:12 -0400 Subject: [PATCH 080/339] Changelog issue link for #3189 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a77cb4ceb..1fcc691a5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -143,6 +143,7 @@ Fixes: Thanks to :user:`GuilhermeHideki`. :bug:`3033` * When updating the database, beets no longer tries to move album art twice. + :bug:`3189` * :doc:`/plugins/fetchart`: Added network connection error handling to backends so that beets won't crash if a request fails. Thanks to :user:`Holzhaus`. From be12a89372b96c1502733a4d1ade45e1deecd5f9 Mon Sep 17 00:00:00 2001 From: Alexei Datskevich <4024096+translit@users.noreply.github.com> Date: Sat, 30 Mar 2019 08:34:13 +0100 Subject: [PATCH 081/339] Fix PyYAML yaml.load(input) Deprecation --- beetsplug/lastgenre/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index 2f660206e..c3a4332bd 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -152,7 +152,7 @@ class LastGenrePlugin(plugins.BeetsPlugin): self._log.debug('Loading canonicalization tree {0}', c14n_filename) c14n_filename = normpath(c14n_filename) with codecs.open(c14n_filename, 'r', encoding='utf-8') as f: - genres_tree = yaml.load(f) + genres_tree = yaml.safe_load(f) flatten_tree(genres_tree, [], self.c14n_branches) @property From 777cfbbf610522c0b655659f8f11d519714606b5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Mar 2019 12:37:53 -0400 Subject: [PATCH 082/339] Travis: temporarily pin tox to <=3.8.1 To avoid a Python 2 bug introduced in tox 3.8.2 but not fixed yet: https://github.com/tox-dev/tox/issues/1234 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 46c1bd8d6..f15f3cefb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ addons: # To install dependencies, tell tox to do everything but actually running the # test. install: - - travis_retry pip install tox sphinx + - travis_retry pip install tox<=3.8.1 sphinx # upgrade requests to satisfy sphinx linkcheck (for building man pages) - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then pip install -U requests; fi - travis_retry tox -e $TOX_ENV --notest From ccbab10090c6f37788419c7a5bffe748cfdc72aa Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Mar 2019 12:39:42 -0400 Subject: [PATCH 083/339] Add the full set of __future__ imports in new file --- beets/util/inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/util/inspect.py b/beets/util/inspect.py index 4090030f1..9815a561a 100644 --- a/beets/util/inspect.py +++ b/beets/util/inspect.py @@ -13,7 +13,7 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -from __future__ import absolute_import +from __future__ import division, absolute_import, print_function import inspect from collections import namedtuple From db107e3c4632dc77e50972a6e456bf999981653b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Mar 2019 12:40:52 -0400 Subject: [PATCH 084/339] Appveyor: temporarily pin tox version As in 777cfbbf610522c0b655659f8f11d519714606b5 for Travis. --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 938d3a5a4..b33cffd2d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,7 +22,7 @@ install: - cinst imagemagick -y # TODO: remove --allow-empty-checksums when unrar offers a proper checksum - cinst unrar -y --allow-empty-checksums - - "%PYTHON%/Scripts/pip.exe install tox" + - "%PYTHON%/Scripts/pip.exe install tox<=3.8.1" - "%PYTHON%/Scripts/tox.exe -e %TOX_ENV% --notest" test_script: From 9b3bebfcd77a57c7698d1a26d88acde883cec0a8 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Mar 2019 12:48:34 -0400 Subject: [PATCH 085/339] Travis: fix shell syntax for version specifier --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f15f3cefb..492b90575 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ addons: # To install dependencies, tell tox to do everything but actually running the # test. install: - - travis_retry pip install tox<=3.8.1 sphinx + - travis_retry pip install 'tox<=3.8.1' sphinx # upgrade requests to satisfy sphinx linkcheck (for building man pages) - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then pip install -U requests; fi - travis_retry tox -e $TOX_ENV --notest From 15f86c411e08b5c6e837fdd8828263c6c3ec794c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Mar 2019 12:57:51 -0400 Subject: [PATCH 086/339] Appveyor: fix version specifier syntax --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b33cffd2d..cb747fe1f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,7 +22,7 @@ install: - cinst imagemagick -y # TODO: remove --allow-empty-checksums when unrar offers a proper checksum - cinst unrar -y --allow-empty-checksums - - "%PYTHON%/Scripts/pip.exe install tox<=3.8.1" + - '%PYTHON%/Scripts/pip.exe install "tox<=3.8.1"' - "%PYTHON%/Scripts/tox.exe -e %TOX_ENV% --notest" test_script: From 97c3590aecd868ddbc0250f8c0530f3159d8a19d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Mar 2019 13:11:13 -0400 Subject: [PATCH 087/339] Changelog for #3192 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1fcc691a5..afd2e23b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -160,6 +160,9 @@ Fixes: :bug:`2826` :bug:`3092` * :doc:`/plugins/beatport`: Avoid a crash when the server produces an error. :bug:`3184` +* :doc:`/plugins/lastgenre`: Avoid a deprecation warning from the YAML + library. + :bug:`3192` .. _python-itunes: https://github.com/ocelma/python-itunes From 078c583434eac62789b988021c596585c362b3e1 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 26 Mar 2019 22:00:30 +1100 Subject: [PATCH 088/339] BPD tests: add basic test framework A simple TCP client wraps the low level protocol to allow testing of the BPD server as a black box. --- test/test_player.py | 102 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/test/test_player.py b/test/test_player.py index 523a39d1b..d063c9445 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -20,6 +20,12 @@ from __future__ import division, absolute_import, print_function import unittest from beetsplug import bpd +import multiprocessing as mp +import socket +import time + +from test.helper import TestHelper + class CommandParseTest(unittest.TestCase): def test_no_args(self): @@ -63,6 +69,102 @@ class CommandParseTest(unittest.TestCase): self.assertEqual(c.args, [u'hello \\ there']) +class MPCResponse(object): + def __init__(self, raw_response): + self.body = b'\n'.join(raw_response.split(b'\n')[:-2]) + self.status = raw_response.split(b'\n')[-2] + self.ok = self.status.startswith(b'OK') + self.err = self.status.startswith(b'ACK') + if self.err: + print(self.status) + + +class MPCClient(object): + def __init__(self, host, port, do_hello=True): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((host, port)) + self.buf = b'' + if do_hello: + hello = self.get_response() + if not hello.ok: + raise RuntimeError('Bad hello: {}'.format(hello.status)) + + def __del__(self): + self.sock.close() + + def get_response(self): + response = b'' + while True: + line = self.readline() + response += line + if line.startswith(b'OK') or line.startswith(b'ACK'): + return MPCResponse(response) + + def send_command(self, command, *args): + cmd = [command] + for arg in args: + if b' ' in arg: + cmd.append(b'"{}"'.format(arg)) + else: + cmd.append(arg) + self.sock.sendall(b' '.join(cmd) + b'\n') + return self.get_response() + + def readline(self, terminator=b'\n', bufsize=1024): + ''' Reads a line of data from the socket. ''' + + while True: + if terminator in self.buf: + line, self.buf = self.buf.split(terminator, 1) + line += terminator + return line + data = self.sock.recv(bufsize) + if data: + self.buf += data + else: + line = self.buf + self.buf = b'' + return line + + +class BPDTest(unittest.TestCase, TestHelper): + def setUp(self): + self.setup_beets() + self.load_plugins('bpd') + self.item = self.add_item() + self.lib.add_album([self.item]) + + self.server_proc, self.client = self.make_server_client() + + def tearDown(self): + self.server_proc.terminate() + self.teardown_beets() + self.unload_plugins() + + def make_server(self, host, port, password=None): + bpd_server = bpd.BaseServer(host, port, password) + server_proc = mp.Process(target=bpd_server.run) + server_proc.start() + return server_proc + + def make_client(self, host='localhost', port=9876, do_hello=True): + return MPCClient(host, port, do_hello) + + def make_server_client(self, host='localhost', port=9876, password=None): + server_proc = self.make_server(host, port, password) + time.sleep(1) + client = self.make_client(host, port) + return server_proc, client + + def test_server_hello(self): + alt_client = self.make_client(do_hello=False) + self.assertEqual(alt_client.readline(), b'OK MPD 0.13.0\n') + + def test_cmd_ping(self): + response = self.client.send_command(b'ping') + self.assertTrue(response.ok) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 59e66aaa7f91a99038d7cbd0a928c345f88e8823 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 26 Mar 2019 22:00:54 +1100 Subject: [PATCH 089/339] BPD tests: password command --- test/test_player.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_player.py b/test/test_player.py index d063c9445..e332d6fa1 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -164,6 +164,26 @@ class BPDTest(unittest.TestCase, TestHelper): response = self.client.send_command(b'ping') self.assertTrue(response.ok) + def test_cmd_password(self): + self.server_proc.terminate() + self.server_proc, self.client = self.make_server_client( + password='abc123') + + response = self.client.send_command(b'status') + self.assertTrue(response.err) + self.assertEqual(response.status, + b'ACK [4@0] {} insufficient privileges') + + response = self.client.send_command(b'password', b'wrong') + self.assertTrue(response.err) + self.assertEqual(response.status, + b'ACK [3@0] {password} incorrect password') + + response = self.client.send_command(b'password', b'abc123') + self.assertTrue(response.ok) + response = self.client.send_command(b'status') + self.assertTrue(response.ok) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From acc3f3efb00613d8019b6c83b78031911bb79db8 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 26 Mar 2019 23:49:34 +1100 Subject: [PATCH 090/339] BPD tests: check all specified commands known The current MPD spec is several versions ahead of BPD. These tests just compare the full specified list of commands against what BPD claims to support (without actually testing their implementations). It's handy as a measure of progress against the specification, but can perhaps be dropped later once all of the individual protocol methods have tests. --- test/test_player.py | 77 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/test/test_player.py b/test/test_player.py index e332d6fa1..e61a59429 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -24,6 +24,7 @@ import multiprocessing as mp import socket import time +from test import _common from test.helper import TestHelper @@ -127,6 +128,15 @@ class MPCClient(object): return line +def implements(commands, expectedFailure=False): + def _test(self): + response = self.client.send_command(b'commands') + implemented = {line[9:] for line in response.body.split(b'\n')} + self.assertEqual(commands.intersection(implemented), commands) + return unittest.expectedFailure(_test) if expectedFailure else _test + + +@_common.slow_test() class BPDTest(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets() @@ -152,7 +162,7 @@ class BPDTest(unittest.TestCase, TestHelper): def make_server_client(self, host='localhost', port=9876, password=None): server_proc = self.make_server(host, port, password) - time.sleep(1) + time.sleep(0.1) # wait for the server to start client = self.make_client(host, port) return server_proc, client @@ -184,6 +194,71 @@ class BPDTest(unittest.TestCase, TestHelper): response = self.client.send_command(b'status') self.assertTrue(response.ok) + test_implements_query = implements({ + b'clearerror', b'currentsong', b'idle', b'status', b'stats', + }, expectedFailure=True) + + test_implements_playback = implements({ + b'consume', b'crossfade', b'mixrampdb', b'mixrampdelay', b'random', + b'repeat', b'setvol', b'single', b'replay_gain_mode', + b'replay_gain_status', b'volume', + }, expectedFailure=True) + + test_implements_control = implements({ + b'next', b'pause', b'play', b'playid', b'previous', b'seek', + b'seekid', b'seekcur', b'stop', + }, expectedFailure=True) + + test_implements_queue = implements({ + b'add', b'addid', b'clear', b'delete', b'deleteid', b'move', + b'moveid', b'playlist', b'playlistfind', b'playlistid', + b'playlistinfo', b'playlistsearch', b'plchanges', + b'plchangesposid', b'prio', b'prioid', b'rangeid', b'shuffle', + b'swap', b'swapid', b'addtagid', b'cleartagid', + }, expectedFailure=True) + + test_implements_playlists = implements({ + b'listplaylist', b'listplaylistinfo', b'listplaylists', b'load', + b'playlistadd', b'playlistclear', b'playlistdelete', + b'playlistmove', b'rename', b'rm', b'save', + }, expectedFailure=True) + + test_implements_database = implements({ + b'albumart', b'count', b'find', b'findadd', b'list', b'listall', + b'listallinfo', b'listfiles', b'lsinfo', b'readcomments', + b'search', b'searchadd', b'searchaddpl', b'update', b'rescan', + }, expectedFailure=True) + + test_implements_mounts = implements({ + b'mount', b'unmount', b'listmounts', b'listneighbors', + }, expectedFailure=True) + + test_implements_stickers = implements({ + b'sticker', + }, expectedFailure=True) + + test_implements_connection = implements({ + b'close', b'kill', b'password', b'ping', b'tagtypes', + }, expectedFailure=True) + + test_implements_partitions = implements({ + b'partition', b'listpartitions', b'newpartition', + }, expectedFailure=True) + + test_implements_devices = implements({ + b'disableoutput', b'enableoutput', b'toggleoutput', b'outputs', + }, expectedFailure=True) + + test_implements_reflection = implements({ + b'config', b'commands', b'notcommands', b'urlhandlers', + b'decoders', + }, expectedFailure=True) + + test_implements_peers = implements({ + b'subscribe', b'unsubscribe', b'channels', b'readmessages', + b'sendmessage', + }, expectedFailure=True) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From acd66ce560bfe1fcd00900b55b599c236e76d5b4 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 27 Mar 2019 00:02:35 +1100 Subject: [PATCH 091/339] BPD tests: run BPD Server with a fake gstplayer --- test/test_player.py | 96 ++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index e61a59429..6ba92db06 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -18,8 +18,8 @@ from __future__ import division, absolute_import, print_function import unittest -from beetsplug import bpd - +import sys +import imp import multiprocessing as mp import socket import time @@ -27,6 +27,25 @@ import time from test import _common from test.helper import TestHelper +from beetsplug import bpd + +# Intercept and mock the GstPlayer player: +gstplayer = imp.new_module('beetsplug.bpg.gstplayer') +gstplayer.GstPlayer = type('fake.GstPlayer', (), { + '__init__': lambda self, callback: None, + 'playing': False, + 'volume': 0.0, + 'run': lambda self: None, + 'time': lambda self: (0, 0), + 'play': lambda self: None, + 'pause': lambda self: None, + 'play_file': lambda self, path: None, + 'seek': lambda self, pos: None, + 'stop': lambda self: None, + }) +sys.modules['beetsplug.bpd.gstplayer'] = gstplayer +bpd.gstplayer = gstplayer + class CommandParseTest(unittest.TestCase): def test_no_args(self): @@ -94,6 +113,11 @@ class MPCClient(object): self.sock.close() def get_response(self): + """ Wait for a full server response and wrap it in a helper class. + If the request was a batch request then this will return a list of + `MPCResponse`s, one for each processed subcommand. + """ + response = b'' while True: line = self.readline() @@ -112,7 +136,8 @@ class MPCClient(object): return self.get_response() def readline(self, terminator=b'\n', bufsize=1024): - ''' Reads a line of data from the socket. ''' + """ Reads a line of data from the socket. + """ while True: if terminator in self.buf: @@ -144,7 +169,8 @@ class BPDTest(unittest.TestCase, TestHelper): self.item = self.add_item() self.lib.add_album([self.item]) - self.server_proc, self.client = self.make_server_client() + self.server_proc = None + self.client = self.make_server_client() def tearDown(self): self.server_proc.terminate() @@ -152,48 +178,26 @@ class BPDTest(unittest.TestCase, TestHelper): self.unload_plugins() def make_server(self, host, port, password=None): - bpd_server = bpd.BaseServer(host, port, password) - server_proc = mp.Process(target=bpd_server.run) - server_proc.start() - return server_proc + bpd_server = bpd.Server(self.lib, host, port, password) + self.server = bpd_server + if self.server_proc: + self.server_proc.terminate() + self.server_proc = mp.Process(target=bpd_server.run) + self.server_proc.start() def make_client(self, host='localhost', port=9876, do_hello=True): return MPCClient(host, port, do_hello) def make_server_client(self, host='localhost', port=9876, password=None): - server_proc = self.make_server(host, port, password) + self.make_server(host, port, password) time.sleep(0.1) # wait for the server to start client = self.make_client(host, port) - return server_proc, client + return client def test_server_hello(self): alt_client = self.make_client(do_hello=False) self.assertEqual(alt_client.readline(), b'OK MPD 0.13.0\n') - def test_cmd_ping(self): - response = self.client.send_command(b'ping') - self.assertTrue(response.ok) - - def test_cmd_password(self): - self.server_proc.terminate() - self.server_proc, self.client = self.make_server_client( - password='abc123') - - response = self.client.send_command(b'status') - self.assertTrue(response.err) - self.assertEqual(response.status, - b'ACK [4@0] {} insufficient privileges') - - response = self.client.send_command(b'password', b'wrong') - self.assertTrue(response.err) - self.assertEqual(response.status, - b'ACK [3@0] {password} incorrect password') - - response = self.client.send_command(b'password', b'abc123') - self.assertTrue(response.ok) - response = self.client.send_command(b'status') - self.assertTrue(response.ok) - test_implements_query = implements({ b'clearerror', b'currentsong', b'idle', b'status', b'stats', }, expectedFailure=True) @@ -239,7 +243,29 @@ class BPDTest(unittest.TestCase, TestHelper): test_implements_connection = implements({ b'close', b'kill', b'password', b'ping', b'tagtypes', - }, expectedFailure=True) + }) + + def test_cmd_password(self): + self.client = self.make_server_client(password='abc123') + + response = self.client.send_command(b'status') + self.assertTrue(response.err) + self.assertEqual(response.status, + b'ACK [4@0] {} insufficient privileges') + + response = self.client.send_command(b'password', b'wrong') + self.assertTrue(response.err) + self.assertEqual(response.status, + b'ACK [3@0] {password} incorrect password') + + response = self.client.send_command(b'password', b'abc123') + self.assertTrue(response.ok) + response = self.client.send_command(b'status') + self.assertTrue(response.ok) + + def test_cmd_ping(self): + response = self.client.send_command(b'ping') + self.assertTrue(response.ok) test_implements_partitions = implements({ b'partition', b'listpartitions', b'newpartition', From 9a5be1f9717c46b1dc5ef84dbb161ccf7dc9f008 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 27 Mar 2019 00:48:29 +1100 Subject: [PATCH 092/339] BPD tests: test supported tag types MPD now supports more fields ("tags") than what BPD advertises. Fixing this should be a simple task of extending the mapping of fields to tags in BPD's implementation. --- test/test_player.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/test_player.py b/test/test_player.py index 6ba92db06..ca41b9382 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -124,6 +124,8 @@ class MPCClient(object): response += line if line.startswith(b'OK') or line.startswith(b'ACK'): return MPCResponse(response) + elif not line: + raise RuntimeError('Empty response') def send_command(self, command, *args): cmd = [command] @@ -132,7 +134,8 @@ class MPCClient(object): cmd.append(b'"{}"'.format(arg)) else: cmd.append(arg) - self.sock.sendall(b' '.join(cmd) + b'\n') + request = b' '.join(cmd) + b'\n' + self.sock.sendall(request) return self.get_response() def readline(self, terminator=b'\n', bufsize=1024): @@ -144,6 +147,7 @@ class MPCClient(object): line, self.buf = self.buf.split(terminator, 1) line += terminator return line + self.sock.settimeout(1) data = self.sock.recv(bufsize) if data: self.buf += data @@ -267,6 +271,24 @@ class BPDTest(unittest.TestCase, TestHelper): response = self.client.send_command(b'ping') self.assertTrue(response.ok) + @unittest.expectedFailure + def test_cmd_tagtypes(self): + response = self.client.send_command(b'tagtypes') + types = {line[9:].lower() for line in response.body.split(b'\n')} + self.assertEqual({ + b'artist', b'artistsort', b'album', b'albumsort', b'albumartist', + b'albumartistsort', b'title', b'track', b'name', b'genre', b'date', + b'composer', b'performer', b'comment', b'disc', b'label', + b'musicbrainz_artistid', b'musicbrainz_albumid', + b'musicbrainz_albumartistid', b'musicbrainz_trackid', + b'musicbrainz_releasetrackid', b'musicbrainz_workid', + }, types) + + @unittest.expectedFailure + def test_tagtypes_mask(self): + response = self.client.send_command(b'tagtypes', b'clear') + self.assertTrue(response.ok) + test_implements_partitions = implements({ b'partition', b'listpartitions', b'newpartition', }, expectedFailure=True) From 7302cd6ab5b818b0d832b79e870d3a198695cda2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 27 Mar 2019 16:27:46 +1100 Subject: [PATCH 093/339] BPD tests: test helper for BPD batch commands The MPD protocol allows batching commands. There are two choices for the first message that announced a batch is starting, and we just go for the one that causes the server to respond with a marker message between each individual response, since that's easier. This might need to be tweaked in order to test the behaviour with the other batch indicator. --- test/test_player.py | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index ca41b9382..8182f4d18 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -93,9 +93,10 @@ class MPCResponse(object): def __init__(self, raw_response): self.body = b'\n'.join(raw_response.split(b'\n')[:-2]) self.status = raw_response.split(b'\n')[-2] - self.ok = self.status.startswith(b'OK') + self.ok = (self.status.startswith(b'OK') or + self.status.startswith(b'list_OK')) self.err = self.status.startswith(b'ACK') - if self.err: + if not self.ok: print(self.status) @@ -119,22 +120,51 @@ class MPCClient(object): """ response = b'' + responses = [] while True: line = self.readline() response += line if line.startswith(b'OK') or line.startswith(b'ACK'): - return MPCResponse(response) + if any(responses): + if line.startswith(b'ACK'): + responses.append(MPCResponse(response)) + return responses + else: + return MPCResponse(response) + if line.startswith(b'list_OK'): + responses.append(MPCResponse(response)) + response = b'' elif not line: - raise RuntimeError('Empty response') + raise RuntimeError('Unexpected response: {!r}'.format(line)) - def send_command(self, command, *args): + def serialise_command(self, command, *args): cmd = [command] for arg in args: if b' ' in arg: - cmd.append(b'"{}"'.format(arg)) + cmd.append(b'"' + arg + b'"') else: cmd.append(arg) - request = b' '.join(cmd) + b'\n' + return b' '.join(cmd) + b'\n' + + def send_command(self, command, *args): + request = self.serialise_command(command, *args) + self.sock.sendall(request) + return self.get_response() + + def send_commands(self, *commands): + """ Use MPD command batching to send multiple commands at once. + Each item of commands is a tuple containing a command followed by + any arguments. + """ + + requests = [] + for command_and_args in commands: + command = command_and_args[0] + args = command_and_args[1:] + requests.append(self.serialise_command(command, *args)) + requests.insert(0, b'command_list_ok_begin\n') + requests.append(b'command_list_end\n') + request = b''.join(requests) self.sock.sendall(request) return self.get_response() From 5d2b883b726ffb06c67b7a02dccd269f635abd0d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 27 Mar 2019 16:36:37 +1100 Subject: [PATCH 094/339] BPD tests: test several simple commands --- test/test_player.py | 57 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 8182f4d18..c3ce43638 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -200,8 +200,13 @@ class BPDTest(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets() self.load_plugins('bpd') - self.item = self.add_item() - self.lib.add_album([self.item]) + self.item1 = self.add_item(title='Track One Title', + album='Album Title', artist='Artist Name', + track=1) + self.item2 = self.add_item(title='Track Two Title', + album='Album Title', artist='Artist Name', + track=2) + self.lib.add_album([self.item1, self.item2]) self.server_proc = None self.client = self.make_server_client() @@ -247,6 +252,16 @@ class BPDTest(unittest.TestCase, TestHelper): b'seekid', b'seekcur', b'stop', }, expectedFailure=True) + def test_cmd_play(self): + responses = self.client.send_commands( + (b'add', b'Artist Name/Album Title/01 Track One Title.mp3'), + (b'status',), + (b'play',), + (b'status',)) + self.assertIn(b'state: stop', responses[1].body.split(b'\n')) + self.assertTrue(responses[2].ok) + self.assertIn(b'state: play', responses[3].body.split(b'\n')) + test_implements_queue = implements({ b'add', b'addid', b'clear', b'delete', b'deleteid', b'move', b'moveid', b'playlist', b'playlistfind', b'playlistid', @@ -255,6 +270,27 @@ class BPDTest(unittest.TestCase, TestHelper): b'swap', b'swapid', b'addtagid', b'cleartagid', }, expectedFailure=True) + def test_cmd_add(self): + response = self.client.send_command( + b'add', + b'Artist Name/Album Title/01 Track One Title.mp3') + self.assertTrue(response.ok) + + def test_cmd_playlistinfo(self): + responses = self.client.send_commands( + (b'add', b'Artist Name/Album Title/01 Track One Title.mp3'), + (b'playlistinfo',), + (b'playlistinfo', b'0')) + self.assertTrue(responses[1].ok) + self.assertTrue(responses[2].ok) + self.assertEqual(responses[1].body, responses[2].body) + + response = self.client.send_command(b'playlistinfo', b'1') + self.assertTrue(response.err) + self.assertEqual( + b'ACK [2@0] {playlistinfo} argument out of range', + response.status) + test_implements_playlists = implements({ b'listplaylist', b'listplaylistinfo', b'listplaylists', b'load', b'playlistadd', b'playlistclear', b'playlistdelete', @@ -267,6 +303,23 @@ class BPDTest(unittest.TestCase, TestHelper): b'search', b'searchadd', b'searchaddpl', b'update', b'rescan', }, expectedFailure=True) + def test_cmd_search(self): + response = self.client.send_command(b'search', b'track', b'1') + self.assertEqual( + b'file: Artist Name/Album Title/01 Track One Title.mp3', + response.body.split(b'\n')[0]) + + def test_cmd_list_simple(self): + response = self.client.send_command(b'list', b'album') + self.assertEqual(b'Album: Album Title', response.body) + + response = self.client.send_command(b'list', b'track') + self.assertEqual(b'Track: 1\nTrack: 2', response.body) + + def test_cmd_count(self): + response = self.client.send_command(b'count', b'track', b'1') + self.assertEqual(b'songs: 1\nplaytime: 0', response.body) + test_implements_mounts = implements({ b'mount', b'unmount', b'listmounts', b'listneighbors', }, expectedFailure=True) From 76221428f8dc4526ebf1f2bdb14a33433040de6b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 27 Mar 2019 19:33:39 +1100 Subject: [PATCH 095/339] Add a DummyIn::close no-op implementation The BOD tests are currently forking a process with a server running, and this attempts to close stdin. Tests were failing due to DummyIn not implementing the close() method. Adding this simple no-op does the trick to allow forking and seems like a harmless addition. --- test/_common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/_common.py b/test/_common.py index f5e65ca76..99f2e968f 100644 --- a/test/_common.py +++ b/test/_common.py @@ -284,6 +284,9 @@ class DummyIn(object): else: self.buf.append(s + '\n') + def close(self): + pass + def readline(self): if not self.buf: if self.out: From d9537f27dc97d93746c9f35406d9059f25f95e13 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 27 Mar 2019 19:44:35 +1100 Subject: [PATCH 096/339] BPD tests: fix CI failures --- test/test_player.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index c3ce43638..b7bc01528 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -31,7 +31,7 @@ from beetsplug import bpd # Intercept and mock the GstPlayer player: gstplayer = imp.new_module('beetsplug.bpg.gstplayer') -gstplayer.GstPlayer = type('fake.GstPlayer', (), { +gstplayer.GstPlayer = type('GstPlayer', (), { '__init__': lambda self, callback: None, 'playing': False, 'volume': 0.0, @@ -97,7 +97,7 @@ class MPCResponse(object): self.status.startswith(b'list_OK')) self.err = self.status.startswith(b'ACK') if not self.ok: - print(self.status) + print(self.status.decode('utf-8')) class MPCClient(object): @@ -187,7 +187,7 @@ class MPCClient(object): return line -def implements(commands, expectedFailure=False): +def implements(commands, expectedFailure=False): # noqa: N803 def _test(self): response = self.client.send_command(b'commands') implemented = {line[9:] for line in response.body.split(b'\n')} From 767441d5d50610ac49e245b341ee641412294776 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 28 Mar 2019 13:00:08 +1100 Subject: [PATCH 097/339] BPD tests: improve test helpers Decode the bytes to strings: the MPD protocol specifies that the communications are all in UTF-8. Also parse the body into a dict since this is typically more convenient than having to do it manually in each test. --- test/test_player.py | 175 +++++++++++++++++++++++++------------------- 1 file changed, 100 insertions(+), 75 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index b7bc01528..dfe583fca 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -91,13 +91,37 @@ class CommandParseTest(unittest.TestCase): class MPCResponse(object): def __init__(self, raw_response): - self.body = b'\n'.join(raw_response.split(b'\n')[:-2]) - self.status = raw_response.split(b'\n')[-2] - self.ok = (self.status.startswith(b'OK') or - self.status.startswith(b'list_OK')) - self.err = self.status.startswith(b'ACK') + body = b'\n'.join(raw_response.split(b'\n')[:-2]).decode('utf-8') + self.data = self._parse_body(body) + self.status = raw_response.split(b'\n')[-2].decode('utf-8') + self.ok = (self.status.startswith('OK') or + self.status.startswith('list_OK')) + self.err = self.status.startswith('ACK') if not self.ok: - print(self.status.decode('utf-8')) + print(self.status) + + def _parse_body(self, body): + """ Messages are generally in the format "header: content". + Convert them into a dict, storing the values for repeated headers as + lists of strings, and non-repeated ones as string. + """ + data = {} + repeated_headers = set() + for line in body.split('\n'): + if not line: + continue + if ':' not in line: + raise RuntimeError('Unexpected line: {!r}'.format(line)) + header, content = line.split(':', 1) + content = content.lstrip() + if header in repeated_headers: + data[header].append(content) + elif header in data: + data[header] = [data[header], content] + repeated_headers.add(header) + else: + data[header] = content + return data class MPCClient(object): @@ -138,8 +162,8 @@ class MPCClient(object): raise RuntimeError('Unexpected response: {!r}'.format(line)) def serialise_command(self, command, *args): - cmd = [command] - for arg in args: + cmd = [command.encode('utf-8')] + for arg in [a.encode('utf-8') for a in args]: if b' ' in arg: cmd.append(b'"' + arg + b'"') else: @@ -189,8 +213,8 @@ class MPCClient(object): def implements(commands, expectedFailure=False): # noqa: N803 def _test(self): - response = self.client.send_command(b'commands') - implemented = {line[9:] for line in response.body.split(b'\n')} + response = self.client.send_command('commands') + implemented = response.data['command'] self.assertEqual(commands.intersection(implemented), commands) return unittest.expectedFailure(_test) if expectedFailure else _test @@ -238,156 +262,157 @@ class BPDTest(unittest.TestCase, TestHelper): self.assertEqual(alt_client.readline(), b'OK MPD 0.13.0\n') test_implements_query = implements({ - b'clearerror', b'currentsong', b'idle', b'status', b'stats', + 'clearerror', 'currentsong', 'idle', 'status', 'stats', }, expectedFailure=True) test_implements_playback = implements({ - b'consume', b'crossfade', b'mixrampdb', b'mixrampdelay', b'random', - b'repeat', b'setvol', b'single', b'replay_gain_mode', - b'replay_gain_status', b'volume', + 'consume', 'crossfade', 'mixrampd', 'mixrampdelay', 'random', + 'repeat', 'setvol', 'single', 'replay_gain_mode', + 'replay_gain_status', 'volume', }, expectedFailure=True) test_implements_control = implements({ - b'next', b'pause', b'play', b'playid', b'previous', b'seek', - b'seekid', b'seekcur', b'stop', + 'next', 'pause', 'play', 'playid', 'previous', 'seek', + 'seekid', 'seekcur', 'stop', }, expectedFailure=True) def test_cmd_play(self): responses = self.client.send_commands( - (b'add', b'Artist Name/Album Title/01 Track One Title.mp3'), - (b'status',), - (b'play',), - (b'status',)) - self.assertIn(b'state: stop', responses[1].body.split(b'\n')) + ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), + ('status',), + ('play',), + ('status',)) + self.assertEqual('stop', responses[1].data['state']) self.assertTrue(responses[2].ok) - self.assertIn(b'state: play', responses[3].body.split(b'\n')) + self.assertEqual('play', responses[3].data['state']) test_implements_queue = implements({ - b'add', b'addid', b'clear', b'delete', b'deleteid', b'move', - b'moveid', b'playlist', b'playlistfind', b'playlistid', - b'playlistinfo', b'playlistsearch', b'plchanges', - b'plchangesposid', b'prio', b'prioid', b'rangeid', b'shuffle', - b'swap', b'swapid', b'addtagid', b'cleartagid', + 'add', 'addid', 'clear', 'delete', 'deleteid', 'move', + 'moveid', 'playlist', 'playlistfind', 'playlistid', + 'playlistinfo', 'playlistsearch', 'plchanges', + 'plchangesposid', 'prio', 'prioid', 'rangeid', 'shuffle', + 'swap', 'swapid', 'addtagid', 'cleartagid', }, expectedFailure=True) def test_cmd_add(self): response = self.client.send_command( - b'add', - b'Artist Name/Album Title/01 Track One Title.mp3') + 'add', + 'Artist Name/Album Title/01 Track One Title.mp3') self.assertTrue(response.ok) def test_cmd_playlistinfo(self): responses = self.client.send_commands( - (b'add', b'Artist Name/Album Title/01 Track One Title.mp3'), - (b'playlistinfo',), - (b'playlistinfo', b'0')) + ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), + ('playlistinfo',), + ('playlistinfo', '0')) self.assertTrue(responses[1].ok) self.assertTrue(responses[2].ok) - self.assertEqual(responses[1].body, responses[2].body) + self.assertEqual(responses[1].data, responses[2].data) - response = self.client.send_command(b'playlistinfo', b'1') + response = self.client.send_command('playlistinfo', '1') self.assertTrue(response.err) self.assertEqual( - b'ACK [2@0] {playlistinfo} argument out of range', + 'ACK [2@0] {playlistinfo} argument out of range', response.status) test_implements_playlists = implements({ - b'listplaylist', b'listplaylistinfo', b'listplaylists', b'load', - b'playlistadd', b'playlistclear', b'playlistdelete', - b'playlistmove', b'rename', b'rm', b'save', + 'listplaylist', 'listplaylistinfo', 'listplaylists', 'load', + 'playlistadd', 'playlistclear', 'playlistdelete', + 'playlistmove', 'rename', 'rm', 'save', }, expectedFailure=True) test_implements_database = implements({ - b'albumart', b'count', b'find', b'findadd', b'list', b'listall', - b'listallinfo', b'listfiles', b'lsinfo', b'readcomments', - b'search', b'searchadd', b'searchaddpl', b'update', b'rescan', + 'albumart', 'count', 'find', 'findadd', 'list', 'listall', + 'listallinfo', 'listfiles', 'lsinfo', 'readcomments', + 'search', 'searchadd', 'searchaddpl', 'update', 'rescan', }, expectedFailure=True) def test_cmd_search(self): - response = self.client.send_command(b'search', b'track', b'1') + response = self.client.send_command('search', 'track', '1') self.assertEqual( - b'file: Artist Name/Album Title/01 Track One Title.mp3', - response.body.split(b'\n')[0]) + 'Artist Name/Album Title/01 Track One Title.mp3', + response.data['file']) def test_cmd_list_simple(self): - response = self.client.send_command(b'list', b'album') - self.assertEqual(b'Album: Album Title', response.body) + response = self.client.send_command('list', 'album') + self.assertEqual('Album Title', response.data['Album']) - response = self.client.send_command(b'list', b'track') - self.assertEqual(b'Track: 1\nTrack: 2', response.body) + response = self.client.send_command('list', 'track') + self.assertEqual(['1', '2'], response.data['Track']) def test_cmd_count(self): - response = self.client.send_command(b'count', b'track', b'1') - self.assertEqual(b'songs: 1\nplaytime: 0', response.body) + response = self.client.send_command('count', 'track', '1') + self.assertEqual('1', response.data['songs']) + self.assertEqual('0', response.data['playtime']) test_implements_mounts = implements({ - b'mount', b'unmount', b'listmounts', b'listneighbors', + 'mount', 'unmount', 'listmounts', 'listneighbors', }, expectedFailure=True) test_implements_stickers = implements({ - b'sticker', + 'sticker', }, expectedFailure=True) test_implements_connection = implements({ - b'close', b'kill', b'password', b'ping', b'tagtypes', + 'close', 'kill', 'password', 'ping', 'tagtypes', }) def test_cmd_password(self): self.client = self.make_server_client(password='abc123') - response = self.client.send_command(b'status') + response = self.client.send_command('status') self.assertTrue(response.err) self.assertEqual(response.status, - b'ACK [4@0] {} insufficient privileges') + 'ACK [4@0] {} insufficient privileges') - response = self.client.send_command(b'password', b'wrong') + response = self.client.send_command('password', 'wrong') self.assertTrue(response.err) self.assertEqual(response.status, - b'ACK [3@0] {password} incorrect password') + 'ACK [3@0] {password} incorrect password') - response = self.client.send_command(b'password', b'abc123') + response = self.client.send_command('password', 'abc123') self.assertTrue(response.ok) - response = self.client.send_command(b'status') + response = self.client.send_command('status') self.assertTrue(response.ok) def test_cmd_ping(self): - response = self.client.send_command(b'ping') + response = self.client.send_command('ping') self.assertTrue(response.ok) @unittest.expectedFailure def test_cmd_tagtypes(self): - response = self.client.send_command(b'tagtypes') - types = {line[9:].lower() for line in response.body.split(b'\n')} + response = self.client.send_command('tagtypes') + types = {tag.lower() for tag in response.data['tag']} self.assertEqual({ - b'artist', b'artistsort', b'album', b'albumsort', b'albumartist', - b'albumartistsort', b'title', b'track', b'name', b'genre', b'date', - b'composer', b'performer', b'comment', b'disc', b'label', - b'musicbrainz_artistid', b'musicbrainz_albumid', - b'musicbrainz_albumartistid', b'musicbrainz_trackid', - b'musicbrainz_releasetrackid', b'musicbrainz_workid', + 'artist', 'artistsort', 'album', 'albumsort', 'albumartist', + 'albumartistsort', 'title', 'track', 'name', 'genre', 'date', + 'composer', 'performer', 'comment', 'disc', 'label', + 'musicbrainz_artistid', 'musicbrainz_albumid', + 'musicbrainz_albumartistid', 'musicbrainz_trackid', + 'musicbrainz_releasetrackid', 'musicbrainz_workid', }, types) @unittest.expectedFailure def test_tagtypes_mask(self): - response = self.client.send_command(b'tagtypes', b'clear') + response = self.client.send_command('tagtypes', 'clear') self.assertTrue(response.ok) test_implements_partitions = implements({ - b'partition', b'listpartitions', b'newpartition', + 'partition', 'listpartitions', 'newpartition', }, expectedFailure=True) test_implements_devices = implements({ - b'disableoutput', b'enableoutput', b'toggleoutput', b'outputs', + 'disableoutput', 'enableoutput', 'toggleoutput', 'outputs', }, expectedFailure=True) test_implements_reflection = implements({ - b'config', b'commands', b'notcommands', b'urlhandlers', - b'decoders', + 'config', 'commands', 'notcommands', 'urlhandlers', + 'decoders', }, expectedFailure=True) test_implements_peers = implements({ - b'subscribe', b'unsubscribe', b'channels', b'readmessages', - b'sendmessage', + 'subscribe', 'unsubscribe', 'channels', 'readmessages', + 'sendmessage', }, expectedFailure=True) From 2286c0ce7b741894f4b958152d9b5df75e1d0294 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 28 Mar 2019 18:41:29 +1100 Subject: [PATCH 098/339] BPD tests: run bpd in a subprocess --- test/test_player.py | 204 ++++++++++++++++++++++++++------------------ 1 file changed, 122 insertions(+), 82 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index dfe583fca..7788d0fd6 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -18,34 +18,21 @@ from __future__ import division, absolute_import, print_function import unittest +import os import sys -import imp -import multiprocessing as mp +import subprocess import socket import time +import yaml +import tempfile +from contextlib import contextmanager from test import _common from test.helper import TestHelper +from beets.util import confit from beetsplug import bpd -# Intercept and mock the GstPlayer player: -gstplayer = imp.new_module('beetsplug.bpg.gstplayer') -gstplayer.GstPlayer = type('GstPlayer', (), { - '__init__': lambda self, callback: None, - 'playing': False, - 'volume': 0.0, - 'run': lambda self: None, - 'time': lambda self: (0, 0), - 'play': lambda self: None, - 'pause': lambda self: None, - 'play_file': lambda self, path: None, - 'seek': lambda self, pos: None, - 'stop': lambda self: None, - }) -sys.modules['beetsplug.bpd.gstplayer'] = gstplayer -bpd.gstplayer = gstplayer - class CommandParseTest(unittest.TestCase): def test_no_args(self): @@ -97,8 +84,6 @@ class MPCResponse(object): self.ok = (self.status.startswith('OK') or self.status.startswith('list_OK')) self.err = self.status.startswith('ACK') - if not self.ok: - print(self.status) def _parse_body(self, body): """ Messages are generally in the format "header: content". @@ -134,9 +119,6 @@ class MPCClient(object): if not hello.ok: raise RuntimeError('Bad hello: {}'.format(hello.status)) - def __del__(self): - self.sock.close() - def get_response(self): """ Wait for a full server response and wrap it in a helper class. If the request was a batch request then this will return a list of @@ -213,7 +195,8 @@ class MPCClient(object): def implements(commands, expectedFailure=False): # noqa: N803 def _test(self): - response = self.client.send_command('commands') + with self.run_bpd() as client: + response = client.send_command('commands') implemented = response.data['command'] self.assertEqual(commands.intersection(implemented), commands) return unittest.expectedFailure(_test) if expectedFailure else _test @@ -222,7 +205,7 @@ def implements(commands, expectedFailure=False): # noqa: N803 @_common.slow_test() class BPDTest(unittest.TestCase, TestHelper): def setUp(self): - self.setup_beets() + self.setup_beets(disk=True) self.load_plugins('bpd') self.item1 = self.add_item(title='Track One Title', album='Album Title', artist='Artist Name', @@ -232,34 +215,83 @@ class BPDTest(unittest.TestCase, TestHelper): track=2) self.lib.add_album([self.item1, self.item2]) - self.server_proc = None - self.client = self.make_server_client() + with open(os.path.join(self.temp_dir, b'bpd_mock.py'), 'wb') as f: + f.write(b'from unittest import mock\n') + f.write(b'import sys, imp\n') + f.write(b'gstplayer = imp.new_module("beetsplug.bpg.gstplayer")\n') + f.write(b'gstplayer._GstPlayer = mock.MagicMock(spec_set=["time",') + f.write(b'"volume", "playing", "run", "play_file", "pause", "stop') + f.write(b'", "seek"])\n') + f.write(b'gstplayer._GstPlayer.time.return_value = (0, 0)\n') + f.write(b'gstplayer._GstPlayer.volume = 0.0\n') + f.write(b'gstplayer._GstPlayer.playing = False\n') + f.write(b'gstplayer.GstPlayer = lambda _: gstplayer._GstPlayer\n') + f.write(b'sys.modules["beetsplug.bpd.gstplayer"] = gstplayer\n') + f.write(b'import beetsplug, beetsplug.bpd\n') + f.write(b'beetsplug.bpd.gstplayer = gstplayer\n') + f.write(b'sys.modules["beetsplug.bpd_mock"] = beetsplug.bpd\n') + f.write(b'beetsplug.bpd_mock = beetsplug.bpd\n') def tearDown(self): - self.server_proc.terminate() self.teardown_beets() self.unload_plugins() - def make_server(self, host, port, password=None): - bpd_server = bpd.Server(self.lib, host, port, password) - self.server = bpd_server - if self.server_proc: - self.server_proc.terminate() - self.server_proc = mp.Process(target=bpd_server.run) - self.server_proc.start() + @contextmanager + def run_bpd(self, host='localhost', port=9876, password=None, + do_hello=True): + """ Runs BPD in another process, configured with the same library + database as we created in the setUp method. Exposes a client that is + connected to the server, and kills the server at the end. + """ + # Create a config file: + config = { + 'pluginpath': [self.temp_dir.decode('utf-8')], + 'plugins': 'bpd_mock', + 'bpd': {'host': host, 'port': port}, + } + if password: + config['bpd']['password'] = password + config_file = tempfile.NamedTemporaryFile( + mode='wb', dir=self.temp_dir, suffix=b'.yaml', delete=False) + config_file.write( + yaml.dump(config, Dumper=confit.Dumper, encoding='utf-8')) + config_file.close() - def make_client(self, host='localhost', port=9876, do_hello=True): - return MPCClient(host, port, do_hello) + # Launch BPD in a new process: + env = dict(os.environ.items()) + env['PYTHONPATH'] = ':'.join( + [self.temp_dir.decode('utf-8')] + sys.path[1:]) + beet = os.path.join(os.path.dirname(__file__), '..', 'beet') + server = subprocess.Popen([ + beet, + '--library', self.config['library'].as_filename(), + '--directory', self.libdir, + '--config', config_file.name, + '--verbose', '--verbose', + 'bpd', '--debug', + ], env=env) - def make_server_client(self, host='localhost', port=9876, password=None): - self.make_server(host, port, password) - time.sleep(0.1) # wait for the server to start - client = self.make_client(host, port) - return client + # Wait until the socket is bound: + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + for _ in range(20): + if self.sock.connect_ex((host, port)) == 0: + self.sock.close() + break + else: + time.sleep(0.01) + else: + raise RuntimeError('Timed out waiting for the BPD server') + + try: + yield MPCClient(host, port, do_hello) + finally: + server.terminate() + server.wait(timeout=0.1) + server.kill() def test_server_hello(self): - alt_client = self.make_client(do_hello=False) - self.assertEqual(alt_client.readline(), b'OK MPD 0.13.0\n') + with self.run_bpd(do_hello=False) as client: + self.assertEqual(client.readline(), b'OK MPD 0.13.0\n') test_implements_query = implements({ 'clearerror', 'currentsong', 'idle', 'status', 'stats', @@ -277,11 +309,12 @@ class BPDTest(unittest.TestCase, TestHelper): }, expectedFailure=True) def test_cmd_play(self): - responses = self.client.send_commands( - ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), - ('status',), - ('play',), - ('status',)) + with self.run_bpd() as client: + responses = client.send_commands( + ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), + ('status',), + ('play',), + ('status',)) self.assertEqual('stop', responses[1].data['state']) self.assertTrue(responses[2].ok) self.assertEqual('play', responses[3].data['state']) @@ -295,21 +328,23 @@ class BPDTest(unittest.TestCase, TestHelper): }, expectedFailure=True) def test_cmd_add(self): - response = self.client.send_command( - 'add', - 'Artist Name/Album Title/01 Track One Title.mp3') + with self.run_bpd() as client: + response = client.send_command( + 'add', + 'Artist Name/Album Title/01 Track One Title.mp3') self.assertTrue(response.ok) def test_cmd_playlistinfo(self): - responses = self.client.send_commands( - ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), - ('playlistinfo',), - ('playlistinfo', '0')) + with self.run_bpd() as client: + responses = client.send_commands( + ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), + ('playlistinfo',), + ('playlistinfo', '0')) + response = client.send_command('playlistinfo', '200') + self.assertTrue(responses[1].ok) self.assertTrue(responses[2].ok) - self.assertEqual(responses[1].data, responses[2].data) - response = self.client.send_command('playlistinfo', '1') self.assertTrue(response.err) self.assertEqual( 'ACK [2@0] {playlistinfo} argument out of range', @@ -328,20 +363,22 @@ class BPDTest(unittest.TestCase, TestHelper): }, expectedFailure=True) def test_cmd_search(self): - response = self.client.send_command('search', 'track', '1') + with self.run_bpd() as client: + response = client.send_command('search', 'track', '1') self.assertEqual( 'Artist Name/Album Title/01 Track One Title.mp3', response.data['file']) def test_cmd_list_simple(self): - response = self.client.send_command('list', 'album') - self.assertEqual('Album Title', response.data['Album']) - - response = self.client.send_command('list', 'track') - self.assertEqual(['1', '2'], response.data['Track']) + with self.run_bpd() as client: + response1 = client.send_command('list', 'album') + response2 = client.send_command('list', 'track') + self.assertEqual('Album Title', response1.data['Album']) + self.assertEqual(['1', '2'], response2.data['Track']) def test_cmd_count(self): - response = self.client.send_command('count', 'track', '1') + with self.run_bpd() as client: + response = client.send_command('count', 'track', '1') self.assertEqual('1', response.data['songs']) self.assertEqual('0', response.data['playtime']) @@ -358,30 +395,32 @@ class BPDTest(unittest.TestCase, TestHelper): }) def test_cmd_password(self): - self.client = self.make_server_client(password='abc123') + with self.run_bpd(password='abc123') as client: - response = self.client.send_command('status') - self.assertTrue(response.err) - self.assertEqual(response.status, - 'ACK [4@0] {} insufficient privileges') + response = client.send_command('status') + self.assertTrue(response.err) + self.assertEqual(response.status, + 'ACK [4@0] {} insufficient privileges') - response = self.client.send_command('password', 'wrong') - self.assertTrue(response.err) - self.assertEqual(response.status, - 'ACK [3@0] {password} incorrect password') + response = client.send_command('password', 'wrong') + self.assertTrue(response.err) + self.assertEqual(response.status, + 'ACK [3@0] {password} incorrect password') - response = self.client.send_command('password', 'abc123') - self.assertTrue(response.ok) - response = self.client.send_command('status') - self.assertTrue(response.ok) + response = client.send_command('password', 'abc123') + self.assertTrue(response.ok) + response = client.send_command('status') + self.assertTrue(response.ok) def test_cmd_ping(self): - response = self.client.send_command('ping') + with self.run_bpd() as client: + response = client.send_command('ping') self.assertTrue(response.ok) @unittest.expectedFailure def test_cmd_tagtypes(self): - response = self.client.send_command('tagtypes') + with self.run_bpd() as client: + response = client.send_command('tagtypes') types = {tag.lower() for tag in response.data['tag']} self.assertEqual({ 'artist', 'artistsort', 'album', 'albumsort', 'albumartist', @@ -394,7 +433,8 @@ class BPDTest(unittest.TestCase, TestHelper): @unittest.expectedFailure def test_tagtypes_mask(self): - response = self.client.send_command('tagtypes', 'clear') + with self.run_bpd() as client: + response = client.send_command('tagtypes', 'clear') self.assertTrue(response.ok) test_implements_partitions = implements({ From c42c0c06bc63da00cff18840501471b52b3cbf6f Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 28 Mar 2019 18:51:11 +1100 Subject: [PATCH 099/339] BPD tests: fix CI failures --- test/test_player.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 7788d0fd6..fa9385722 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -216,7 +216,7 @@ class BPDTest(unittest.TestCase, TestHelper): self.lib.add_album([self.item1, self.item2]) with open(os.path.join(self.temp_dir, b'bpd_mock.py'), 'wb') as f: - f.write(b'from unittest import mock\n') + f.write(b'import mock\n') f.write(b'import sys, imp\n') f.write(b'gstplayer = imp.new_module("beetsplug.bpg.gstplayer")\n') f.write(b'gstplayer._GstPlayer = mock.MagicMock(spec_set=["time",') @@ -260,7 +260,7 @@ class BPDTest(unittest.TestCase, TestHelper): # Launch BPD in a new process: env = dict(os.environ.items()) env['PYTHONPATH'] = ':'.join( - [self.temp_dir.decode('utf-8')] + sys.path[1:]) + [self.temp_dir.decode('utf-8')] + sys.path[1:]).encode('utf-8') beet = os.path.join(os.path.dirname(__file__), '..', 'beet') server = subprocess.Popen([ beet, From 35bf041ad0ead098d1984e83888c6715230509ec Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 29 Mar 2019 11:22:58 +1100 Subject: [PATCH 100/339] BPD tests: fork and launch beets --- test/test_player.py | 69 +++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index fa9385722..5a375f6b7 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -13,25 +13,36 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Tests for BPD and music playing. +"""Tests for BPD's implementation of the MPD protocol. """ from __future__ import division, absolute_import, print_function import unittest -import os import sys -import subprocess +import multiprocessing as mp import socket import time import yaml import tempfile from contextlib import contextmanager -from test import _common +from beets.util import confit, py3_path +from beetsplug import bpd + from test.helper import TestHelper -from beets.util import confit -from beetsplug import bpd +# Mock GstPlayer so that the forked process doesn't attempt to import gi: +import mock +import imp +gstplayer = imp.new_module("beetsplug.bpg.gstplayer") +gstplayer._GstPlayer = mock.MagicMock(spec_set=[ + "time", "volume", "playing", "run", "play_file", "pause", "stop", "seek"]) +gstplayer._GstPlayer.time.return_value = (0, 0) +gstplayer._GstPlayer.volume = 0.0 +gstplayer._GstPlayer.playing = False +gstplayer.GstPlayer = lambda _: gstplayer._GstPlayer +sys.modules["beetsplug.bpd.gstplayer"] = gstplayer +bpd.gstplayer = gstplayer class CommandParseTest(unittest.TestCase): @@ -193,6 +204,11 @@ class MPCClient(object): return line +def start_beets(*args): + import beets.ui + beets.ui.main(list(args)) + + def implements(commands, expectedFailure=False): # noqa: N803 def _test(self): with self.run_bpd() as client: @@ -202,7 +218,6 @@ def implements(commands, expectedFailure=False): # noqa: N803 return unittest.expectedFailure(_test) if expectedFailure else _test -@_common.slow_test() class BPDTest(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets(disk=True) @@ -215,23 +230,6 @@ class BPDTest(unittest.TestCase, TestHelper): track=2) self.lib.add_album([self.item1, self.item2]) - with open(os.path.join(self.temp_dir, b'bpd_mock.py'), 'wb') as f: - f.write(b'import mock\n') - f.write(b'import sys, imp\n') - f.write(b'gstplayer = imp.new_module("beetsplug.bpg.gstplayer")\n') - f.write(b'gstplayer._GstPlayer = mock.MagicMock(spec_set=["time",') - f.write(b'"volume", "playing", "run", "play_file", "pause", "stop') - f.write(b'", "seek"])\n') - f.write(b'gstplayer._GstPlayer.time.return_value = (0, 0)\n') - f.write(b'gstplayer._GstPlayer.volume = 0.0\n') - f.write(b'gstplayer._GstPlayer.playing = False\n') - f.write(b'gstplayer.GstPlayer = lambda _: gstplayer._GstPlayer\n') - f.write(b'sys.modules["beetsplug.bpd.gstplayer"] = gstplayer\n') - f.write(b'import beetsplug, beetsplug.bpd\n') - f.write(b'beetsplug.bpd.gstplayer = gstplayer\n') - f.write(b'sys.modules["beetsplug.bpd_mock"] = beetsplug.bpd\n') - f.write(b'beetsplug.bpd_mock = beetsplug.bpd\n') - def tearDown(self): self.teardown_beets() self.unload_plugins() @@ -246,7 +244,7 @@ class BPDTest(unittest.TestCase, TestHelper): # Create a config file: config = { 'pluginpath': [self.temp_dir.decode('utf-8')], - 'plugins': 'bpd_mock', + 'plugins': 'bpd', 'bpd': {'host': host, 'port': port}, } if password: @@ -257,19 +255,16 @@ class BPDTest(unittest.TestCase, TestHelper): yaml.dump(config, Dumper=confit.Dumper, encoding='utf-8')) config_file.close() - # Launch BPD in a new process: - env = dict(os.environ.items()) - env['PYTHONPATH'] = ':'.join( - [self.temp_dir.decode('utf-8')] + sys.path[1:]).encode('utf-8') - beet = os.path.join(os.path.dirname(__file__), '..', 'beet') - server = subprocess.Popen([ - beet, + # Fork and launch BPD in the new process: + args = ( '--library', self.config['library'].as_filename(), - '--directory', self.libdir, - '--config', config_file.name, + '--directory', py3_path(self.libdir), + '--config', py3_path(config_file.name), '--verbose', '--verbose', - 'bpd', '--debug', - ], env=env) + 'bpd', '--debug' + ) + server = mp.Process(target=start_beets, args=args) + server.start() # Wait until the socket is bound: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -286,7 +281,7 @@ class BPDTest(unittest.TestCase, TestHelper): yield MPCClient(host, port, do_hello) finally: server.terminate() - server.wait(timeout=0.1) + server.join(timeout=0.1) server.kill() def test_server_hello(self): From 7c8bbd30113967c9b2dfd12f3d7564b218f04420 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 29 Mar 2019 11:28:03 +1100 Subject: [PATCH 101/339] BPD tests: avoid Py3.7-only mp.Process.kill --- test/test_player.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 5a375f6b7..cbbd63195 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -260,8 +260,7 @@ class BPDTest(unittest.TestCase, TestHelper): '--library', self.config['library'].as_filename(), '--directory', py3_path(self.libdir), '--config', py3_path(config_file.name), - '--verbose', '--verbose', - 'bpd', '--debug' + 'bpd' ) server = mp.Process(target=start_beets, args=args) server.start() @@ -281,8 +280,7 @@ class BPDTest(unittest.TestCase, TestHelper): yield MPCClient(host, port, do_hello) finally: server.terminate() - server.join(timeout=0.1) - server.kill() + server.join(timeout=0.2) def test_server_hello(self): with self.run_bpd(do_hello=False) as client: From 14d0cc3b85c6dfa790053e0609c1e2321d92b73e Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 29 Mar 2019 11:49:20 +1100 Subject: [PATCH 102/339] BPD tests: be more careful with paths --- test/test_player.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index cbbd63195..b8a5d992f 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -18,6 +18,9 @@ from __future__ import division, absolute_import, print_function import unittest +from test.helper import TestHelper + +import os import sys import multiprocessing as mp import socket @@ -29,7 +32,6 @@ from contextlib import contextmanager from beets.util import confit, py3_path from beetsplug import bpd -from test.helper import TestHelper # Mock GstPlayer so that the forked process doesn't attempt to import gi: import mock @@ -225,6 +227,8 @@ class BPDTest(unittest.TestCase, TestHelper): self.item1 = self.add_item(title='Track One Title', album='Album Title', artist='Artist Name', track=1) + self.item1_path = os.path.join( + 'Artist Name', 'Album Title', '01 Track One Title.mp3') self.item2 = self.add_item(title='Track Two Title', album='Album Title', artist='Artist Name', track=2) @@ -243,14 +247,15 @@ class BPDTest(unittest.TestCase, TestHelper): """ # Create a config file: config = { - 'pluginpath': [self.temp_dir.decode('utf-8')], + 'pluginpath': [os.fsdecode(self.temp_dir)], 'plugins': 'bpd', 'bpd': {'host': host, 'port': port}, } if password: config['bpd']['password'] = password config_file = tempfile.NamedTemporaryFile( - mode='wb', dir=self.temp_dir, suffix=b'.yaml', delete=False) + mode='wb', dir=os.fsdecode(self.temp_dir), suffix='.yaml', + delete=False) config_file.write( yaml.dump(config, Dumper=confit.Dumper, encoding='utf-8')) config_file.close() @@ -304,7 +309,7 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_play(self): with self.run_bpd() as client: responses = client.send_commands( - ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), + ('add', self.item1_path), ('status',), ('play',), ('status',)) @@ -322,15 +327,13 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_add(self): with self.run_bpd() as client: - response = client.send_command( - 'add', - 'Artist Name/Album Title/01 Track One Title.mp3') + response = client.send_command('add', self.item1_path) self.assertTrue(response.ok) def test_cmd_playlistinfo(self): with self.run_bpd() as client: responses = client.send_commands( - ('add', 'Artist Name/Album Title/01 Track One Title.mp3'), + ('add', self.item1_path), ('playlistinfo',), ('playlistinfo', '0')) response = client.send_command('playlistinfo', '200') @@ -358,9 +361,7 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_search(self): with self.run_bpd() as client: response = client.send_command('search', 'track', '1') - self.assertEqual( - 'Artist Name/Album Title/01 Track One Title.mp3', - response.data['file']) + self.assertEqual(self.item1_path, response.data['file']) def test_cmd_list_simple(self): with self.run_bpd() as client: From dfc5da70cd9f5152e3fe4a01819a88604aa51ac7 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 29 Mar 2019 11:55:09 +1100 Subject: [PATCH 103/339] BPD tests: avoid os.fsdecode new in Py3.2 --- test/test_player.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index b8a5d992f..bcc696d69 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -247,14 +247,14 @@ class BPDTest(unittest.TestCase, TestHelper): """ # Create a config file: config = { - 'pluginpath': [os.fsdecode(self.temp_dir)], + 'pluginpath': [py3_path(self.temp_dir)], 'plugins': 'bpd', 'bpd': {'host': host, 'port': port}, } if password: config['bpd']['password'] = password config_file = tempfile.NamedTemporaryFile( - mode='wb', dir=os.fsdecode(self.temp_dir), suffix='.yaml', + mode='wb', dir=py3_path(self.temp_dir), suffix='.yaml', delete=False) config_file.write( yaml.dump(config, Dumper=confit.Dumper, encoding='utf-8')) From 4c627cbacb30baa420c224d0fc4d97acfd5965df Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 29 Mar 2019 12:16:13 +1100 Subject: [PATCH 104/339] [skip travis] BPD tests: add items by path --- test/test_player.py | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index bcc696d69..5a47326ed 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -132,7 +132,7 @@ class MPCClient(object): if not hello.ok: raise RuntimeError('Bad hello: {}'.format(hello.status)) - def get_response(self): + def get_response(self, force_multi=None): """ Wait for a full server response and wrap it in a helper class. If the request was a batch request then this will return a list of `MPCResponse`s, one for each processed subcommand. @@ -144,7 +144,7 @@ class MPCClient(object): line = self.readline() response += line if line.startswith(b'OK') or line.startswith(b'ACK'): - if any(responses): + if force_multi or any(responses): if line.startswith(b'ACK'): responses.append(MPCResponse(response)) return responses @@ -185,7 +185,7 @@ class MPCClient(object): requests.append(b'command_list_end\n') request = b''.join(requests) self.sock.sendall(request) - return self.get_response() + return self.get_response(force_multi=True) def readline(self, terminator=b'\n', bufsize=1024): """ Reads a line of data from the socket. @@ -227,8 +227,6 @@ class BPDTest(unittest.TestCase, TestHelper): self.item1 = self.add_item(title='Track One Title', album='Album Title', artist='Artist Name', track=1) - self.item1_path = os.path.join( - 'Artist Name', 'Album Title', '01 Track One Title.mp3') self.item2 = self.add_item(title='Track Two Title', album='Album Title', artist='Artist Name', track=2) @@ -306,16 +304,24 @@ class BPDTest(unittest.TestCase, TestHelper): 'seekid', 'seekcur', 'stop', }, expectedFailure=True) + def bpd_add_item(self, client, item): + """ Add the given item to the BPD playlist + """ + name = py3_path(os.path.basename(item.path)) + path = '/'.join([item.artist, item.album, name]) + response = client.send_command('add', path) + self.assertTrue(response.ok) + def test_cmd_play(self): with self.run_bpd() as client: + self.bpd_add_item(client, self.item1) responses = client.send_commands( - ('add', self.item1_path), ('status',), ('play',), ('status',)) - self.assertEqual('stop', responses[1].data['state']) - self.assertTrue(responses[2].ok) - self.assertEqual('play', responses[3].data['state']) + self.assertEqual('stop', responses[0].data['state']) + self.assertTrue(responses[1].ok) + self.assertEqual('play', responses[2].data['state']) test_implements_queue = implements({ 'add', 'addid', 'clear', 'delete', 'deleteid', 'move', @@ -327,19 +333,18 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_add(self): with self.run_bpd() as client: - response = client.send_command('add', self.item1_path) - self.assertTrue(response.ok) + self.bpd_add_item(client, self.item1) def test_cmd_playlistinfo(self): with self.run_bpd() as client: + self.bpd_add_item(client, self.item1) responses = client.send_commands( - ('add', self.item1_path), ('playlistinfo',), ('playlistinfo', '0')) response = client.send_command('playlistinfo', '200') + self.assertTrue(responses[0].ok) self.assertTrue(responses[1].ok) - self.assertTrue(responses[2].ok) self.assertTrue(response.err) self.assertEqual( @@ -361,7 +366,7 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_search(self): with self.run_bpd() as client: response = client.send_command('search', 'track', '1') - self.assertEqual(self.item1_path, response.data['file']) + self.assertEqual(self.item1.title, response.data['Title']) def test_cmd_list_simple(self): with self.run_bpd() as client: @@ -370,6 +375,15 @@ class BPDTest(unittest.TestCase, TestHelper): self.assertEqual('Album Title', response1.data['Album']) self.assertEqual(['1', '2'], response2.data['Track']) + def test_cmd_lsinfo(self): + with self.run_bpd() as client: + response1 = client.send_command('lsinfo') + response2 = client.send_command( + 'lsinfo', response1.data['directory']) + response3 = client.send_command( + 'lsinfo', response2.data['directory']) + self.assertIn(self.item1.title, response3.data['Title']) + def test_cmd_count(self): with self.run_bpd() as client: response = client.send_command('count', 'track', '1') From 3eaf0f7a19d78efa0eee6d2a435f08b7e857e604 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 29 Mar 2019 12:55:38 +1100 Subject: [PATCH 105/339] BPD tests: use canonical capitalisation of tags --- test/test_player.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 5a47326ed..4fc1f7438 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -429,15 +429,14 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_tagtypes(self): with self.run_bpd() as client: response = client.send_command('tagtypes') - types = {tag.lower() for tag in response.data['tag']} self.assertEqual({ - 'artist', 'artistsort', 'album', 'albumsort', 'albumartist', - 'albumartistsort', 'title', 'track', 'name', 'genre', 'date', - 'composer', 'performer', 'comment', 'disc', 'label', - 'musicbrainz_artistid', 'musicbrainz_albumid', - 'musicbrainz_albumartistid', 'musicbrainz_trackid', - 'musicbrainz_releasetrackid', 'musicbrainz_workid', - }, types) + 'Artist', 'ArtistSort', 'Album', 'AlbumSort', 'AlbumArtist', + 'AlbumArtistSort', 'Title', 'Track', 'Name', 'Genre', 'Date', + 'Composer', 'Performer', 'Comment', 'Disc', 'Label', + 'OriginalDate', 'MUSICBRAINZ_ARTISTID', 'MUSICBRAINZ_ALBUMID', + 'MUSICBRAINZ_ALBUMARTISTID', 'MUSICBRAINZ_TRACKID', + 'MUSICBRAINZ_RELEASETRACKID', 'MUSICBRAINZ_WORKID', + }, set(response.data['tag'])) @unittest.expectedFailure def test_tagtypes_mask(self): From a6c976b88026712f309bb9828b00d7d7511daf39 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 12:39:11 +1100 Subject: [PATCH 106/339] BPD tests: small code reformatting --- test/test_player.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 4fc1f7438..84d12e880 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -224,12 +224,12 @@ class BPDTest(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets(disk=True) self.load_plugins('bpd') - self.item1 = self.add_item(title='Track One Title', - album='Album Title', artist='Artist Name', - track=1) - self.item2 = self.add_item(title='Track Two Title', - album='Album Title', artist='Artist Name', - track=2) + self.item1 = self.add_item( + title='Track One Title', track=1, + album='Album Title', artist='Artist Name') + self.item2 = self.add_item( + title='Track Two Title', track=2, + album='Album Title', artist='Artist Name') self.lib.add_album([self.item1, self.item2]) def tearDown(self): @@ -238,7 +238,7 @@ class BPDTest(unittest.TestCase, TestHelper): @contextmanager def run_bpd(self, host='localhost', port=9876, password=None, - do_hello=True): + do_hello=True): """ Runs BPD in another process, configured with the same library database as we created in the setUp method. Exposes a client that is connected to the server, and kills the server at the end. @@ -404,7 +404,6 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_password(self): with self.run_bpd(password='abc123') as client: - response = client.send_command('status') self.assertTrue(response.err) self.assertEqual(response.status, From 2a2d9b5c983627da446c526b7bd92656b2b78197 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 12:57:23 +1100 Subject: [PATCH 107/339] BPD tests: explicitly close client socket --- test/test_player.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 84d12e880..3c2409e19 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -123,9 +123,8 @@ class MPCResponse(object): class MPCClient(object): - def __init__(self, host, port, do_hello=True): - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect((host, port)) + def __init__(self, sock, do_hello=True): + self.sock = sock self.buf = b'' if do_hello: hello = self.get_response() @@ -268,20 +267,22 @@ class BPDTest(unittest.TestCase, TestHelper): server = mp.Process(target=start_beets, args=args) server.start() - # Wait until the socket is bound: - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Wait until the socket is connected: + sock = None for _ in range(20): - if self.sock.connect_ex((host, port)) == 0: - self.sock.close() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if sock.connect_ex((host, port)) == 0: break else: + sock.close() time.sleep(0.01) else: raise RuntimeError('Timed out waiting for the BPD server') try: - yield MPCClient(host, port, do_hello) + yield MPCClient(sock, do_hello) finally: + sock.close() server.terminate() server.join(timeout=0.2) From eb5c7513a13c8371b54a59c10ae20096bc9ff7d3 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 01:06:06 +1100 Subject: [PATCH 108/339] BPD tests: fix GstPlayer mock --- test/test_player.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 3c2409e19..9ea079b3c 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -36,12 +36,20 @@ from beetsplug import bpd # Mock GstPlayer so that the forked process doesn't attempt to import gi: import mock import imp -gstplayer = imp.new_module("beetsplug.bpg.gstplayer") -gstplayer._GstPlayer = mock.MagicMock(spec_set=[ - "time", "volume", "playing", "run", "play_file", "pause", "stop", "seek"]) -gstplayer._GstPlayer.time.return_value = (0, 0) -gstplayer._GstPlayer.volume = 0.0 -gstplayer._GstPlayer.playing = False +gstplayer = imp.new_module("beetsplug.bpd.gstplayer") +def _gstplayer_play(_): # noqa: 42 + bpd.gstplayer._GstPlayer.playing = True + return mock.DEFAULT +gstplayer._GstPlayer = mock.MagicMock( + spec_set=[ + "time", "volume", "playing", "run", "play_file", "pause", "stop", + "seek" + ], **{ + 'playing': False, + 'volume': 0.0, + 'time.return_value': (0, 0), + 'play_file.side_effect': _gstplayer_play, + }) gstplayer.GstPlayer = lambda _: gstplayer._GstPlayer sys.modules["beetsplug.bpd.gstplayer"] = gstplayer bpd.gstplayer = gstplayer From 2e35c27dce4789b72a72ff46c240365e64feac5e Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 01:07:08 +1100 Subject: [PATCH 109/339] BPD tests: improve helpers --- test/test_player.py | 117 +++++++++++++++++++++++++++++--------------- 1 file changed, 77 insertions(+), 40 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 9ea079b3c..5060b3322 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -101,10 +101,21 @@ class MPCResponse(object): def __init__(self, raw_response): body = b'\n'.join(raw_response.split(b'\n')[:-2]).decode('utf-8') self.data = self._parse_body(body) - self.status = raw_response.split(b'\n')[-2].decode('utf-8') - self.ok = (self.status.startswith('OK') or - self.status.startswith('list_OK')) - self.err = self.status.startswith('ACK') + status = raw_response.split(b'\n')[-2].decode('utf-8') + self.ok, self.err_data = self._parse_status(status) + + def _parse_status(self, status): + """ Parses the first response line, which contains the status. + """ + if status.startswith('OK') or status.startswith('list_OK'): + return True, None + elif status.startswith('ACK'): + code, rest = status[5:].split('@', 1) + pos, rest = rest.split(']', 1) + cmd, rest = rest[2:].split('}') + return False, (int(code), int(pos), cmd, rest[1:]) + else: + raise RuntimeError('Unexpected status: {!r}'.format(status)) def _parse_body(self, body): """ Messages are generally in the format "header: content". @@ -137,7 +148,7 @@ class MPCClient(object): if do_hello: hello = self.get_response() if not hello.ok: - raise RuntimeError('Bad hello: {}'.format(hello.status)) + raise RuntimeError('Bad hello') def get_response(self, force_multi=None): """ Wait for a full server response and wrap it in a helper class. @@ -154,6 +165,8 @@ class MPCClient(object): if force_multi or any(responses): if line.startswith(b'ACK'): responses.append(MPCResponse(response)) + n_remaining = force_multi - len(responses) + responses.extend([None] * n_remaining) return responses else: return MPCResponse(response) @@ -192,7 +205,7 @@ class MPCClient(object): requests.append(b'command_list_end\n') request = b''.join(requests) self.sock.sendall(request) - return self.get_response(force_multi=True) + return self.get_response(force_multi=len(commands)) def readline(self, terminator=b'\n', bufsize=1024): """ Reads a line of data from the socket. @@ -222,6 +235,7 @@ def implements(commands, expectedFailure=False): # noqa: N803 def _test(self): with self.run_bpd() as client: response = client.send_command('commands') + self._assert_ok(response) implemented = response.data['command'] self.assertEqual(commands.intersection(implemented), commands) return unittest.expectedFailure(_test) if expectedFailure else _test @@ -294,6 +308,25 @@ class BPDTest(unittest.TestCase, TestHelper): server.terminate() server.join(timeout=0.2) + def _assert_ok(self, *responses): + for response in responses: + self.assertTrue(response is not None) + self.assertTrue(response.ok, 'Response failed: {}'.format( + response.err_data)) + + def _assert_failed(self, response, code, pos=None): + """ Check that a command failed with a specific error code. If this + is a list of responses, first check all preceding commands were OK. + """ + if pos is not None: + previous_commands = response[0:pos] + self._assert_ok(*previous_commands) + response = response[pos] + self.assertEqual(pos, response.err_data[1]) + self.assertFalse(response.ok) + if code is not None: + self.assertEqual(code, response.err_data[0]) + def test_server_hello(self): with self.run_bpd(do_hello=False) as client: self.assertEqual(client.readline(), b'OK MPD 0.13.0\n') @@ -313,23 +346,29 @@ class BPDTest(unittest.TestCase, TestHelper): 'seekid', 'seekcur', 'stop', }, expectedFailure=True) - def bpd_add_item(self, client, item): + def _bpd_add(self, client, *items): """ Add the given item to the BPD playlist """ - name = py3_path(os.path.basename(item.path)) - path = '/'.join([item.artist, item.album, name]) - response = client.send_command('add', path) - self.assertTrue(response.ok) + paths = ['/'.join([ + item.artist, item.album, + py3_path(os.path.basename(item.path))]) for item in items] + responses = client.send_commands(*[('add', path) for path in paths]) + self._assert_ok(*responses) + + def test_unknown_cmd(self): + with self.run_bpd() as client: + response = client.send_command('notacommand') + self._assert_failed(response, bpd.ERROR_UNKNOWN) def test_cmd_play(self): with self.run_bpd() as client: - self.bpd_add_item(client, self.item1) + self._bpd_add(client, self.item1) responses = client.send_commands( ('status',), ('play',), ('status',)) + self._assert_ok(*responses) self.assertEqual('stop', responses[0].data['state']) - self.assertTrue(responses[1].ok) self.assertEqual('play', responses[2].data['state']) test_implements_queue = implements({ @@ -342,23 +381,17 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_add(self): with self.run_bpd() as client: - self.bpd_add_item(client, self.item1) + self._bpd_add(client, self.item1) def test_cmd_playlistinfo(self): with self.run_bpd() as client: - self.bpd_add_item(client, self.item1) + self._bpd_add(client, self.item1) responses = client.send_commands( ('playlistinfo',), - ('playlistinfo', '0')) - response = client.send_command('playlistinfo', '200') + ('playlistinfo', '0'), + ('playlistinfo', '200')) - self.assertTrue(responses[0].ok) - self.assertTrue(responses[1].ok) - - self.assertTrue(response.err) - self.assertEqual( - 'ACK [2@0] {playlistinfo} argument out of range', - response.status) + self._assert_failed(responses, bpd.ERROR_ARG, pos=2) test_implements_playlists = implements({ 'listplaylist', 'listplaylistinfo', 'listplaylists', 'load', @@ -375,27 +408,34 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_search(self): with self.run_bpd() as client: response = client.send_command('search', 'track', '1') + self._assert_ok(response) self.assertEqual(self.item1.title, response.data['Title']) def test_cmd_list_simple(self): with self.run_bpd() as client: - response1 = client.send_command('list', 'album') - response2 = client.send_command('list', 'track') - self.assertEqual('Album Title', response1.data['Album']) - self.assertEqual(['1', '2'], response2.data['Track']) + responses = client.send_commands( + ('list', 'album'), + ('list', 'track')) + self._assert_ok(*responses) + self.assertEqual('Album Title', responses[0].data['Album']) + self.assertEqual(['1', '2'], responses[1].data['Track']) def test_cmd_lsinfo(self): with self.run_bpd() as client: response1 = client.send_command('lsinfo') + self._assert_ok(response1) response2 = client.send_command( 'lsinfo', response1.data['directory']) + self._assert_ok(response2) response3 = client.send_command( 'lsinfo', response2.data['directory']) + self._assert_ok(response3) self.assertIn(self.item1.title, response3.data['Title']) def test_cmd_count(self): with self.run_bpd() as client: response = client.send_command('count', 'track', '1') + self._assert_ok(response) self.assertEqual('1', response.data['songs']) self.assertEqual('0', response.data['playtime']) @@ -414,29 +454,26 @@ class BPDTest(unittest.TestCase, TestHelper): def test_cmd_password(self): with self.run_bpd(password='abc123') as client: response = client.send_command('status') - self.assertTrue(response.err) - self.assertEqual(response.status, - 'ACK [4@0] {} insufficient privileges') + self._assert_failed(response, bpd.ERROR_PERMISSION) response = client.send_command('password', 'wrong') - self.assertTrue(response.err) - self.assertEqual(response.status, - 'ACK [3@0] {password} incorrect password') + self._assert_failed(response, bpd.ERROR_PASSWORD) - response = client.send_command('password', 'abc123') - self.assertTrue(response.ok) - response = client.send_command('status') - self.assertTrue(response.ok) + responses = client.send_commands( + ('password', 'abc123'), + ('status',)) + self._assert_ok(*responses) def test_cmd_ping(self): with self.run_bpd() as client: response = client.send_command('ping') - self.assertTrue(response.ok) + self._assert_ok(response) @unittest.expectedFailure def test_cmd_tagtypes(self): with self.run_bpd() as client: response = client.send_command('tagtypes') + self._assert_ok(response) self.assertEqual({ 'Artist', 'ArtistSort', 'Album', 'AlbumSort', 'AlbumArtist', 'AlbumArtistSort', 'Title', 'Track', 'Name', 'Genre', 'Date', @@ -450,7 +487,7 @@ class BPDTest(unittest.TestCase, TestHelper): def test_tagtypes_mask(self): with self.run_bpd() as client: response = client.send_command('tagtypes', 'clear') - self.assertTrue(response.ok) + self._assert_ok(response) test_implements_partitions = implements({ 'partition', 'listpartitions', 'newpartition', From 999cf14401e8cf70d2d2e351e63a320e6ce4dd76 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 10:50:42 +1100 Subject: [PATCH 110/339] BPD tests: change expectedFailure -> skip for nose --- test/test_player.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 5060b3322..1c2b0165d 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -469,7 +469,7 @@ class BPDTest(unittest.TestCase, TestHelper): response = client.send_command('ping') self._assert_ok(response) - @unittest.expectedFailure + @unittest.skip def test_cmd_tagtypes(self): with self.run_bpd() as client: response = client.send_command('tagtypes') @@ -483,7 +483,7 @@ class BPDTest(unittest.TestCase, TestHelper): 'MUSICBRAINZ_RELEASETRACKID', 'MUSICBRAINZ_WORKID', }, set(response.data['tag'])) - @unittest.expectedFailure + @unittest.skip def test_tagtypes_mask(self): with self.run_bpd() as client: response = client.send_command('tagtypes', 'clear') From 0b4293de6e5ad16b4ab667ea8dcf1223a6b8d028 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 18:19:09 +1100 Subject: [PATCH 111/339] the: log a debug only when text is changed Previously the `the` plugin would log a debug message when the text _didn't_ get changed by the plugin, whereas I think what was intended was the opposite. With this change the logged messages show the actual transformations made by the plugin. --- beetsplug/the.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/the.py b/beetsplug/the.py index 83d1089de..238aec32f 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -93,8 +93,8 @@ class ThePlugin(BeetsPlugin): for p in self.patterns: r = self.unthe(text, p) if r != text: + self._log.debug(u'\"{0}\" -> \"{1}\"', text, r) break - self._log.debug(u'\"{0}\" -> \"{1}\"', text, r) return r else: return u'' From 7557bb06d621bf90fb71bc1309d29ae21be5129b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 19:04:32 +1100 Subject: [PATCH 112/339] bpd: use log instead of stdout --- beetsplug/bpd/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 1049f0c76..9c3d2becd 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -729,7 +729,7 @@ class CommandList(list): e.index = i # Give the error the correct index. raise e - # Otherwise, possibly send the output delimeter if we're in a + # Otherwise, possibly send the output delimiter if we're in a # verbose ("OK") command list. if self.verbose: yield conn.send(RESP_CLIST_VERBOSE) @@ -807,9 +807,9 @@ class Server(BaseServer): """ # Path is ignored. Also, the real MPD does this asynchronously; # this is done inline. - print(u'Building directory tree...') + log.debug(u'Building directory tree...') self.tree = vfs.libtree(self.lib) - print(u'... done.') + log.debug(u'Finished building directory tree.') self.updated_time = time.time() # Path (directory tree) browsing. From ca60555ffa4af0382269ed2c3fb0c68a8e519b19 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 19:24:21 +1100 Subject: [PATCH 113/339] Fix deprecated call log.warn -> log.warning https://bugs.python.org/issue13235 --- beetsplug/discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index d7797e409..5a2bf57e0 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -284,7 +284,7 @@ class DiscogsPlugin(BeetsPlugin): # https://www.discogs.com/help/doc/submission-guidelines-general-rules if not all([result.data.get(k) for k in ['artists', 'title', 'id', 'tracklist']]): - self._log.warn(u"Release does not contain the required fields") + self._log.warning(u"Release does not contain the required fields") return None artist, artist_id = self.get_artist([a.data for a in result.artists]) From 3997141250ce310ea64127692295e14f797757b2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 19:36:33 +1100 Subject: [PATCH 114/339] Fix deprecated imports from collections for Py3 Since Python 3.3 the abstract base classes from `collections` are moved to the `collections.abc` module. From Python 3.8 they will be unavailable from their original location. https://docs.python.org/3/library/collections.abc.html --- beets/dbcore/db.py | 7 +++++-- beets/util/confit.py | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 24c20ef1a..b0c29e84c 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -23,7 +23,6 @@ from collections import defaultdict import threading import sqlite3 import contextlib -import collections import beets from beets.util.functemplate import Template @@ -31,6 +30,10 @@ from beets.util import py3_path from beets.dbcore import types from .query import MatchQuery, NullSort, TrueQuery import six +if six.PY2: + from collections import Mapping +else: + from collections.abc import Mapping class DBAccessError(Exception): @@ -42,7 +45,7 @@ class DBAccessError(Exception): """ -class FormattedMapping(collections.Mapping): +class FormattedMapping(Mapping): """A `dict`-like formatted view of a model. The accessor `mapping[key]` returns the formatted version of diff --git a/beets/util/confit.py b/beets/util/confit.py index 5f4d862ea..a5e522552 100644 --- a/beets/util/confit.py +++ b/beets/util/confit.py @@ -22,9 +22,13 @@ import os import pkgutil import sys import yaml -import collections import re +import six from collections import OrderedDict +if six.PY2: + from collections import Mapping, Sequence +else: + from collections.abc import Mapping, Sequence UNIX_DIR_VAR = 'XDG_CONFIG_HOME' UNIX_DIR_FALLBACK = '~/.config' @@ -1165,7 +1169,7 @@ class Choice(Template): view ) - if isinstance(self.choices, collections.Mapping): + if isinstance(self.choices, Mapping): return self.choices[value] else: return value @@ -1306,11 +1310,11 @@ class Pairs(StrSeq): return (super(Pairs, self)._convert_value(x, view), self.default_value) except ConfigTypeError: - if isinstance(x, collections.Mapping): + if isinstance(x, Mapping): if len(x) != 1: self.fail(u'must be a single-element mapping', view, True) k, v = iter_first(x.items()) - elif isinstance(x, collections.Sequence): + elif isinstance(x, Sequence): if len(x) != 2: self.fail(u'must be a two-element list', view, True) k, v = x @@ -1367,7 +1371,7 @@ class Filename(Template): return 'Filename({0})'.format(', '.join(args)) def resolve_relative_to(self, view, template): - if not isinstance(template, (collections.Mapping, MappingTemplate)): + if not isinstance(template, (Mapping, MappingTemplate)): # disallow config.get(Filename(relative_to='foo')) raise ConfigTemplateError( u'relative_to may only be used when getting multiple values.' @@ -1486,7 +1490,7 @@ def as_template(value): if isinstance(value, Template): # If it's already a Template, pass it through. return value - elif isinstance(value, collections.Mapping): + elif isinstance(value, Mapping): # Dictionaries work as templates. return MappingTemplate(value) elif value is int: @@ -1507,9 +1511,9 @@ def as_template(value): elif value is None: return Template() elif value is dict: - return TypeTemplate(collections.Mapping) + return TypeTemplate(Mapping) elif value is list: - return TypeTemplate(collections.Sequence) + return TypeTemplate(Sequence) elif isinstance(value, type): return TypeTemplate(value) else: From e4c03fd63f9c351b45234f978a9e5bbb524c8920 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 19:44:00 +1100 Subject: [PATCH 115/339] Fix deprecated placement of inline regex flags https://bugs.python.org/issue22493 --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 6ecdbd1d0..9e44eeef6 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -446,7 +446,7 @@ def _scrape_strip_cruft(html, plain_text_out=False): html = html.replace('\r', '\n') # Normalize EOL. html = re.sub(r' +', ' ', html) # Whitespaces collapse. html = BREAK_RE.sub('\n', html) #
eats up surrounding '\n'. - html = re.sub(r'<(script).*?(?s)', '', html) # Strip script tags. + html = re.sub(r'(?s)<(script).*?', '', html) # Strip script tags. if plain_text_out: # Strip remaining HTML tags html = COMMENT_RE.sub('', html) From a6305c36e111ffb4a3dcc17ee0f4baf7ab28be8d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 19:53:17 +1100 Subject: [PATCH 116/339] Fix deprecated plistlib function https://docs.python.org/3.7/library/plistlib.html#plistlib.readPlist --- beetsplug/metasync/itunes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/beetsplug/metasync/itunes.py b/beetsplug/metasync/itunes.py index 17ab1637f..d594fa591 100644 --- a/beetsplug/metasync/itunes.py +++ b/beetsplug/metasync/itunes.py @@ -24,6 +24,7 @@ import shutil import tempfile import plistlib +import six from six.moves.urllib.parse import urlparse, unquote from time import mktime @@ -84,7 +85,11 @@ class Itunes(MetaSource): self._log.debug( u'loading iTunes library from {0}'.format(library_path)) with create_temporary_copy(library_path) as library_copy: - raw_library = plistlib.readPlist(library_copy) + if six.PY2: + raw_library = plistlib.readPlist(library_copy) + else: + with open(library_copy, 'rb') as library_copy_f: + raw_library = plistlib.load(library_copy_f) except IOError as e: raise ConfigValueError(u'invalid iTunes library: ' + e.strerror) except Exception: From bed89df69a13ef37caa50da0a68d7fe8371063ed Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 20:22:44 +1100 Subject: [PATCH 117/339] Fix test_ui:test_version under pytest The test `CommonOptionsParserCliTest.test_version` was passing with nose but failing with pytest (see output below). The reason for the failure seemed to be that the `test` plugin was loaded when it wasn't expected to be loaded, changing the output of the `version` command. I'm not sur exactly why that was happening, but since that test already inherited from `TestHelper`, just invoking the plugin load/unload helper was enough to fix it. I also removed the line setting the `self.lib` variable since that's already done in the helper. --- self = def test_version(self): l = self.run_with_output(u'version') self.assertIn(u'Python version', l) > self.assertIn(u'no plugins loaded', l) E AssertionError: 'no plugins loaded' not found in 'beets version 1.4.8\nPython version 3.7.3rc1\nplugins: test\n' test/test_ui.py:1292: AssertionError --- test/test_ui.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_ui.py b/test/test_ui.py index bc9bb4829..8267c9be8 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1223,13 +1223,14 @@ class CommonOptionsParserCliTest(unittest.TestCase, TestHelper): """ def setUp(self): self.setup_beets() - self.lib = library.Library(':memory:') self.item = _common.item() self.item.path = b'xxx/yyy' self.lib.add(self.item) self.lib.add_album([self.item]) + self.load_plugins() def tearDown(self): + self.unload_plugins() self.teardown_beets() def test_base(self): From f9f2bddba0b97adc7abfdb9da59362a78c92ddf7 Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Sun, 24 Mar 2019 17:40:33 +0100 Subject: [PATCH 118/339] Add class wrapper for lazily converting attributes --- beets/dbcore/db.py | 133 +++++++++++++++++++++++++++++++++++---------- beets/importer.py | 8 +-- 2 files changed, 107 insertions(+), 34 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 112b8ee93..b8243e9df 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -88,6 +88,100 @@ class FormattedMapping(collections.Mapping): return value +class LazyConvertDict(object): + """Lazily convert types for attributes fetched from the database + """ + + def __init__(self, model_cls): + """Initialize the object empty + """ + self.data = {} + self.model_cls = model_cls + self._converted = {} + + def init(self, data): + """Set the base data that should be lazily converted + """ + self.data = data + + def _convert(self, key, value): + """Convert the attribute type according the the SQL type + """ + return self.model_cls._type(key).from_sql(value) + + def __setitem__(self, key, value): + """Set an attribute value, assume it's already converted + """ + self._converted[key] = value + + def __getitem__(self, key): + """Get an attribute value, converting the type on demand + if needed + """ + if key in self._converted: + return self._converted[key] + elif key in self.data: + value = self._convert(key, self.data[key]) + self._converted[key] = value + return value + + def __delitem__(self, key): + """Delete both converted and base data + """ + if key in self._converted: + del self._converted[key] + if key in self.data: + del self.data[key] + + def keys(self): + """Get a list of available field names for this object. + """ + return list(self._converted.keys()) + list(self.data.keys()) + + def copy(self): + """Create a copy of the object. + """ + new = self.__class__(self.model_cls) + new.data = self.data.copy() + new._converted = self._converted.copy() + return new + + # Act like a dictionary. + + def update(self, values): + """Assign all values in the given dict. + """ + for key, value in values.items(): + self[key] = value + + def items(self): + """Iterate over (key, value) pairs that this object contains. + Computed fields are not included. + """ + for key in self: + yield key, self[key] + + def get(self, key, default=None): + """Get the value for a given key or `default` if it does not + exist. + """ + if key in self: + return self[key] + else: + return default + + def __contains__(self, key): + """Determine whether `key` is an attribute on this object. + """ + return key in self.keys() + + def __iter__(self): + """Iterate over the available field names (excluding computed + fields). + """ + return iter(self.keys()) + + # Abstract base for model classes. class Model(object): @@ -177,10 +271,8 @@ class Model(object): """ self._db = db self._dirty = set() - self._raw_values_fixed = {} - self._raw_values_flex = {} - self._values_fixed = {} - self._values_flex = {} + self._values_fixed = LazyConvertDict(self) + self._values_flex = LazyConvertDict(self) # Initial contents. self.update(values) @@ -194,10 +286,10 @@ class Model(object): ordinary construction are bypassed. """ obj = cls(db) - for key, value in fixed_values.items(): - obj._raw_values_fixed[key] = value - for key, value in flex_values.items(): - obj._raw_values_flex[key] = value + + obj._values_fixed.init(fixed_values) + obj._values_flex.init(flex_values) + return obj def __repr__(self): @@ -234,9 +326,7 @@ class Model(object): """ new = self.__class__() new._db = self._db - new._raw_values_fixed = self._raw_values_fixed.copy() new._values_fixed = self._values_fixed.copy() - new._raw_values_flex = self._raw_values_flex.copy() new._values_flex = self._values_flex.copy() new._dirty = self._dirty.copy() return new @@ -262,16 +352,10 @@ class Model(object): elif key in self._fields: # Fixed. if key in self._values_fixed: return self._values_fixed[key] - elif key in self._raw_values_fixed: - self._values_fixed[key] = self._type(key).from_sql(self._raw_values_fixed[key]) - return self._values_fixed[key] else: return self._type(key).null elif key in self._values_flex: # Flexible. return self._values_flex[key] - elif key in self._raw_values_flex: # Flexible. - self._values_flex[key] = self._type(key).from_sql(self._raw_values_flex[key]) - return self._values_flex[key] else: raise KeyError(key) @@ -281,12 +365,8 @@ class Model(object): """ # Choose where to place the value. if key in self._fields: - if not key in self._values_fixed and key in self._raw_values_fixed: - self._values_fixed[key] = self._type(key).from_sql(self._raw_values_fixed[key]) source = self._values_fixed else: - if not key in self._values_flex and key in self._raw_values_flex: - self._values_flex[key] = self._type(key).from_sql(self._raw_values_flex[key]) source = self._values_flex # If the field has a type, filter the value. @@ -311,11 +391,6 @@ class Model(object): """ if key in self._values_flex: # Flexible. del self._values_flex[key] - if key in self._raw_values_flex: - del self._raw_values_flex[key] - self._dirty.add(key) # Mark for dropping on store. - elif key in self._raw_values_flex: # Flexible - del self._raw_values_flex[key] self._dirty.add(key) # Mark for dropping on store. elif key in self._fields: # Fixed setattr(self, key, self._type(key).null) @@ -329,7 +404,7 @@ class Model(object): `computed` parameter controls whether computed (plugin-provided) fields are included in the key list. """ - base_keys = list(self._fields) + list(self._values_flex.keys()) + list(self._raw_values_flex.keys()) + base_keys = list(self._fields) + list(self._values_flex.keys()) if computed: return base_keys + list(self._getters().keys()) else: @@ -458,10 +533,8 @@ class Model(object): self._check_db() stored_obj = self._db._get(type(self), self.id) assert stored_obj is not None, u"object {0} not in DB".format(self.id) - self._raw_values_fixed = {} - self._values_fixed = {} - self._raw_values_flex = {} - self._values_flex = {} + self._values_fixed = LazyConvertDict(self) + self._values_flex = LazyConvertDict(self) self.update(dict(stored_obj)) self.clear_dirty() diff --git a/beets/importer.py b/beets/importer.py index 50e4545c5..889f1297e 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -785,14 +785,14 @@ class ImportTask(BaseImportTask): replaced_album = self.replaced_albums.get(self.album.path) if replaced_album: self.album.added = replaced_album.added - self.album.update(replaced_album._raw_values_flex) + self.album.update(replaced_album._values_flex) self.album.artpath = replaced_album.artpath self.album.store() log.debug( u'Reimported album: added {0}, flexible ' u'attributes {1} from album {2} for {3}', self.album.added, - replaced_album._raw_values_flex.keys(), + replaced_album._values_flex.keys(), replaced_album.id, displayable_path(self.album.path) ) @@ -809,11 +809,11 @@ class ImportTask(BaseImportTask): dup_item.id, displayable_path(item.path) ) - item.update(dup_item._raw_values_flex) + item.update(dup_item._values_flex) log.debug( u'Reimported item flexible attributes {0} ' u'from item {1} for {2}', - dup_item._raw_values_flex.keys(), + dup_item._values_flex.keys(), dup_item.id, displayable_path(item.path) ) From 1916fe0ad51f620cfc5540c8bb5dabf9672e9921 Mon Sep 17 00:00:00 2001 From: Thomas McWork Date: Sun, 31 Mar 2019 13:06:27 +0200 Subject: [PATCH 119/339] Provide an example for the languages list It's hard to impossible to figure out the correct syntax for non python users. --- docs/reference/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index cc5b39c21..87f4165fd 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -571,7 +571,7 @@ default is ``apply``. languages ~~~~~~~~~ -A list of locale names to search for preferred aliases. For example, setting +A list of locale names to search for preferred aliases (e.g. ``[de,en]``. For example, setting this to "en" uses the transliterated artist name "Pyotr Ilyich Tchaikovsky" instead of the Cyrillic script for the composer's name when tagging from MusicBrainz. Defaults to an empty list, meaning that no language is preferred. From b4c4f3ca19c7fe6c238776b2f982d6502fee2b33 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 18:35:39 +1100 Subject: [PATCH 120/339] bpd: use plugin logger instead of global logger --- beetsplug/bpd/__init__.py | 47 ++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 9c3d2becd..a29690b03 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -29,7 +29,6 @@ import time import beets from beets.plugins import BeetsPlugin import beets.ui -from beets import logging from beets import vfs from beets.util import bluelet from beets.library import Item @@ -73,10 +72,6 @@ SAFE_COMMANDS = ( ITEM_KEYS_WRITABLE = set(MediaFile.fields()).intersection(Item._fields.keys()) -# Loggers. -log = logging.getLogger('beets.bpd') -global_log = logging.getLogger('beets') - # Gstreamer import error. class NoGstreamerError(Exception): @@ -166,12 +161,13 @@ class BaseServer(object): This is a generic superclass and doesn't support many commands. """ - def __init__(self, host, port, password): + def __init__(self, host, port, password, log): """Create a new server bound to address `host` and listening on port `port`. If `password` is given, it is required to do anything significant on the server. """ self.host, self.port, self.password = host, port, password + self._log = log # Default server values. self.random = False @@ -573,7 +569,9 @@ class Connection(object): if isinstance(lines, six.string_types): lines = [lines] out = NEWLINE.join(lines) + NEWLINE - log.debug('{}', out[:-1]) # Don't log trailing newline. + # Don't log trailing newline: + message = out[:-1].replace(u'\n', u'\n' + u' ' * 13) + self.server._log.debug('server: {}', message) if isinstance(out, six.text_type): out = out.encode('utf-8') return self.sock.sendall(out) @@ -594,6 +592,7 @@ class Connection(object): """Send a greeting to the client and begin processing commands as they arrive. """ + self.server._log.debug('New client connected') yield self.send(HELLO) clist = None # Initially, no command list is being constructed. @@ -605,7 +604,8 @@ class Connection(object): if not line: break line = line.decode('utf8') # MPD protocol uses UTF-8. - log.debug(u'{}', line) + message = line.replace(u'\n', u'\n' + u' ' * 13) + self.server._log.debug(u'client: {}', message) if clist is not None: # Command list already opened. @@ -699,7 +699,7 @@ class Command(object): except Exception as e: # An "unintentional" error. Hide it from the client. - log.error('{}', traceback.format_exc(e)) + conn.server._log.error('{}', traceback.format_exc(e)) raise BPDError(ERROR_SYSTEM, u'server error', self.name) @@ -743,7 +743,7 @@ class Server(BaseServer): to store its library. """ - def __init__(self, library, host, port, password): + def __init__(self, library, host, port, password, log): try: from beetsplug.bpd import gstplayer except ImportError as e: @@ -752,7 +752,7 @@ class Server(BaseServer): raise NoGstreamerError() else: raise - super(Server, self).__init__(host, port, password) + super(Server, self).__init__(host, port, password, log) self.lib = library self.player = gstplayer.GstPlayer(self.play_finished) self.cmd_update(None) @@ -807,9 +807,9 @@ class Server(BaseServer): """ # Path is ignored. Also, the real MPD does this asynchronously; # this is done inline. - log.debug(u'Building directory tree...') + self._log.debug(u'Building directory tree...') self.tree = vfs.libtree(self.lib) - log.debug(u'Finished building directory tree.') + self._log.debug(u'Finished building directory tree.') self.updated_time = time.time() # Path (directory tree) browsing. @@ -1156,29 +1156,21 @@ class BPDPlugin(BeetsPlugin): }) self.config['password'].redact = True - def start_bpd(self, lib, host, port, password, volume, debug): + def start_bpd(self, lib, host, port, password, volume): """Starts a BPD server.""" - if debug: # FIXME this should be managed by BeetsPlugin - self._log.setLevel(logging.DEBUG) - else: - self._log.setLevel(logging.WARNING) try: - server = Server(lib, host, port, password) + server = Server(lib, host, port, password, self._log) server.cmd_setvol(None, volume) server.run() except NoGstreamerError: - global_log.error(u'Gstreamer Python bindings not found.') - global_log.error(u'Install "gstreamer1.0" and "python-gi"' - u'or similar package to use BPD.') + self._log.error(u'Gstreamer Python bindings not found.') + self._log.error(u'Install "gstreamer1.0" and "python-gi"' + u'or similar package to use BPD.') def commands(self): cmd = beets.ui.Subcommand( 'bpd', help=u'run an MPD-compatible music player server' ) - cmd.parser.add_option( - '-d', '--debug', action='store_true', - help=u'dump all MPD traffic to stdout' - ) def func(lib, opts, args): host = self.config['host'].as_str() @@ -1188,8 +1180,7 @@ class BPDPlugin(BeetsPlugin): raise beets.ui.UserError(u'too many arguments') password = self.config['password'].as_str() volume = self.config['volume'].get(int) - debug = opts.debug or False - self.start_bpd(lib, host, int(port), password, volume, debug) + self.start_bpd(lib, host, int(port), password, volume) cmd.func = func return [cmd] From 8df213e9b62afb1c76db9295075887a0796b32ab Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 31 Mar 2019 18:21:26 +0100 Subject: [PATCH 121/339] Support Python 3.8 (fixes #3201) --- .travis.yml | 3 +++ beets/util/functemplate.py | 8 +++++++- setup.py | 1 + tox.ini | 9 +++++---- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 492b90575..6e4a59ac2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,9 @@ matrix: - python: 3.7 env: {TOX_ENV: py37-test} dist: xenial + - python: 3.8-dev + env: {TOX_ENV: py38-test} + dist: xenial # - python: pypy # - env: {TOX_ENV: pypy-test} - python: 3.4 diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 0e13db4a0..6a34a3bb3 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -141,7 +141,13 @@ def compile_func(arg_names, statements, name='_the_func', debug=False): decorator_list=[], ) - mod = ast.Module([func_def]) + # The ast.Module signature changed in 3.8 to accept a list of types to + # ignore. + if sys.version_info >= (3, 8): + mod = ast.Module([func_def], []) + else: + mod = ast.Module([func_def]) + ast.fix_missing_locations(mod) prog = compile(mod, '', 'exec') diff --git a/setup.py b/setup.py index ae8f76ff8..24f9f389c 100755 --- a/setup.py +++ b/setup.py @@ -154,6 +154,7 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', ], ) diff --git a/tox.ini b/tox.ini index eeacf2af5..e3250bd6b 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py27-flake8, docs +envlist = py27-test, py37-test, py38-test, py27-flake8, docs # The exhaustive list of environments is: # envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs @@ -40,17 +40,18 @@ passenv = deps = {test,cov}: {[_test]deps} py27: pathlib - py{27,34,35,36,37}-flake8: {[_flake8]deps} + py{27,34,35,36,37,38}-flake8: {[_flake8]deps} commands = py27-cov: python -m nose --with-coverage {posargs} py27-test: python -m nose {posargs} - py3{4,5,6,7}-cov: python -bb -m nose --with-coverage {posargs} - py3{4,5,6,7}-test: python -bb -m nose {posargs} + py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py3{4,5,6,7,8}-test: python -bb -m nose {posargs} py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} + py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} [testenv:docs] basepython = python2.7 From 3701d145b07b1a782573d4f6bf4fd899db0bb71b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 31 Mar 2019 21:39:55 -0400 Subject: [PATCH 122/339] Changelog for #3195 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1fcc691a5..c6229b494 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -160,6 +160,10 @@ Fixes: :bug:`2826` :bug:`3092` * :doc:`/plugins/beatport`: Avoid a crash when the server produces an error. :bug:`3184` +* :doc:`/plugins/the`: Log a message when something has changed, not when it + hasn't. + Thanks to :user:`arcresu`. + :bug:`3195` .. _python-itunes: https://github.com/ocelma/python-itunes From 0d31b339483c58e668a4b69d574f5116f2ed8d54 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 31 Mar 2019 21:42:16 -0400 Subject: [PATCH 123/339] Changelog for #3196 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c6229b494..4d26c37da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -164,6 +164,10 @@ Fixes: hasn't. Thanks to :user:`arcresu`. :bug:`3195` +* :doc:`/plugins/bpd`: The plugin now uses the main beets logging system. + The special-purpose ``--debug`` flag has been removed. + Thanks to :user:`arcresu`. + :bug:`3196` .. _python-itunes: https://github.com/ocelma/python-itunes From a1f965a3e52614a3e28891bf11da74a17ed551dd Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 31 Mar 2019 21:45:08 -0400 Subject: [PATCH 124/339] Changelog for #3197 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4d26c37da..8b2375c6a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -168,6 +168,9 @@ Fixes: The special-purpose ``--debug`` flag has been removed. Thanks to :user:`arcresu`. :bug:`3196` +* Fix several uses of deprecated standard-library features on Python 3.7. + Thanks to :user:`arcresu`. + :bug:`3197` .. _python-itunes: https://github.com/ocelma/python-itunes From 6c9c8819895f051b519f5dde0a1fae5099133bbf Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 31 Mar 2019 21:52:30 -0400 Subject: [PATCH 125/339] Changelog for #3089 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ea642e758..4f6cabaa7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -77,6 +77,10 @@ New features: :bug:`3081` * The `badfiles` plugin now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. +* Querying the library is now faster because we only convert fields that need + to be displayed. + Thanks to :user:`pprkut`. + :bug:`3089` Changes: From 422189ca3d7eec083d32d5f5a4c1bfd5618b98fb Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 31 Mar 2019 22:06:01 -0400 Subject: [PATCH 126/339] Changelog for #3202 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4f6cabaa7..c822af608 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -179,6 +179,8 @@ Fixes: library. Thanks to :user:`translit`. :bug:`3192` +* Fix compatibility with pre-release versions of Python 3.8. + :bug:`3201` :bug:`3202` .. _python-itunes: https://github.com/ocelma/python-itunes From de6718abdf9866c39b1b3ed5794a78800ea61b2e Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 12:02:15 +1100 Subject: [PATCH 127/339] bpd: separate tests by command category --- test/test_player.py | 83 ++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 1c2b0165d..1e917f078 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -241,7 +241,7 @@ def implements(commands, expectedFailure=False): # noqa: N803 return unittest.expectedFailure(_test) if expectedFailure else _test -class BPDTest(unittest.TestCase, TestHelper): +class BPDTestHelper(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets(disk=True) self.load_plugins('bpd') @@ -327,25 +327,6 @@ class BPDTest(unittest.TestCase, TestHelper): if code is not None: self.assertEqual(code, response.err_data[0]) - def test_server_hello(self): - with self.run_bpd(do_hello=False) as client: - self.assertEqual(client.readline(), b'OK MPD 0.13.0\n') - - test_implements_query = implements({ - 'clearerror', 'currentsong', 'idle', 'status', 'stats', - }, expectedFailure=True) - - test_implements_playback = implements({ - 'consume', 'crossfade', 'mixrampd', 'mixrampdelay', 'random', - 'repeat', 'setvol', 'single', 'replay_gain_mode', - 'replay_gain_status', 'volume', - }, expectedFailure=True) - - test_implements_control = implements({ - 'next', 'pause', 'play', 'playid', 'previous', 'seek', - 'seekid', 'seekcur', 'stop', - }, expectedFailure=True) - def _bpd_add(self, client, *items): """ Add the given item to the BPD playlist """ @@ -355,11 +336,38 @@ class BPDTest(unittest.TestCase, TestHelper): responses = client.send_commands(*[('add', path) for path in paths]) self._assert_ok(*responses) + +class BPDTest(BPDTestHelper): + def test_server_hello(self): + with self.run_bpd(do_hello=False) as client: + self.assertEqual(client.readline(), b'OK MPD 0.13.0\n') + def test_unknown_cmd(self): with self.run_bpd() as client: response = client.send_command('notacommand') self._assert_failed(response, bpd.ERROR_UNKNOWN) + +class BPDQueryTest(BPDTestHelper): + test_implements_query = implements({ + 'clearerror', 'currentsong', 'idle', 'status', 'stats', + }, expectedFailure=True) + + +class BPDPlaybackTest(BPDTestHelper): + test_implements_playback = implements({ + 'consume', 'crossfade', 'mixrampd', 'mixrampdelay', 'random', + 'repeat', 'setvol', 'single', 'replay_gain_mode', + 'replay_gain_status', 'volume', + }, expectedFailure=True) + + +class BPDControlTest(BPDTestHelper): + test_implements_control = implements({ + 'next', 'pause', 'playid', 'previous', 'seek', + 'seekid', 'seekcur', 'stop', + }, expectedFailure=True) + def test_cmd_play(self): with self.run_bpd() as client: self._bpd_add(client, self.item1) @@ -371,10 +379,12 @@ class BPDTest(unittest.TestCase, TestHelper): self.assertEqual('stop', responses[0].data['state']) self.assertEqual('play', responses[2].data['state']) + +class BPDQueueTest(BPDTestHelper): test_implements_queue = implements({ - 'add', 'addid', 'clear', 'delete', 'deleteid', 'move', + 'addid', 'clear', 'delete', 'deleteid', 'move', 'moveid', 'playlist', 'playlistfind', 'playlistid', - 'playlistinfo', 'playlistsearch', 'plchanges', + 'playlistsearch', 'plchanges', 'plchangesposid', 'prio', 'prioid', 'rangeid', 'shuffle', 'swap', 'swapid', 'addtagid', 'cleartagid', }, expectedFailure=True) @@ -390,19 +400,22 @@ class BPDTest(unittest.TestCase, TestHelper): ('playlistinfo',), ('playlistinfo', '0'), ('playlistinfo', '200')) - self._assert_failed(responses, bpd.ERROR_ARG, pos=2) + +class BPDPlaylistsTest(BPDTestHelper): test_implements_playlists = implements({ 'listplaylist', 'listplaylistinfo', 'listplaylists', 'load', 'playlistadd', 'playlistclear', 'playlistdelete', 'playlistmove', 'rename', 'rm', 'save', }, expectedFailure=True) + +class BPDDatabaseTest(BPDTestHelper): test_implements_database = implements({ - 'albumart', 'count', 'find', 'findadd', 'list', 'listall', - 'listallinfo', 'listfiles', 'lsinfo', 'readcomments', - 'search', 'searchadd', 'searchaddpl', 'update', 'rescan', + 'albumart', 'find', 'findadd', 'listall', + 'listallinfo', 'listfiles', 'readcomments', + 'searchadd', 'searchaddpl', 'update', 'rescan', }, expectedFailure=True) def test_cmd_search(self): @@ -411,7 +424,7 @@ class BPDTest(unittest.TestCase, TestHelper): self._assert_ok(response) self.assertEqual(self.item1.title, response.data['Title']) - def test_cmd_list_simple(self): + def test_cmd_list(self): with self.run_bpd() as client: responses = client.send_commands( ('list', 'album'), @@ -439,16 +452,22 @@ class BPDTest(unittest.TestCase, TestHelper): self.assertEqual('1', response.data['songs']) self.assertEqual('0', response.data['playtime']) + +class BPDMountsTest(BPDTestHelper): test_implements_mounts = implements({ 'mount', 'unmount', 'listmounts', 'listneighbors', }, expectedFailure=True) + +class BPDStickerTest(BPDTestHelper): test_implements_stickers = implements({ 'sticker', }, expectedFailure=True) + +class BPDConnectionTest(BPDTestHelper): test_implements_connection = implements({ - 'close', 'kill', 'password', 'ping', 'tagtypes', + 'close', 'kill', 'tagtypes', }) def test_cmd_password(self): @@ -489,19 +508,27 @@ class BPDTest(unittest.TestCase, TestHelper): response = client.send_command('tagtypes', 'clear') self._assert_ok(response) + +class BPDPartitionTest(BPDTestHelper): test_implements_partitions = implements({ 'partition', 'listpartitions', 'newpartition', }, expectedFailure=True) + +class BPDDeviceTest(BPDTestHelper): test_implements_devices = implements({ 'disableoutput', 'enableoutput', 'toggleoutput', 'outputs', }, expectedFailure=True) + +class BPDReflectionTest(BPDTestHelper): test_implements_reflection = implements({ 'config', 'commands', 'notcommands', 'urlhandlers', 'decoders', }, expectedFailure=True) + +class BPDPeersTest(BPDTestHelper): test_implements_peers = implements({ 'subscribe', 'unsubscribe', 'channels', 'readmessages', 'sendmessage', From d94a5393b2929cdfc0c759728ad185b8d40f68dd Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 19:03:38 +1100 Subject: [PATCH 128/339] bpd: fix crossfade command Although crossfade is not implemented in bpd, we can store the setting and repeat is back to clients. Also log a warning that the operation is not implemented. The real MPD doesn't show the crossfade in status if it's zero since that means no crossfade, so now we don't either. --- beetsplug/bpd/__init__.py | 6 +++++- test/test_player.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index a29690b03..0d16e22ca 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -307,9 +307,11 @@ class BaseServer(object): u'random: ' + six.text_type(int(self.random)), u'playlist: ' + six.text_type(self.playlist_version), u'playlistlength: ' + six.text_type(len(self.playlist)), - u'xfade: ' + six.text_type(self.crossfade), ) + if self.crossfade > 0: + yield u'xfade: ' + six.text_type(self.crossfade) + if self.current_index == -1: state = u'stop' elif self.paused: @@ -353,6 +355,8 @@ class BaseServer(object): crossfade = cast_arg(int, crossfade) if crossfade < 0: raise BPDError(ERROR_ARG, u'crossfade time must be nonnegative') + self._log.warning(u'crossfade is not implemented in bpd') + self.crossfade = crossfade def cmd_clear(self, conn): """Clear the playlist.""" diff --git a/test/test_player.py b/test/test_player.py index 1e917f078..d7f65ff3f 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -356,11 +356,24 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ - 'consume', 'crossfade', 'mixrampd', 'mixrampdelay', 'random', + 'consume', 'mixrampd', 'mixrampdelay', 'random', 'repeat', 'setvol', 'single', 'replay_gain_mode', 'replay_gain_status', 'volume', }, expectedFailure=True) + def test_cmd_crossfade(self): + with self.run_bpd() as client: + responses = client.send_commands( + ('status',), + ('crossfade', '123'), + ('status',), + ('crossfade', '-2')) + response = client.send_command('crossfade', '0.5') + self._assert_failed(responses, bpd.ERROR_ARG, pos=3) + self._assert_failed(response, bpd.ERROR_ARG) + self.assertNotIn('xfade', responses[0].data) + self.assertAlmostEqual(123, int(responses[2].data['xfade'])) + class BPDControlTest(BPDTestHelper): test_implements_control = implements({ From 0f53ae9a87c059ac7d8d7f76adc95d67bed4d096 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 19:06:01 +1100 Subject: [PATCH 129/339] bpd: error instead of crashing on extra argument If an MPC client is expecting a command to take an argument that bpd isn't expecting (e.g. because of a difference in protocol versions) then bpd currently crashes completely. Instead, do what the real MPD does and return an error message over the protocol. --- beetsplug/bpd/__init__.py | 10 +++++++++- test/test_player.py | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 0d16e22ca..4f04d01fc 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -674,7 +674,8 @@ class Command(object): # Attempt to get correct command function. func_name = 'cmd_' + self.name if not hasattr(conn.server, func_name): - raise BPDError(ERROR_UNKNOWN, u'unknown command', self.name) + raise BPDError(ERROR_UNKNOWN, + u'unknown command "{}"'.format(self.name)) func = getattr(conn.server, func_name) # Ensure we have permission for this command. @@ -690,6 +691,13 @@ class Command(object): for data in results: yield conn.send(data) + except TypeError: + # The client provided too many arguments. + raise BPDError(ERROR_ARG, + u'wrong number of arguments for "{}"' + .format(self.name), + self.name) + except BPDError as e: # An exposed error. Set the command name and then let # the Connection handle it. diff --git a/test/test_player.py b/test/test_player.py index d7f65ff3f..d2ea496c4 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -347,6 +347,11 @@ class BPDTest(BPDTestHelper): response = client.send_command('notacommand') self._assert_failed(response, bpd.ERROR_UNKNOWN) + def test_unexpected_argument(self): + with self.run_bpd() as client: + response = client.send_command('clearerror', 'extra argument') + self._assert_failed(response, bpd.ERROR_ARG) + class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ From 1511e313f7958a989e56dfb0bae14f4b4fe8e6f3 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 20:18:08 +1100 Subject: [PATCH 130/339] bpd: add mixramp commands These are a more sophisticated version of crossfade so we're free to ignore them, at least for now. We now track the values of the two settings, and show them in the status output. Like MPD, we suppress the mixrampdb value if it's set to nan, which is how it signals that the feature should be turned off. --- beetsplug/bpd/__init__.py | 22 ++++++++++++++++++++++ test/test_player.py | 22 +++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 4f04d01fc..5c394212c 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -25,6 +25,7 @@ from string import Template import traceback import random import time +import math import beets from beets.plugins import BeetsPlugin @@ -174,6 +175,8 @@ class BaseServer(object): self.repeat = False self.volume = VOLUME_MAX self.crossfade = 0 + self.mixrampdb = 0.0 + self.mixrampdelay = float('nan') self.playlist = [] self.playlist_version = 0 self.current_index = -1 @@ -307,8 +310,11 @@ class BaseServer(object): u'random: ' + six.text_type(int(self.random)), u'playlist: ' + six.text_type(self.playlist_version), u'playlistlength: ' + six.text_type(len(self.playlist)), + u'mixrampdb: ' + six.text_type(self.mixrampdb), ) + if not math.isnan(self.mixrampdelay): + yield u'mixrampdelay: ' + six.text_type(self.mixrampdelay) if self.crossfade > 0: yield u'xfade: ' + six.text_type(self.crossfade) @@ -358,6 +364,22 @@ class BaseServer(object): self._log.warning(u'crossfade is not implemented in bpd') self.crossfade = crossfade + def cmd_mixrampdb(self, conn, db): + """Set the mixramp normalised max volume in dB.""" + db = cast_arg(float, db) + if db > 0: + raise BPDError(ERROR_ARG, u'mixrampdb time must be negative') + self._log.warning('mixramp is not implemented in bpd') + self.mixrampdb = db + + def cmd_mixrampdelay(self, conn, delay): + """Set the mixramp delay in seconds.""" + delay = cast_arg(float, delay) + if delay < 0: + raise BPDError(ERROR_ARG, u'mixrampdelay time must be nonnegative') + self._log.warning('mixramp is not implemented in bpd') + self.mixrampdelay = delay + def cmd_clear(self, conn): """Clear the playlist.""" self.playlist = [] diff --git a/test/test_player.py b/test/test_player.py index d2ea496c4..5589b4d6c 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -361,7 +361,7 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ - 'consume', 'mixrampd', 'mixrampdelay', 'random', + 'consume', 'random', 'repeat', 'setvol', 'single', 'replay_gain_mode', 'replay_gain_status', 'volume', }, expectedFailure=True) @@ -379,6 +379,26 @@ class BPDPlaybackTest(BPDTestHelper): self.assertNotIn('xfade', responses[0].data) self.assertAlmostEqual(123, int(responses[2].data['xfade'])) + def test_cmd_mixrampdb(self): + with self.run_bpd() as client: + responses = client.send_commands( + ('mixrampdb', '-17'), + ('status',)) + self._assert_ok(*responses) + self.assertAlmostEqual(-17, float(responses[1].data['mixrampdb'])) + + def test_cmd_mixrampdelay(self): + with self.run_bpd() as client: + responses = client.send_commands( + ('mixrampdelay', '2'), + ('status',), + ('mixrampdelay', 'nan'), + ('status',), + ('mixrampdelay', '-2')) + self._assert_failed(responses, bpd.ERROR_ARG, pos=4) + self.assertAlmostEqual(2, float(responses[1].data['mixrampdelay'])) + self.assertNotIn('mixrampdelay', responses[3].data) + class BPDControlTest(BPDTestHelper): test_implements_control = implements({ From 67a0b38d200cee2fa4f38a084d3128a132512bc8 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 20:25:20 +1100 Subject: [PATCH 131/339] bpd: add dummy command for volume MPD supports a deprecated command 'volume' which was used to change the volume by a relative amount unlike its replacement 'setvol' which uses an absolute amount. As far as I can tell 'volume' always responds with a system error message "No mixer". --- beetsplug/bpd/__init__.py | 4 ++++ test/test_player.py | 23 ++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 5c394212c..4fb4cb25b 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -356,6 +356,10 @@ class BaseServer(object): raise BPDError(ERROR_ARG, u'volume out of range') self.volume = vol + def cmd_volume(self, conn, vol_delta): + """Deprecated command to change the volume by a relative amount.""" + raise BPDError(ERROR_SYSTEM, u'No mixer') + def cmd_crossfade(self, conn, crossfade): """Set the number of seconds of crossfading.""" crossfade = cast_arg(int, crossfade) diff --git a/test/test_player.py b/test/test_player.py index 5589b4d6c..db70a17be 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -46,7 +46,7 @@ gstplayer._GstPlayer = mock.MagicMock( "seek" ], **{ 'playing': False, - 'volume': 0.0, + 'volume': 0, 'time.return_value': (0, 0), 'play_file.side_effect': _gstplayer_play, }) @@ -362,8 +362,8 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ 'consume', 'random', - 'repeat', 'setvol', 'single', 'replay_gain_mode', - 'replay_gain_status', 'volume', + 'repeat', 'single', 'replay_gain_mode', + 'replay_gain_status', }, expectedFailure=True) def test_cmd_crossfade(self): @@ -399,6 +399,23 @@ class BPDPlaybackTest(BPDTestHelper): self.assertAlmostEqual(2, float(responses[1].data['mixrampdelay'])) self.assertNotIn('mixrampdelay', responses[3].data) + def test_cmd_setvol(self): + with self.run_bpd() as client: + responses = client.send_commands( + ('setvol', '67'), + ('status',), + ('setvol', '32'), + ('status',), + ('setvol', '101')) + self._assert_failed(responses, bpd.ERROR_ARG, pos=4) + self.assertEqual('67', responses[1].data['volume']) + self.assertEqual('32', responses[3].data['volume']) + + def test_cmd_volume(self): + with self.run_bpd() as client: + response = client.send_command('volume', '10') + self._assert_failed(response, bpd.ERROR_SYSTEM) + class BPDControlTest(BPDTestHelper): test_implements_control = implements({ From e5851866d74e4ac945f63511b2ff48f16233550e Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 30 Mar 2019 21:30:34 +1100 Subject: [PATCH 132/339] bpd: add replay_gain_* commands There's a special status command for checking the replay gain mode, which can be set to one of a short list of possible values. For now at least we can ignore this feature, but track the setting anyway. --- beetsplug/bpd/__init__.py | 12 ++++++++++++ test/test_player.py | 12 ++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 4fb4cb25b..d5e4489f4 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -177,6 +177,7 @@ class BaseServer(object): self.crossfade = 0 self.mixrampdb = 0.0 self.mixrampdelay = float('nan') + self.replay_gain_mode = 'off' self.playlist = [] self.playlist_version = 0 self.current_index = -1 @@ -384,6 +385,17 @@ class BaseServer(object): self._log.warning('mixramp is not implemented in bpd') self.mixrampdelay = delay + def cmd_replay_gain_mode(self, conn, mode): + """Set the replay gain mode.""" + if mode not in ['off', 'track', 'album', 'auto']: + raise BPDError(ERROR_ARG, u'Unrecognised replay gain mode') + self._log.warning('replay gain is not implemented in bpd') + self.replay_gain_mode = mode + + def cmd_replay_gain_status(self, conn): + """Get the replaygain mode.""" + yield u'replay_gain_mode: ' + six.text_type(self.replay_gain_mode) + def cmd_clear(self, conn): """Clear the playlist.""" self.playlist = [] diff --git a/test/test_player.py b/test/test_player.py index db70a17be..44572c3a6 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -362,8 +362,7 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ 'consume', 'random', - 'repeat', 'single', 'replay_gain_mode', - 'replay_gain_status', + 'repeat', 'single', }, expectedFailure=True) def test_cmd_crossfade(self): @@ -416,6 +415,15 @@ class BPDPlaybackTest(BPDTestHelper): response = client.send_command('volume', '10') self._assert_failed(response, bpd.ERROR_SYSTEM) + def test_cmd_replay_gain(self): + with self.run_bpd() as client: + responses = client.send_commands( + ('replay_gain_mode', 'track'), + ('replay_gain_status',), + ('replay_gain_mode', 'notanoption')) + self._assert_failed(responses, bpd.ERROR_ARG, pos=2) + self.assertAlmostEqual('track', responses[1].data['replay_gain_mode']) + class BPDControlTest(BPDTestHelper): test_implements_control = implements({ From 859e16d1e310b6cc8bdafb3f8fd591df8f36b908 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 31 Mar 2019 01:01:45 +1100 Subject: [PATCH 133/339] bpd: support consume command --- beetsplug/bpd/__init__.py | 11 +++++++++++ test/test_player.py | 23 ++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index d5e4489f4..a0d6d6254 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -173,6 +173,7 @@ class BaseServer(object): # Default server values. self.random = False self.repeat = False + self.consume = False self.volume = VOLUME_MAX self.crossfade = 0 self.mixrampdb = 0.0 @@ -309,6 +310,7 @@ class BaseServer(object): u'volume: ' + six.text_type(self.volume), u'repeat: ' + six.text_type(int(self.repeat)), u'random: ' + six.text_type(int(self.random)), + u'consume: ' + six.text_type(int(self.consume)), u'playlist: ' + six.text_type(self.playlist_version), u'playlistlength: ' + six.text_type(len(self.playlist)), u'mixrampdb: ' + six.text_type(self.mixrampdb), @@ -350,6 +352,10 @@ class BaseServer(object): """Set or unset repeat mode.""" self.repeat = cast_arg('intbool', state) + def cmd_consume(self, conn, state): + """Set or unset consume mode.""" + self.consume = cast_arg('intbool', state) + def cmd_setvol(self, conn, vol): """Set the player's volume level (0-100).""" vol = cast_arg(int, vol) @@ -519,7 +525,12 @@ class BaseServer(object): def cmd_next(self, conn): """Advance to the next song in the playlist.""" + old_index = self.current_index self.current_index = self._succ_idx() + if self.consume: + self.playlist.pop(old_index) + if self.current_index > old_index: + self.current_index -= 1 if self.current_index >= len(self.playlist): # Fallen off the end. Just move to stopped state. return self.cmd_stop(conn) diff --git a/test/test_player.py b/test/test_player.py index 44572c3a6..b9544769c 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -361,10 +361,31 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ - 'consume', 'random', + 'random', 'repeat', 'single', }, expectedFailure=True) + def test_cmd_consume(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('consume', '0'), + ('playlistinfo',), + ('next',), + ('playlistinfo',), + ('consume', '1'), + ('playlistinfo',), + ('play', '0'), + ('next',), + ('playlistinfo',), + ('status',)) + self._assert_ok(*responses) + self.assertEqual(responses[1].data['Id'], responses[3].data['Id']) + self.assertEqual(['1', '2'], responses[5].data['Id']) + self.assertEqual('2', responses[8].data['Id']) + self.assertEqual('1', responses[9].data['consume']) + self.assertEqual('play', responses[9].data['state']) + def test_cmd_crossfade(self): with self.run_bpd() as client: responses = client.send_commands( From 71e7621642fa996e6c4b0eff8e5ad1afa58a3ca8 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 11:28:31 +1100 Subject: [PATCH 134/339] bpd: no-op support for persistent playlists The real MPD offers persistent playlist manipulation, storing the playlists in a directory set in the config file. If that directory is not available then the feature is disabled and the relevant commands all respond with errors. Based on this, the initial support in bpd just returns errors matching the MPD server in the disabled mode. For playlistadd, extend the _bpd_add helper to work with playlists other than the queue in order to support testing the real implementations of these commands in the future. --- beetsplug/bpd/__init__.py | 36 +++++++++++++++++++ test/test_player.py | 74 ++++++++++++++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index a0d6d6254..c29b0604d 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1136,6 +1136,42 @@ class Server(BaseServer): yield u'songs: ' + six.text_type(songs) yield u'playtime: ' + six.text_type(int(playtime)) + # Persistent playlist manipulation. In MPD this is an optional feature so + # these dummy implementations match MPD's behaviour with the feature off. + + def cmd_listplaylist(self, conn, playlist): + raise BPDError(ERROR_NO_EXIST, u'No such playlist') + + def cmd_listplaylistinfo(self, conn, playlist): + raise BPDError(ERROR_NO_EXIST, u'No such playlist') + + def cmd_listplaylists(self, conn): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_load(self, conn, playlist): + raise BPDError(ERROR_NO_EXIST, u'Stored playlists are disabled') + + def cmd_playlistadd(self, conn, playlist, uri): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_playlistclear(self, conn, playlist): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_playlistdelete(self, conn, playlist, index): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_playlistmove(self, conn, playlist, from_index, to_index): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_rename(self, conn, playlist, new_name): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_rm(self, conn, playlist): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + + def cmd_save(self, conn, playlist): + raise BPDError(ERROR_UNKNOWN, u'Stored playlists are disabled') + # "Outputs." Just a dummy implementation because we don't control # any outputs. diff --git a/test/test_player.py b/test/test_player.py index b9544769c..80ba595cd 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -327,13 +327,18 @@ class BPDTestHelper(unittest.TestCase, TestHelper): if code is not None: self.assertEqual(code, response.err_data[0]) - def _bpd_add(self, client, *items): - """ Add the given item to the BPD playlist + def _bpd_add(self, client, *items, **kwargs): + """ Add the given item to the BPD playlist or queue. """ paths = ['/'.join([ item.artist, item.album, py3_path(os.path.basename(item.path))]) for item in items] - responses = client.send_commands(*[('add', path) for path in paths]) + playlist = kwargs.get('playlist') + if playlist: + commands = [('playlistadd', playlist, path) for path in paths] + else: + commands = [('add', path) for path in paths] + responses = client.send_commands(*commands) self._assert_ok(*responses) @@ -488,11 +493,64 @@ class BPDQueueTest(BPDTestHelper): class BPDPlaylistsTest(BPDTestHelper): - test_implements_playlists = implements({ - 'listplaylist', 'listplaylistinfo', 'listplaylists', 'load', - 'playlistadd', 'playlistclear', 'playlistdelete', - 'playlistmove', 'rename', 'rm', 'save', - }, expectedFailure=True) + test_implements_playlists = implements({'playlistadd'}) + + def test_cmd_listplaylist(self): + with self.run_bpd() as client: + response = client.send_command('listplaylist', 'anything') + self._assert_failed(response, bpd.ERROR_NO_EXIST) + + def test_cmd_listplaylistinfo(self): + with self.run_bpd() as client: + response = client.send_command('listplaylistinfo', 'anything') + self._assert_failed(response, bpd.ERROR_NO_EXIST) + + def test_cmd_listplaylists(self): + with self.run_bpd() as client: + response = client.send_command('listplaylists') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + + def test_cmd_load(self): + with self.run_bpd() as client: + response = client.send_command('load', 'anything') + self._assert_failed(response, bpd.ERROR_NO_EXIST) + + @unittest.skip + def test_cmd_playlistadd(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, playlist='anything') + + def test_cmd_playlistclear(self): + with self.run_bpd() as client: + response = client.send_command('playlistclear', 'anything') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + + def test_cmd_playlistdelete(self): + with self.run_bpd() as client: + response = client.send_command('playlistdelete', 'anything', '0') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + + def test_cmd_playlistmove(self): + with self.run_bpd() as client: + response = client.send_command( + 'playlistmove', 'anything', '0', '1') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + + def test_cmd_rename(self): + with self.run_bpd() as client: + response = client.send_command('rename', 'anything', 'newname') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + + def test_cmd_rm(self): + with self.run_bpd() as client: + response = client.send_command('rm', 'anything') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + + def test_cmd_save(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1) + response = client.send_command('save', 'newplaylist') + self._assert_failed(response, bpd.ERROR_UNKNOWN) class BPDDatabaseTest(BPDTestHelper): From bae9c40600018f1797c50b1d83cd3bfb886d77fc Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 16:26:00 +1100 Subject: [PATCH 135/339] bpd: support the single command This command instructs bpd to stop playing when the current song finishes. In the MPD 0.20 protocol this flag gains a value 'oneshot' but for now we just support its older version with a boolean value. --- beetsplug/bpd/__init__.py | 9 +++++++++ test/test_player.py | 20 ++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index c29b0604d..afb7b6c5e 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -174,6 +174,7 @@ class BaseServer(object): self.random = False self.repeat = False self.consume = False + self.single = False self.volume = VOLUME_MAX self.crossfade = 0 self.mixrampdb = 0.0 @@ -311,6 +312,7 @@ class BaseServer(object): u'repeat: ' + six.text_type(int(self.repeat)), u'random: ' + six.text_type(int(self.random)), u'consume: ' + six.text_type(int(self.consume)), + u'single: ' + six.text_type(int(self.single)), u'playlist: ' + six.text_type(self.playlist_version), u'playlistlength: ' + six.text_type(len(self.playlist)), u'mixrampdb: ' + six.text_type(self.mixrampdb), @@ -356,6 +358,11 @@ class BaseServer(object): """Set or unset consume mode.""" self.consume = cast_arg('intbool', state) + def cmd_single(self, conn, state): + """Set or unset single mode.""" + # TODO support oneshot in addition to 0 and 1 [MPD 0.20] + self.single = cast_arg('intbool', state) + def cmd_setvol(self, conn, vol): """Set the player's volume level (0-100).""" vol = cast_arg(int, vol) @@ -534,6 +541,8 @@ class BaseServer(object): if self.current_index >= len(self.playlist): # Fallen off the end. Just move to stopped state. return self.cmd_stop(conn) + elif self.single: + return self.cmd_stop(conn) else: return self.cmd_play(conn) diff --git a/test/test_player.py b/test/test_player.py index 80ba595cd..4ecf67c3d 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -367,8 +367,8 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ 'random', - 'repeat', 'single', - }, expectedFailure=True) + 'repeat', + }) def test_cmd_consume(self): with self.run_bpd() as client: @@ -391,6 +391,22 @@ class BPDPlaybackTest(BPDTestHelper): self.assertEqual('1', responses[9].data['consume']) self.assertEqual('play', responses[9].data['state']) + def test_cmd_single(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('status',), + ('single', '1'), + ('play',), + ('status',), + ('next',), + ('status',)) + self._assert_ok(*responses) + self.assertEqual('0', responses[0].data['single']) + self.assertEqual('1', responses[3].data['single']) + self.assertEqual('play', responses[3].data['state']) + self.assertEqual('stop', responses[5].data['state']) + def test_cmd_crossfade(self): with self.run_bpd() as client: responses = client.send_commands( From b245c0e755a5ab87b527bf09d59180c7c2d8c65d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 16:30:45 +1100 Subject: [PATCH 136/339] bpd: test fields returned by status command --- beetsplug/bpd/__init__.py | 14 +++++++++++--- test/test_player.py | 21 ++++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index afb7b6c5e..425dbab3d 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1009,11 +1009,19 @@ class Server(BaseServer): if self.current_index > -1: item = self.playlist[self.current_index] - yield u'bitrate: ' + six.text_type(item.bitrate / 1000) - # Missing 'audio'. + yield ( + u'bitrate: ' + six.text_type(item.bitrate / 1000), + # TODO provide a real value samplerate:bits:channels 44100:24:2 + u'audio: 0:0:0', + ) (pos, total) = self.player.time() - yield u'time: ' + six.text_type(pos) + u':' + six.text_type(total) + yield ( + u'time: ' + six.text_type(pos) + u':' + six.text_type(total), + # TODO provide elapsed and duration with higher precision + u'elapsed: ' + six.text_type(float(pos)), + u'duration: ' + six.text_type(float(total)), + ) # Also missing 'updating_db'. diff --git a/test/test_player.py b/test/test_player.py index 4ecf67c3d..c11cbaf23 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -360,9 +360,28 @@ class BPDTest(BPDTestHelper): class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ - 'clearerror', 'currentsong', 'idle', 'status', 'stats', + 'clearerror', 'currentsong', 'idle', 'stats', }, expectedFailure=True) + def test_cmd_status(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('status',), + ('play',), + ('status',)) + self._assert_ok(*responses) + fields_not_playing = { + 'repeat', 'random', 'single', 'consume', 'playlist', + 'playlistlength', 'mixrampdb', 'state', + 'volume' # not (always?) returned by MPD + } + self.assertEqual(fields_not_playing, set(responses[0].data.keys())) + fields_playing = fields_not_playing | { + 'song', 'songid', 'time', 'elapsed', 'bitrate', 'duration', 'audio' + } + self.assertEqual(fields_playing, set(responses[2].data.keys())) + class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ From 0c3a63ef9f527b3c2289f34e85cf44c5f97a0129 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 17:39:35 +1100 Subject: [PATCH 137/339] bpd: fix repeat mode behaviour The repeat flag indicates that the entire playlist should be repeated. If both the repeat and single flags are set then this triggers the old behaviour of looping over a single track. --- beetsplug/bpd/__init__.py | 12 ++++++---- test/test_player.py | 50 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 425dbab3d..c43d24fd5 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -233,10 +233,10 @@ class BaseServer(object): def _succ_idx(self): """Returns the index for the next song to play. - It also considers random and repeat flags. + It also considers random, single and repeat flags. No boundaries are checked. """ - if self.repeat: + if self.repeat and self.single: return self.current_index if self.random: return self._random_idx() @@ -535,13 +535,17 @@ class BaseServer(object): old_index = self.current_index self.current_index = self._succ_idx() if self.consume: + # TODO how does consume interact with single+repeat? self.playlist.pop(old_index) if self.current_index > old_index: self.current_index -= 1 if self.current_index >= len(self.playlist): - # Fallen off the end. Just move to stopped state. + # Fallen off the end. Move to stopped state or loop. + if self.repeat: + self.current_index = -1 + return self.cmd_play(conn) return self.cmd_stop(conn) - elif self.single: + elif self.single and not self.repeat: return self.cmd_stop(conn) else: return self.cmd_play(conn) diff --git a/test/test_player.py b/test/test_player.py index c11cbaf23..7707a825c 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -386,7 +386,6 @@ class BPDQueryTest(BPDTestHelper): class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ 'random', - 'repeat', }) def test_cmd_consume(self): @@ -426,6 +425,38 @@ class BPDPlaybackTest(BPDTestHelper): self.assertEqual('play', responses[3].data['state']) self.assertEqual('stop', responses[5].data['state']) + def test_cmd_repeat(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('repeat', '1'), + ('play',), + ('currentsong',), + ('next',), + ('currentsong',), + ('next',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('1', responses[2].data['Id']) + self.assertEqual('2', responses[4].data['Id']) + self.assertEqual('1', responses[6].data['Id']) + + def test_cmd_repeat_with_single(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('repeat', '1'), + ('single', '1'), + ('play',), + ('currentsong',), + ('next',), + ('status',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('1', responses[3].data['Id']) + self.assertEqual('play', responses[5].data['state']) + self.assertEqual('1', responses[6].data['Id']) + def test_cmd_crossfade(self): with self.run_bpd() as client: responses = client.send_commands( @@ -488,7 +519,7 @@ class BPDPlaybackTest(BPDTestHelper): class BPDControlTest(BPDTestHelper): test_implements_control = implements({ - 'next', 'pause', 'playid', 'previous', 'seek', + 'pause', 'playid', 'previous', 'seek', 'seekid', 'seekcur', 'stop', }, expectedFailure=True) @@ -503,6 +534,21 @@ class BPDControlTest(BPDTestHelper): self.assertEqual('stop', responses[0].data['state']) self.assertEqual('play', responses[2].data['state']) + def test_cmd_next(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('play',), + ('currentsong',), + ('next',), + ('currentsong',), + ('next',), + ('status',)) + self._assert_ok(*responses) + self.assertEqual('1', responses[1].data['Id']) + self.assertEqual('2', responses[3].data['Id']) + self.assertEqual('stop', responses[5].data['state']) + class BPDQueueTest(BPDTestHelper): test_implements_queue = implements({ From a4fe6875a1dfc60cbf6c60eb26b697a8c36aeff4 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 17:41:25 +1100 Subject: [PATCH 138/339] bpd: fix bug in bounds check of current song index The songs are indexed starting from zero for the play command, however the bound check was off by one. An index matching the length of the playlist would crash the server instead of responding with an error message over the protocol. --- beetsplug/bpd/__init__.py | 2 +- test/test_player.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index c43d24fd5..9b21fae9d 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -569,7 +569,7 @@ class BaseServer(object): """Begin playback, possibly at a specified playlist index.""" index = cast_arg(int, index) - if index < -1 or index > len(self.playlist): + if index < -1 or index >= len(self.playlist): raise ArgumentIndexError() if index == -1: # No index specified: start where we are. diff --git a/test/test_player.py b/test/test_player.py index 7707a825c..5e196debd 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -525,14 +525,17 @@ class BPDControlTest(BPDTestHelper): def test_cmd_play(self): with self.run_bpd() as client: - self._bpd_add(client, self.item1) + self._bpd_add(client, self.item1, self.item2) responses = client.send_commands( ('status',), ('play',), - ('status',)) + ('status',), + ('play', '1'), + ('currentsong',)) self._assert_ok(*responses) self.assertEqual('stop', responses[0].data['state']) self.assertEqual('play', responses[2].data['state']) + self.assertEqual('2', responses[4].data['Id']) def test_cmd_next(self): with self.run_bpd() as client: From 12e49b3c8807febb18da61b87d181ed2933b0768 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 17:51:21 +1100 Subject: [PATCH 139/339] bpd: skipping backwards through zero keeps playing Previously issuing the 'previous' command when at position 0 on the playlist would cause bpd to stop playing. MPD instead just restarts the currently playing song instead, so we now match this behaviour. --- beetsplug/bpd/__init__.py | 5 ++--- test/test_player.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 9b21fae9d..ad2a3f921 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -554,9 +554,8 @@ class BaseServer(object): """Step back to the last song.""" self.current_index = self._prev_idx() if self.current_index < 0: - return self.cmd_stop(conn) - else: - return self.cmd_play(conn) + self.current_index = 0 + return self.cmd_play(conn) def cmd_pause(self, conn, state=None): """Set the pause state playback.""" diff --git a/test/test_player.py b/test/test_player.py index 5e196debd..b4c5c1cd0 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -519,7 +519,7 @@ class BPDPlaybackTest(BPDTestHelper): class BPDControlTest(BPDTestHelper): test_implements_control = implements({ - 'pause', 'playid', 'previous', 'seek', + 'pause', 'playid', 'seek', 'seekid', 'seekcur', 'stop', }, expectedFailure=True) @@ -552,6 +552,23 @@ class BPDControlTest(BPDTestHelper): self.assertEqual('2', responses[3].data['Id']) self.assertEqual('stop', responses[5].data['state']) + def test_cmd_previous(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('play', '1'), + ('currentsong',), + ('previous',), + ('currentsong',), + ('previous',), + ('status',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('2', responses[1].data['Id']) + self.assertEqual('1', responses[3].data['Id']) + self.assertEqual('play', responses[5].data['state']) + self.assertEqual('1', responses[6].data['Id']) + class BPDQueueTest(BPDTestHelper): test_implements_queue = implements({ From 146c5f5e13f4d1c5fd3bea9642d10613256a0837 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 1 Apr 2019 18:05:10 +1100 Subject: [PATCH 140/339] bpd: fix repeat, consume and single in reverse These flags are all relevant to the 'previous' command as well as the 'next' command. --- beetsplug/bpd/__init__.py | 10 +++++++-- test/test_player.py | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index ad2a3f921..930710e8b 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -247,7 +247,7 @@ class BaseServer(object): It also considers random and repeat flags. No boundaries are checked. """ - if self.repeat: + if self.repeat and self.single: return self.current_index if self.random: return self._random_idx() @@ -552,9 +552,15 @@ class BaseServer(object): def cmd_previous(self, conn): """Step back to the last song.""" + old_index = self.current_index self.current_index = self._prev_idx() + if self.consume: + self.playlist.pop(old_index) if self.current_index < 0: - self.current_index = 0 + if self.repeat: + self.current_index = len(self.playlist) - 1 + else: + self.current_index = 0 return self.cmd_play(conn) def cmd_pause(self, conn, state=None): diff --git a/test/test_player.py b/test/test_player.py index b4c5c1cd0..ecf08c7b3 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -409,6 +409,21 @@ class BPDPlaybackTest(BPDTestHelper): self.assertEqual('1', responses[9].data['consume']) self.assertEqual('play', responses[9].data['state']) + def test_cmd_consume_in_reverse(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('consume', '1'), + ('play', '1'), + ('playlistinfo',), + ('previous',), + ('playlistinfo',), + ('status',)) + self._assert_ok(*responses) + self.assertEqual(['1', '2'], responses[2].data['Id']) + self.assertEqual('1', responses[4].data['Id']) + self.assertEqual('play', responses[5].data['state']) + def test_cmd_single(self): with self.run_bpd() as client: self._bpd_add(client, self.item1, self.item2) @@ -457,6 +472,35 @@ class BPDPlaybackTest(BPDTestHelper): self.assertEqual('play', responses[5].data['state']) self.assertEqual('1', responses[6].data['Id']) + def test_cmd_repeat_in_reverse(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('repeat', '1'), + ('play',), + ('currentsong',), + ('previous',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('1', responses[2].data['Id']) + self.assertEqual('2', responses[4].data['Id']) + + def test_cmd_repeat_with_single_in_reverse(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('repeat', '1'), + ('single', '1'), + ('play',), + ('currentsong',), + ('previous',), + ('status',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('1', responses[3].data['Id']) + self.assertEqual('play', responses[5].data['state']) + self.assertEqual('1', responses[6].data['Id']) + def test_cmd_crossfade(self): with self.run_bpd() as client: responses = client.send_commands( From 4cbe116e4258dafdff0da2dbaf38f75df1b7c63b Mon Sep 17 00:00:00 2001 From: Thomas McWork Date: Mon, 1 Apr 2019 09:13:36 +0200 Subject: [PATCH 141/339] use most simple syntax --- docs/reference/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 87f4165fd..5fcdce3ae 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -571,7 +571,7 @@ default is ``apply``. languages ~~~~~~~~~ -A list of locale names to search for preferred aliases (e.g. ``[de,en]``. For example, setting +A list of locale names to search for preferred aliases (e.g. ``en jp es``. For example, setting this to "en" uses the transliterated artist name "Pyotr Ilyich Tchaikovsky" instead of the Cyrillic script for the composer's name when tagging from MusicBrainz. Defaults to an empty list, meaning that no language is preferred. From e839e4ea191a0eef15661d1a164bee6302d8f800 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 09:39:07 +1100 Subject: [PATCH 142/339] bpd: improve exception handling Check function signature instead of using TypeError to crudely guess that the wrong number of arguments were provided. Prevent bpd from crashing when trying to log a traceback. The `traceback.format_exc` function takes an optional argument which is supposed to be an integer restricting the length of the backtrace to show. Instead we were passing the exception object to this function and causing a new exception to be raised. --- beetsplug/bpd/__init__.py | 29 +++++++++++++++++++---------- test/test_player.py | 12 +++++++++++- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 930710e8b..52619e7e1 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -31,7 +31,7 @@ import beets from beets.plugins import BeetsPlugin import beets.ui from beets import vfs -from beets.util import bluelet +from beets.util import bluelet, inspect from beets.library import Item from beets import dbcore from beets.mediafile import MediaFile @@ -613,12 +613,17 @@ class BaseServer(object): index = self._id_to_index(track_id) return self.cmd_seek(conn, index, pos) + # Debugging/testing commands that are not part of the MPD protocol. + def cmd_profile(self, conn): """Memory profiling for debugging.""" from guppy import hpy heap = hpy().heap() print(heap) + def cmd_crash_TypeError(self, conn): + 'a' + 2 + class Connection(object): """A connection between a client and the server. Handles input and @@ -744,6 +749,17 @@ class Command(object): raise BPDError(ERROR_UNKNOWN, u'unknown command "{}"'.format(self.name)) func = getattr(conn.server, func_name) + argspec = inspect.getargspec(func) + max_args = len(argspec.args) - 1 + min_args = max_args + if argspec.defaults: + min_args -= len(argspec.defaults) + wrong_num = (len(self.args) > max_args) or (len(self.args) < min_args) + if wrong_num and not argspec.varargs: + raise BPDError(ERROR_ARG, + u'wrong number of arguments for "{}"' + .format(self.name), + self.name) # Ensure we have permission for this command. if conn.server.password and \ @@ -758,13 +774,6 @@ class Command(object): for data in results: yield conn.send(data) - except TypeError: - # The client provided too many arguments. - raise BPDError(ERROR_ARG, - u'wrong number of arguments for "{}"' - .format(self.name), - self.name) - except BPDError as e: # An exposed error. Set the command name and then let # the Connection handle it. @@ -776,9 +785,9 @@ class Command(object): # it on the Connection. raise - except Exception as e: + except Exception: # An "unintentional" error. Hide it from the client. - conn.server._log.error('{}', traceback.format_exc(e)) + conn.server._log.error('{}', traceback.format_exc()) raise BPDError(ERROR_SYSTEM, u'server error', self.name) diff --git a/test/test_player.py b/test/test_player.py index ecf08c7b3..98fd13f63 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -354,9 +354,19 @@ class BPDTest(BPDTestHelper): def test_unexpected_argument(self): with self.run_bpd() as client: - response = client.send_command('clearerror', 'extra argument') + response = client.send_command('ping', 'extra argument') self._assert_failed(response, bpd.ERROR_ARG) + def test_missing_argument(self): + with self.run_bpd() as client: + response = client.send_command('add') + self._assert_failed(response, bpd.ERROR_ARG) + + def test_system_error(self): + with self.run_bpd() as client: + response = client.send_command('crash_TypeError') + self._assert_failed(response, bpd.ERROR_SYSTEM) + class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ From 9622e7433beba9336ec3a48e7c03b85f690febf6 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 09:44:34 +1100 Subject: [PATCH 143/339] bpd: return real audio data --- beetsplug/bpd/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 52619e7e1..1e0d57425 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1029,8 +1029,11 @@ class Server(BaseServer): yield ( u'bitrate: ' + six.text_type(item.bitrate / 1000), - # TODO provide a real value samplerate:bits:channels 44100:24:2 - u'audio: 0:0:0', + u'audio: {}:{}:{}'.format( + six.text_type(item.samplerate), + six.text_type(item.bitdepth), + six.text_type(item.channels), + ), ) (pos, total) = self.player.time() From 36c85a8aebe31e721cd4155a3445040ef9f52d50 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 10:10:59 +1100 Subject: [PATCH 144/339] Fix beets.util.inspect for Python 3 Under the original compatibility shim we weren't correctly inclusing `self` in the argument list for bound methods. --- beets/util/inspect.py | 26 +------------------------- beetsplug/bpd/__init__.py | 2 +- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/beets/util/inspect.py b/beets/util/inspect.py index 9815a561a..b90c0fe45 100644 --- a/beets/util/inspect.py +++ b/beets/util/inspect.py @@ -16,35 +16,11 @@ from __future__ import division, absolute_import, print_function import inspect -from collections import namedtuple from six import PY2 -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - - def getargspec(func): if PY2: return inspect.getargspec(func) - - sig = inspect.signature(func) - args = [ - p.name for p in sig.parameters.values() - if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD - ] - varargs = [ - p.name for p in sig.parameters.values() - if p.kind == inspect.Parameter.VAR_POSITIONAL - ] - varargs = varargs[0] if varargs else None - varkw = [ - p.name for p in sig.parameters.values() - if p.kind == inspect.Parameter.VAR_KEYWORD - ] - varkw = varkw[0] if varkw else None - defaults = tuple(p.default for p in sig.parameters.values() - if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD - and p.default is not p.empty) or None - - return ArgSpec(args, varargs, varkw, defaults) + return inspect.getfullargspec(func) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 1e0d57425..086af21b3 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -750,7 +750,7 @@ class Command(object): u'unknown command "{}"'.format(self.name)) func = getattr(conn.server, func_name) argspec = inspect.getargspec(func) - max_args = len(argspec.args) - 1 + max_args = len(argspec.args) - 2 min_args = max_args if argspec.defaults: min_args -= len(argspec.defaults) From 4be2e1b5e68d37e806b9766c30436f987113912b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 10:22:47 +1100 Subject: [PATCH 145/339] Remove beets.util.inspect wrapper --- beets/plugins.py | 9 ++++++--- beets/util/inspect.py | 26 -------------------------- beetsplug/bpd/__init__.py | 11 +++++++++-- test/test_plugins.py | 2 +- 4 files changed, 16 insertions(+), 32 deletions(-) delete mode 100644 beets/util/inspect.py diff --git a/beets/plugins.py b/beets/plugins.py index 7019b70a0..5ca9ae3bb 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -19,6 +19,7 @@ from __future__ import division, absolute_import, print_function import traceback import re +import inspect from collections import defaultdict from functools import wraps @@ -26,7 +27,6 @@ from functools import wraps import beets from beets import logging from beets import mediafile -from beets.util import inspect import six PLUGIN_NAMESPACE = 'beetsplug' @@ -127,7 +127,10 @@ class BeetsPlugin(object): value after the function returns). Also determines which params may not be sent for backwards-compatibility. """ - argspec = inspect.getargspec(func) + if six.PY2: + func_args = inspect.getargspec(func).args + else: + func_args = inspect.getfullargspec(func).args @wraps(func) def wrapper(*args, **kwargs): @@ -142,7 +145,7 @@ class BeetsPlugin(object): if exc.args[0].startswith(func.__name__): # caused by 'func' and not stuff internal to 'func' kwargs = dict((arg, val) for arg, val in kwargs.items() - if arg in argspec.args) + if arg in func_args) return func(*args, **kwargs) else: raise diff --git a/beets/util/inspect.py b/beets/util/inspect.py deleted file mode 100644 index b90c0fe45..000000000 --- a/beets/util/inspect.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2019, Vladimir Zhelezov. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -from __future__ import division, absolute_import, print_function - -import inspect - -from six import PY2 - - -def getargspec(func): - if PY2: - return inspect.getargspec(func) - return inspect.getfullargspec(func) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 086af21b3..c8fc10509 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -26,12 +26,13 @@ import traceback import random import time import math +import inspect import beets from beets.plugins import BeetsPlugin import beets.ui from beets import vfs -from beets.util import bluelet, inspect +from beets.util import bluelet from beets.library import Item from beets import dbcore from beets.mediafile import MediaFile @@ -749,7 +750,13 @@ class Command(object): raise BPDError(ERROR_UNKNOWN, u'unknown command "{}"'.format(self.name)) func = getattr(conn.server, func_name) - argspec = inspect.getargspec(func) + + if six.PY2: + # caution: the fields of the namedtuple are slightly different + argspec = inspect.getargspec(func) + else: + argspec = inspect.getfullargspec(func) + max_args = len(argspec.args) - 2 min_args = max_args if argspec.defaults: diff --git a/test/test_plugins.py b/test/test_plugins.py index 7c32e9aca..b14158699 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -322,7 +322,7 @@ class ListenersTest(unittest.TestCase, TestHelper): @patch('beets.plugins.find_plugins') @patch('beets.plugins.inspect') def test_events_called(self, mock_inspect, mock_find_plugins): - mock_inspect.getargspec.return_value = None + mock_inspect.getargspec.args.return_value = None class DummyPlugin(plugins.BeetsPlugin): def __init__(self): From 28db7d3d33b8bd989e9dc32fd9c0fc97dbbf42c7 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 11:15:00 +1100 Subject: [PATCH 146/339] bpd: provide precision time in status --- beetsplug/bpd/__init__.py | 10 ++++++---- beetsplug/bpd/gstplayer.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index c8fc10509..fd714ab63 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1045,10 +1045,12 @@ class Server(BaseServer): (pos, total) = self.player.time() yield ( - u'time: ' + six.text_type(pos) + u':' + six.text_type(total), - # TODO provide elapsed and duration with higher precision - u'elapsed: ' + six.text_type(float(pos)), - u'duration: ' + six.text_type(float(total)), + u'time: {}:{}'.format( + six.text_type(int(pos)), + six.text_type(int(total)), + ), + u'elapsed: ' + u'{:.3f}'.format(pos), + u'duration: ' + u'{:.3f}'.format(total), ) # Also missing 'updating_db'. diff --git a/beetsplug/bpd/gstplayer.py b/beetsplug/bpd/gstplayer.py index 705692aa5..fffa8a6ed 100644 --- a/beetsplug/bpd/gstplayer.py +++ b/beetsplug/bpd/gstplayer.py @@ -177,12 +177,12 @@ class GstPlayer(object): posq = self.player.query_position(fmt) if not posq[0]: raise QueryError("query_position failed") - pos = posq[1] // (10 ** 9) + pos = posq[1] / (10 ** 9) lengthq = self.player.query_duration(fmt) if not lengthq[0]: raise QueryError("query_duration failed") - length = lengthq[1] // (10 ** 9) + length = lengthq[1] / (10 ** 9) self.cached_time = (pos, length) return (pos, length) From d074dac771689fc33a59cdcd115c4bfe80f6afed Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 13:37:40 +1100 Subject: [PATCH 147/339] bpd: add comments to the error handling code --- beetsplug/bpd/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index fd714ab63..93237ca87 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -623,6 +623,11 @@ class BaseServer(object): print(heap) def cmd_crash_TypeError(self, conn): + """Deliberately trigger a TypeError for testing purposes. + We want to test that the server properly responds with ERROR_SYSTEM + without crashing, and that this is not treated as ERROR_ARG (since it + is caused by a programming error, not a protocol error). + """ 'a' + 2 @@ -753,15 +758,21 @@ class Command(object): if six.PY2: # caution: the fields of the namedtuple are slightly different + # between the results of getargspec and getfullargspec. argspec = inspect.getargspec(func) else: argspec = inspect.getfullargspec(func) + # Check that `func` is able to handle the number of arguments sent + # by the client (so we can raise ERROR_ARG instead of ERROR_SYSTEM). + # Maximum accepted arguments: argspec includes "self" and "conn". max_args = len(argspec.args) - 2 + # Minimum accepted arguments: some arguments might be optional/ min_args = max_args if argspec.defaults: min_args -= len(argspec.defaults) wrong_num = (len(self.args) > max_args) or (len(self.args) < min_args) + # If the command accepts a variable number of arguments skip the check. if wrong_num and not argspec.varargs: raise BPDError(ERROR_ARG, u'wrong number of arguments for "{}"' From 20e2f8beec316b2c6bd535ae110c316b7274a0c2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 13:38:43 +1100 Subject: [PATCH 148/339] bpd: output an info-level message when ready --- beetsplug/bpd/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 93237ca87..88faad703 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -858,10 +858,13 @@ class Server(BaseServer): raise NoGstreamerError() else: raise + log.info(u'Starting server...') super(Server, self).__init__(host, port, password, log) self.lib = library self.player = gstplayer.GstPlayer(self.play_finished) self.cmd_update(None) + log.info(u'Server ready and listening on {}:{}'.format( + host, port)) def run(self): self.player.run() From 140d25df5287c0cf949094b696266113c01d7998 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 13:50:16 +1100 Subject: [PATCH 149/339] Changelog for #3200 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c822af608..1acfdb9c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -81,6 +81,11 @@ New features: to be displayed. Thanks to :user:`pprkut`. :bug:`3089` +* :doc:`/plugins/bpd`: MPD protocol commands ``consume`` and ``single`` are now + supported along with updated semantics for ``repeat`` and ``previous`` and + new fields for ``status``. The bpd server now understands and ignores some + additional commands. + :bug:`3200` :bug:`800` Changes: @@ -181,6 +186,8 @@ Fixes: :bug:`3192` * Fix compatibility with pre-release versions of Python 3.8. :bug:`3201` :bug:`3202` +* :doc:`/plugins/bpd`: Fix crashes in the bpd server during exception handling. + :bug:`3200` .. _python-itunes: https://github.com/ocelma/python-itunes From 95dd513b2585d0a1a997f25ccfae856027039e69 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 2 Apr 2019 14:25:56 +1100 Subject: [PATCH 150/339] bpd: add flake8 exception for test command --- beetsplug/bpd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 88faad703..598e2971f 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -622,7 +622,7 @@ class BaseServer(object): heap = hpy().heap() print(heap) - def cmd_crash_TypeError(self, conn): + def cmd_crash_TypeError(self, conn): # noqa: N802 """Deliberately trigger a TypeError for testing purposes. We want to test that the server properly responds with ERROR_SYSTEM without crashing, and that this is not treated as ERROR_ARG (since it From f35944f49c5afcd5e02a20c616802446dc534a43 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 6 Apr 2019 11:06:19 +1100 Subject: [PATCH 151/339] random: move implementation to beets.util.random --- beets/util/random.py | 110 +++++++++++++++++++++++++++++++++++++++++++ beetsplug/random.py | 92 +----------------------------------- 2 files changed, 111 insertions(+), 91 deletions(-) create mode 100644 beets/util/random.py diff --git a/beets/util/random.py b/beets/util/random.py new file mode 100644 index 000000000..2f385a246 --- /dev/null +++ b/beets/util/random.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Philippe Mongeau. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Get a random song or album from the library. +""" +from __future__ import division, absolute_import, print_function + +import random +from operator import attrgetter +from itertools import groupby + + +def _length(obj, album): + """Get the duration of an item or album. + """ + if album: + return sum(i.length for i in obj.items()) + else: + return obj.length + + +def _equal_chance_permutation(objs, field='albumartist'): + """Generate (lazily) a permutation of the objects where every group + with equal values for `field` have an equal chance of appearing in + any given position. + """ + # Group the objects by artist so we can sample from them. + key = attrgetter(field) + objs.sort(key=key) + objs_by_artists = {} + for artist, v in groupby(objs, key): + objs_by_artists[artist] = list(v) + + # While we still have artists with music to choose from, pick one + # randomly and pick a track from that artist. + while objs_by_artists: + # Choose an artist and an object for that artist, removing + # this choice from the pool. + artist = random.choice(list(objs_by_artists.keys())) + objs_from_artist = objs_by_artists[artist] + i = random.randint(0, len(objs_from_artist) - 1) + yield objs_from_artist.pop(i) + + # Remove the artist if we've used up all of its objects. + if not objs_from_artist: + del objs_by_artists[artist] + + +def _take(iter, num): + """Return a list containing the first `num` values in `iter` (or + fewer, if the iterable ends early). + """ + out = [] + for val in iter: + out.append(val) + num -= 1 + if num <= 0: + break + return out + + +def _take_time(iter, secs, album): + """Return a list containing the first values in `iter`, which should + be Item or Album objects, that add up to the given amount of time in + seconds. + """ + out = [] + total_time = 0.0 + for obj in iter: + length = _length(obj, album) + if total_time + length <= secs: + out.append(obj) + total_time += length + return out + + +def random_objs(objs, album, number=1, time=None, equal_chance=False): + """Get a random subset of the provided `objs`. + + If `number` is provided, produce that many matches. Otherwise, if + `time` is provided, instead select a list whose total time is close + to that number of minutes. If `equal_chance` is true, give each + artist an equal chance of being included so that artists with more + songs are not represented disproportionately. + """ + # Permute the objects either in a straightforward way or an + # artist-balanced way. + if equal_chance: + perm = _equal_chance_permutation(objs) + else: + perm = objs + random.shuffle(perm) # N.B. This shuffles the original list. + + # Select objects by time our count. + if time: + return _take_time(perm, time * 60, album) + else: + return _take(perm, number) diff --git a/beetsplug/random.py b/beetsplug/random.py index 65caaf908..567d5e069 100644 --- a/beetsplug/random.py +++ b/beetsplug/random.py @@ -19,97 +19,7 @@ from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets.ui import Subcommand, decargs, print_ -import random -from operator import attrgetter -from itertools import groupby - - -def _length(obj, album): - """Get the duration of an item or album. - """ - if album: - return sum(i.length for i in obj.items()) - else: - return obj.length - - -def _equal_chance_permutation(objs, field='albumartist'): - """Generate (lazily) a permutation of the objects where every group - with equal values for `field` have an equal chance of appearing in - any given position. - """ - # Group the objects by artist so we can sample from them. - key = attrgetter(field) - objs.sort(key=key) - objs_by_artists = {} - for artist, v in groupby(objs, key): - objs_by_artists[artist] = list(v) - - # While we still have artists with music to choose from, pick one - # randomly and pick a track from that artist. - while objs_by_artists: - # Choose an artist and an object for that artist, removing - # this choice from the pool. - artist = random.choice(list(objs_by_artists.keys())) - objs_from_artist = objs_by_artists[artist] - i = random.randint(0, len(objs_from_artist) - 1) - yield objs_from_artist.pop(i) - - # Remove the artist if we've used up all of its objects. - if not objs_from_artist: - del objs_by_artists[artist] - - -def _take(iter, num): - """Return a list containing the first `num` values in `iter` (or - fewer, if the iterable ends early). - """ - out = [] - for val in iter: - out.append(val) - num -= 1 - if num <= 0: - break - return out - - -def _take_time(iter, secs, album): - """Return a list containing the first values in `iter`, which should - be Item or Album objects, that add up to the given amount of time in - seconds. - """ - out = [] - total_time = 0.0 - for obj in iter: - length = _length(obj, album) - if total_time + length <= secs: - out.append(obj) - total_time += length - return out - - -def random_objs(objs, album, number=1, time=None, equal_chance=False): - """Get a random subset of the provided `objs`. - - If `number` is provided, produce that many matches. Otherwise, if - `time` is provided, instead select a list whose total time is close - to that number of minutes. If `equal_chance` is true, give each - artist an equal chance of being included so that artists with more - songs are not represented disproportionately. - """ - # Permute the objects either in a straightforward way or an - # artist-balanced way. - if equal_chance: - perm = _equal_chance_permutation(objs) - else: - perm = objs - random.shuffle(perm) # N.B. This shuffles the original list. - - # Select objects by time our count. - if time: - return _take_time(perm, time * 60, album) - else: - return _take(perm, number) +from beets.util.random import random_objs def random_func(lib, opts, args): From 438009ba94421c51fdaa341406f84acfb472da0c Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 6 Apr 2019 12:10:27 +1100 Subject: [PATCH 152/339] random: add test for equal_chance_permutation --- test/test_random.py | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 test/test_random.py diff --git a/test/test_random.py b/test/test_random.py new file mode 100644 index 000000000..2e401c245 --- /dev/null +++ b/test/test_random.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Carl Suster +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Test the beets.util.random utilities associated with the random plugin. +""" + +from __future__ import division, absolute_import, print_function + +import unittest +from test.helper import TestHelper + +import math +from random import Random + +from beets.util import random + + +class RandomTest(unittest.TestCase, TestHelper): + def setUp(self): + self.lib = None + self.artist1 = 'Artist 1' + self.artist2 = 'Artist 2' + self.item1 = self.create_item(artist=self.artist1) + self.item2 = self.create_item(artist=self.artist2) + self.items = [self.item1, self.item2] + for _ in range(8): + self.items.append(self.create_item(artist=self.artist2)) + self.random_gen = Random() + self.random_gen.seed(12345) + + def tearDown(self): + pass + + def _stats(self, data): + mean = sum(data) / len(data) + stdev = math.sqrt( + sum((p - mean) ** 2 for p in data) / (len(data) - 1)) + quot, rem = divmod(len(data), 2) + if rem: + median = sorted(data)[quot] + else: + median = sum(sorted(data)[quot - 1:quot + 1]) / 2 + return mean, stdev, median + + def test_equal_permutation(self): + """We have a list of items where only one item is from artist1 and the + rest are from artist2. If we permute weighted by the artist field then + the solo track will almost always end up near the start. If we use a + different field then it'll be in the middle on average. + """ + def experiment(field, histogram=False): + """Permutes the list of items 500 times and calculates the position + of self.item1 each time. Returns stats about that position. + """ + positions = [] + for _ in range(500): + shuffled = list(random._equal_chance_permutation( + self.items, field=field)) + positions.append(shuffled.index(self.item1)) + # Print a histogram (useful for debugging). + if histogram: + for i in range(len(self.items)): + print('{:2d} {}'.format(i, '*' * positions.count(i))) + return self._stats(positions) + + mean1, stdev1, median1 = experiment('artist') + mean2, stdev2, median2 = experiment('track') + self.assertAlmostEqual(0, median1, delta=1) + self.assertAlmostEqual(len(self.items) // 2, median2, delta=1) + self.assertGreater(stdev2, stdev1) From 9056467fdcbffea6b81a8443a1719b6c980dfaa6 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 6 Apr 2019 12:12:52 +1100 Subject: [PATCH 153/339] random: allow custom random generator --- beets/util/random.py | 15 ++++++++++----- test/test_random.py | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/beets/util/random.py b/beets/util/random.py index 2f385a246..5387da4da 100644 --- a/beets/util/random.py +++ b/beets/util/random.py @@ -31,11 +31,13 @@ def _length(obj, album): return obj.length -def _equal_chance_permutation(objs, field='albumartist'): +def _equal_chance_permutation(objs, field='albumartist', random_gen=None): """Generate (lazily) a permutation of the objects where every group with equal values for `field` have an equal chance of appearing in any given position. """ + rand = random_gen or random + # Group the objects by artist so we can sample from them. key = attrgetter(field) objs.sort(key=key) @@ -48,9 +50,9 @@ def _equal_chance_permutation(objs, field='albumartist'): while objs_by_artists: # Choose an artist and an object for that artist, removing # this choice from the pool. - artist = random.choice(list(objs_by_artists.keys())) + artist = rand.choice(list(objs_by_artists.keys())) objs_from_artist = objs_by_artists[artist] - i = random.randint(0, len(objs_from_artist) - 1) + i = rand.randint(0, len(objs_from_artist) - 1) yield objs_from_artist.pop(i) # Remove the artist if we've used up all of its objects. @@ -86,7 +88,8 @@ def _take_time(iter, secs, album): return out -def random_objs(objs, album, number=1, time=None, equal_chance=False): +def random_objs(objs, album, number=1, time=None, equal_chance=False, + random_gen=None): """Get a random subset of the provided `objs`. If `number` is provided, produce that many matches. Otherwise, if @@ -95,13 +98,15 @@ def random_objs(objs, album, number=1, time=None, equal_chance=False): artist an equal chance of being included so that artists with more songs are not represented disproportionately. """ + rand = random_gen or random + # Permute the objects either in a straightforward way or an # artist-balanced way. if equal_chance: perm = _equal_chance_permutation(objs) else: perm = objs - random.shuffle(perm) # N.B. This shuffles the original list. + rand.shuffle(perm) # N.B. This shuffles the original list. # Select objects by time our count. if time: diff --git a/test/test_random.py b/test/test_random.py index 2e401c245..099a52b02 100644 --- a/test/test_random.py +++ b/test/test_random.py @@ -67,7 +67,7 @@ class RandomTest(unittest.TestCase, TestHelper): positions = [] for _ in range(500): shuffled = list(random._equal_chance_permutation( - self.items, field=field)) + self.items, field=field, random_gen=self.random_gen)) positions.append(shuffled.index(self.item1)) # Print a histogram (useful for debugging). if histogram: From ce5981b885d6619b0b72272e4072962679d6723d Mon Sep 17 00:00:00 2001 From: Rainer Unseld Date: Tue, 5 Dec 2017 17:20:00 +0000 Subject: [PATCH 154/339] mpdstats: use currentsong instead of playlist Improved the method to get the path of the current song. Before, the complete playlist was fetched from the server. Now, the command "currentsong" is used for this purpose. This improves performance when a huge playlist is active. --- beetsplug/mpdstats.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index e5e82d480..423cde2b8 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -107,17 +107,17 @@ class MPDClientWrapper(object): self.connect() return self.get(command, retries=retries - 1) - def playlist(self): - """Return the currently active playlist. Prefixes paths with the + def currentsong(self): + """Return the path to the currently playing song. Prefixes paths with the music_directory, to get the absolute path. """ - result = {} - for entry in self.get('playlistinfo'): + result = None + entry = self.get('currentsong') + if 'file' in entry: if not is_url(entry['file']): - result[entry['id']] = os.path.join( - self.music_directory, entry['file']) + result = os.path.join(self.music_directory, entry['file']) else: - result[entry['id']] = entry['file'] + result = entry['file'] return result def status(self): @@ -250,12 +250,16 @@ class MPDStats(object): self.now_playing = None def on_play(self, status): - playlist = self.mpd.playlist() - path = playlist.get(status['songid']) + + path = self.mpd.currentsong() if not path: return + if is_url(path): + self._log.info(u'playing stream {0}', displayable_path(path)) + return + played, duration = map(int, status['time'].split(':', 1)) remaining = duration - played @@ -272,14 +276,6 @@ class MPDStats(object): if diff <= self.time_threshold: return - if self.now_playing['path'] == path and played == 0: - self.handle_song_change(self.now_playing) - - if is_url(path): - self._log.info(u'playing stream {0}', displayable_path(path)) - self.now_playing = None - return - self._log.info(u'playing {0}', displayable_path(path)) self.now_playing = { From 4158bdb95eeeb3e2130d82f31533d6bfbec0863c Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 6 Apr 2019 15:31:03 +1100 Subject: [PATCH 155/339] mpdstats: update tests for currentsong --- test/test_mpdstats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_mpdstats.py b/test/test_mpdstats.py index 7452be86b..0117e22aa 100644 --- a/test/test_mpdstats.py +++ b/test/test_mpdstats.py @@ -65,7 +65,7 @@ class MPDStatsTest(unittest.TestCase, TestHelper): @patch("beetsplug.mpdstats.MPDClientWrapper", return_value=Mock(**{ "events.side_effect": EVENTS, "status.side_effect": STATUSES, - "playlist.return_value": {1: item_path}})) + "currentsong.return_value": item_path})) def test_run_mpdstats(self, mpd_mock): item = Item(title=u'title', path=self.item_path, id=1) item.add(self.lib) From f35eda717be19b67e32d3b01b1491494bec50f93 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 6 Apr 2019 15:36:52 +1100 Subject: [PATCH 156/339] Changelog for #3207 --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1acfdb9c3..5de1fae6d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -188,6 +188,11 @@ Fixes: :bug:`3201` :bug:`3202` * :doc:`/plugins/bpd`: Fix crashes in the bpd server during exception handling. :bug:`3200` +* :doc:`/plugins/mpdstats`: Use the ``currentsong`` MPD command instead of + ``playlist`` to get the current song, improving performance when the playlist + is long. + Thanks to :user:`ray66`. + :bug:`3207` :bug:`2752` .. _python-itunes: https://github.com/ocelma/python-itunes From 0e93b0f58dc4db17b65577f87f6a06539f097be3 Mon Sep 17 00:00:00 2001 From: Thomas McWork Date: Sat, 6 Apr 2019 09:29:15 +0200 Subject: [PATCH 157/339] follow comment explanation --- docs/reference/config.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 5fcdce3ae..4d3d6fff1 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -571,10 +571,12 @@ default is ``apply``. languages ~~~~~~~~~ -A list of locale names to search for preferred aliases (e.g. ``en jp es``. For example, setting -this to "en" uses the transliterated artist name "Pyotr Ilyich Tchaikovsky" +A list of locale names to search for preferred aliases. For example, setting +this to ``en`` uses the transliterated artist name "Pyotr Ilyich Tchaikovsky" instead of the Cyrillic script for the composer's name when tagging from -MusicBrainz. Defaults to an empty list, meaning that no language is preferred. +MusicBrainz. You can use a space-separated list of language abbreviations, like +``en jp es``, to specify a preference order. Defaults to an empty list, meaning +that no language is preferred. .. _detail: From 274852740780d60ea3376b540bb39b523ef5ee9c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 6 Apr 2019 13:49:50 -0400 Subject: [PATCH 158/339] Changelog for #3054 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1acfdb9c3..ce04d93b4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -188,6 +188,8 @@ Fixes: :bug:`3201` :bug:`3202` * :doc:`/plugins/bpd`: Fix crashes in the bpd server during exception handling. :bug:`3200` +* :doc:`/plugins/lastgenre`: The `force` config option now actually works. + :bug:`2704` :bug:`3054` .. _python-itunes: https://github.com/ocelma/python-itunes From c41197cc5cfea2d10f159d6fa523dd789934cc30 Mon Sep 17 00:00:00 2001 From: gdtwst Date: Sun, 7 Apr 2019 08:38:40 +0700 Subject: [PATCH 159/339] Re-add fixes from #2707 --- beetsplug/mpdstats.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 423cde2b8..876dcacdd 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -256,10 +256,6 @@ class MPDStats(object): if not path: return - if is_url(path): - self._log.info(u'playing stream {0}', displayable_path(path)) - return - played, duration = map(int, status['time'].split(':', 1)) remaining = duration - played @@ -276,6 +272,14 @@ class MPDStats(object): if diff <= self.time_threshold: return + if self.now_playing['path'] == path and played == 0: + self.handle_song_change(self.now_playing) + + if is_url(path): + self._log.info(u'playing stream {0}', displayable_path(path)) + self.now_playing = None + return + self._log.info(u'playing {0}', displayable_path(path)) self.now_playing = { From dc897c0bb9951b25e6685ad59a23dcfc8a604fa4 Mon Sep 17 00:00:00 2001 From: Jack Wilson Date: Sun, 7 Apr 2019 16:31:21 +1000 Subject: [PATCH 160/339] Catch OSError in prune_dirs Since `fnmatch_all` can raise `OSError`s, and we were already silently giving up in `prune_dirs` on some of these exception, just do more of the same. --- beets/util/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index f3dedcb41..f5ad2da22 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -283,13 +283,13 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')): continue clutter = [bytestring_path(c) for c in clutter] match_paths = [bytestring_path(d) for d in os.listdir(directory)] - if fnmatch_all(match_paths, clutter): - # Directory contains only clutter (or nothing). - try: + try: + if fnmatch_all(match_paths, clutter): + # Directory contains only clutter (or nothing). shutil.rmtree(directory) - except OSError: + else: break - else: + except OSError: break From 27599410e80b8941321319ae8479a1380bf94791 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 7 Apr 2019 16:39:28 +1000 Subject: [PATCH 161/339] Changelog for #3209 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b2c8437b6..43b6b20f6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -195,6 +195,8 @@ Fixes: is long. Thanks to :user:`ray66`. :bug:`3207` :bug:`2752` +* Fix an unhandled exception when pruning empty directories. + :bug:`1996` :bug:`3209` .. _python-itunes: https://github.com/ocelma/python-itunes From 9147fabb979b347cdf8a4ee9f017599987676fb2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 7 Apr 2019 11:56:44 +1000 Subject: [PATCH 162/339] random: beets.util.random -> beets.random --- beets/{util => }/random.py | 0 beetsplug/random.py | 2 +- test/test_random.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename beets/{util => }/random.py (100%) diff --git a/beets/util/random.py b/beets/random.py similarity index 100% rename from beets/util/random.py rename to beets/random.py diff --git a/beetsplug/random.py b/beetsplug/random.py index 567d5e069..a8e29313a 100644 --- a/beetsplug/random.py +++ b/beetsplug/random.py @@ -19,7 +19,7 @@ from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets.ui import Subcommand, decargs, print_ -from beets.util.random import random_objs +from beets.random import random_objs def random_func(lib, opts, args): diff --git a/test/test_random.py b/test/test_random.py index 099a52b02..4c31acdd9 100644 --- a/test/test_random.py +++ b/test/test_random.py @@ -13,7 +13,7 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Test the beets.util.random utilities associated with the random plugin. +"""Test the beets.random utilities associated with the random plugin. """ from __future__ import division, absolute_import, print_function @@ -24,7 +24,7 @@ from test.helper import TestHelper import math from random import Random -from beets.util import random +from beets import random class RandomTest(unittest.TestCase, TestHelper): From f8a2c22e8deb574144a27c6d1c3989171c10808b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 4 Apr 2019 17:56:13 +1100 Subject: [PATCH 163/339] bpd: fix typo in comment --- beetsplug/bpd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 598e2971f..4432ab526 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -767,7 +767,7 @@ class Command(object): # by the client (so we can raise ERROR_ARG instead of ERROR_SYSTEM). # Maximum accepted arguments: argspec includes "self" and "conn". max_args = len(argspec.args) - 2 - # Minimum accepted arguments: some arguments might be optional/ + # Minimum accepted arguments: some arguments might be optional. min_args = max_args if argspec.defaults: min_args -= len(argspec.defaults) From 5b0a02eb31dd804deeaf2c357522ab15b390a673 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 5 Apr 2019 18:56:34 +1100 Subject: [PATCH 164/339] bpd: don't send volume if zero in status --- beetsplug/bpd/__init__.py | 4 +++- test/test_player.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 4432ab526..dc7f64db7 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -309,7 +309,6 @@ class BaseServer(object): playlist, playlistlength, and xfade. """ yield ( - u'volume: ' + six.text_type(self.volume), u'repeat: ' + six.text_type(int(self.repeat)), u'random: ' + six.text_type(int(self.random)), u'consume: ' + six.text_type(int(self.consume)), @@ -319,6 +318,9 @@ class BaseServer(object): u'mixrampdb: ' + six.text_type(self.mixrampdb), ) + if self.volume > 0: + yield u'volume: ' + six.text_type(self.volume) + if not math.isnan(self.mixrampdelay): yield u'mixrampdelay: ' + six.text_type(self.mixrampdelay) if self.crossfade > 0: diff --git a/test/test_player.py b/test/test_player.py index 98fd13f63..aa3c3d6a8 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -384,7 +384,7 @@ class BPDQueryTest(BPDTestHelper): fields_not_playing = { 'repeat', 'random', 'single', 'consume', 'playlist', 'playlistlength', 'mixrampdb', 'state', - 'volume' # not (always?) returned by MPD + 'volume' } self.assertEqual(fields_not_playing, set(responses[0].data.keys())) fields_playing = fields_not_playing | { From ee0c31ba6a3b00a671a8c2af1b5934af89d2f3df Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 4 Apr 2019 17:50:21 +1100 Subject: [PATCH 165/339] bpd: track and log client session details Keep track of a list of currently-connected clients. Use `socket.getpeername()` to get an identifier for each connection and include this in each log message. This function is documented as not being available on all systems, but it's unclear which systems this involves. Also log a message on client connect and disconnect events. If the disconnection reason is because the client sent a blank line, match MPD by returning a protocol error then hanging up. Escape curly braces. --- beetsplug/bpd/__init__.py | 39 +++++++++++++++++++++++++++++++++------ test/test_player.py | 5 +++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index dc7f64db7..153fd675e 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -187,9 +187,22 @@ class BaseServer(object): self.paused = False self.error = None + # Current connections + self.connections = set() + # Object for random numbers generation self.random_obj = random.Random() + def connect(self, conn): + """A new client has connected. + """ + self.connections.add(conn) + + def disconnect(self, conn): + """Client has disconnected; clean up residual state. + """ + self.connections.remove(conn) + def run(self): """Block and start listening for connections from clients. An interrupt (^C) closes the server. @@ -643,6 +656,7 @@ class Connection(object): self.server = server self.sock = sock self.authenticated = False + self.address = u'{}:{}'.format(*sock.sock.getpeername()) def send(self, lines): """Send lines, which which is either a single string or an @@ -653,9 +667,9 @@ class Connection(object): if isinstance(lines, six.string_types): lines = [lines] out = NEWLINE.join(lines) + NEWLINE - # Don't log trailing newline: - message = out[:-1].replace(u'\n', u'\n' + u' ' * 13) - self.server._log.debug('server: {}', message) + session = u'>[{}]: '.format(self.address) + for l in out.split(NEWLINE)[:-1]: + self.server._log.debug('{}', session + l) if isinstance(out, six.text_type): out = out.encode('utf-8') return self.sock.sendall(out) @@ -672,24 +686,36 @@ class Connection(object): # Send success code. yield self.send(RESP_OK) + def disconnect(self): + """The connection has closed for any reason. + """ + self.server.disconnect(self) + self.server._log.debug(u'*[{}]: disconnected', self.address) + def run(self): """Send a greeting to the client and begin processing commands as they arrive. """ - self.server._log.debug('New client connected') + self.server._log.debug(u'*[{}]: connected', self.address) + self.server.connect(self) yield self.send(HELLO) + session = u'<[{}]: '.format(self.address) clist = None # Initially, no command list is being constructed. while True: line = yield self.sock.readline() if not line: + self.disconnect() # Client disappeared. break line = line.strip() if not line: + err = BPDError(ERROR_UNKNOWN, u'No command given') + yield self.send(err.response()) + self.disconnect() # Client sent a blank line. break line = line.decode('utf8') # MPD protocol uses UTF-8. - message = line.replace(u'\n', u'\n' + u' ' * 13) - self.server._log.debug(u'client: {}', message) + for l in line.split(NEWLINE): + self.server._log.debug('{}', session + l) if clist is not None: # Command list already opened. @@ -710,6 +736,7 @@ class Connection(object): except BPDClose: # Command indicates that the conn should close. self.sock.close() + self.disconnect() # Client explicitly closed. return @classmethod diff --git a/test/test_player.py b/test/test_player.py index aa3c3d6a8..baf1ddfdb 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -367,6 +367,11 @@ class BPDTest(BPDTestHelper): response = client.send_command('crash_TypeError') self._assert_failed(response, bpd.ERROR_SYSTEM) + def test_empty_request(self): + with self.run_bpd() as client: + response = client.send_command('') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ From 7105c800aa3ba0da305110c77a44fd73606af9f4 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 8 Apr 2019 11:36:32 +1000 Subject: [PATCH 166/339] bpd: implement the idle command Getting this command puts the connection into a special mode where it awaits MPD events (like the player changing state or the playlist changing due to other clients interacting with the server. The MPD specification states that events should queue while a client is connected, and when it issues the `idle` command any matching events should be sent immediately if there are any, or as soon as they happen otherwise. --- beetsplug/bpd/__init__.py | 78 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 153fd675e..2cf0c22ce 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -27,6 +27,7 @@ import random import time import math import inspect +import socket import beets from beets.plugins import BeetsPlugin @@ -72,6 +73,10 @@ SAFE_COMMANDS = ( u'close', u'commands', u'notcommands', u'password', u'ping', ) +# List of subsystems/events used by the `idle` command. +SUBSYSTEMS = [ +] + ITEM_KEYS_WRITABLE = set(MediaFile.fields()).intersection(Item._fields.keys()) @@ -147,6 +152,16 @@ class BPDClose(Exception): should be closed. """ + +class BPDIdle(Exception): + """Raised by a command to indicate the client wants to enter the idle state + and should be notified when a relevant event happens. + """ + def __init__(self, subsystems): + super(BPDIdle, self).__init__() + self.subsystems = set(subsystems) + + # Generic server infrastructure, implementing the basic protocol. @@ -211,6 +226,11 @@ class BaseServer(object): bluelet.run(bluelet.server(self.host, self.port, Connection.handler(self))) + def _send_event(self, event): + """Notify subscribed connections of an event.""" + for conn in self.connections: + conn.notify(event) + def _item_info(self, item): """An abstract method that should response lines containing a single song's metadata. @@ -271,6 +291,14 @@ class BaseServer(object): """Succeeds.""" pass + def cmd_idle(self, conn, *subsystems): + subsystems = subsystems or SUBSYSTEMS + for system in subsystems: + if system not in SUBSYSTEMS: + raise BPDError(ERROR_ARG, + u'Unrecognised idle event: {}'.format(system)) + raise BPDIdle(subsystems) # put the connection to sleep + def cmd_kill(self, conn): """Exits the server process.""" exit(0) @@ -657,6 +685,7 @@ class Connection(object): self.sock = sock self.authenticated = False self.address = u'{}:{}'.format(*sock.sock.getpeername()) + self.notifications = set() def send(self, lines): """Send lines, which which is either a single string or an @@ -692,6 +721,50 @@ class Connection(object): self.server.disconnect(self) self.server._log.debug(u'*[{}]: disconnected', self.address) + def poll_notifications(self, subsystems): + """Sleep until we have some notifications from the subsystems given. + In order to promptly detect if the client has disconnected while + idling, try reading a single byte from the socket. According to the MPD + protocol the client can send the special command `noidle` to cancel + idle mode, otherwise we're expecting either a timeout or a zero-byte + reply. When we have notifications, send them to the client. + """ + while True: + mpd_events = self.notifications.intersection(subsystems) + if mpd_events: + break + current_timeout = self.sock.sock.gettimeout() + try: + self.sock.sock.settimeout(0.01) + data = self.sock.sock.recv(1) + if data: # Client sent data when it was meant to by idle. + line = yield self.sock.readline() + command = (data + line).rstrip() + if command == b'noidle': + self.server._log.debug( + u'<[{}]: noidle'.format(self.address)) + break + err = BPDError( + ERROR_UNKNOWN, + u'Got command while idle: {}'.format( + command.decode('utf-8'))) + yield self.send(err.response()) + return + else: # The socket has been closed. + return + except socket.timeout: # The socket is still alive. + pass + finally: + self.sock.sock.settimeout(current_timeout) + yield bluelet.sleep(0.02) + self.notifications = self.notifications.difference(subsystems) + for event in mpd_events: + yield self.send(u'changed: {}'.format(event)) + yield self.send(RESP_OK) + + def notify(self, event): + self.notifications.add(event) + def run(self): """Send a greeting to the client and begin processing commands as they arrive. @@ -738,6 +811,8 @@ class Connection(object): self.sock.close() self.disconnect() # Client explicitly closed. return + except BPDIdle as e: + yield bluelet.call(self.poll_notifications(e.subsystems)) @classmethod def handler(cls, server): @@ -832,6 +907,9 @@ class Command(object): # it on the Connection. raise + except BPDIdle: + raise + except Exception: # An "unintentional" error. Hide it from the client. conn.server._log.error('{}', traceback.format_exc()) From 699de94f4f7558a09cb33e0e11837b4468653026 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 8 Apr 2019 11:37:05 +1000 Subject: [PATCH 167/339] bpd: send all relevant idle events --- beetsplug/bpd/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 2cf0c22ce..664eaa4bb 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -75,6 +75,10 @@ SAFE_COMMANDS = ( # List of subsystems/events used by the `idle` command. SUBSYSTEMS = [ + u'update', u'player', u'mixer', u'options', u'playlist', u'database', + # Related to unsupported commands: + # u'stored_playlist', u'output', u'subscription', u'sticker', u'message', + # u'partition', ] ITEM_KEYS_WRITABLE = set(MediaFile.fields()).intersection(Item._fields.keys()) @@ -393,19 +397,23 @@ class BaseServer(object): def cmd_random(self, conn, state): """Set or unset random (shuffle) mode.""" self.random = cast_arg('intbool', state) + self._send_event('options') def cmd_repeat(self, conn, state): """Set or unset repeat mode.""" self.repeat = cast_arg('intbool', state) + self._send_event('options') def cmd_consume(self, conn, state): """Set or unset consume mode.""" self.consume = cast_arg('intbool', state) + self._send_event('options') def cmd_single(self, conn, state): """Set or unset single mode.""" # TODO support oneshot in addition to 0 and 1 [MPD 0.20] self.single = cast_arg('intbool', state) + self._send_event('options') def cmd_setvol(self, conn, vol): """Set the player's volume level (0-100).""" @@ -413,6 +421,7 @@ class BaseServer(object): if vol < VOLUME_MIN or vol > VOLUME_MAX: raise BPDError(ERROR_ARG, u'volume out of range') self.volume = vol + self._send_event('mixer') def cmd_volume(self, conn, vol_delta): """Deprecated command to change the volume by a relative amount.""" @@ -425,6 +434,7 @@ class BaseServer(object): raise BPDError(ERROR_ARG, u'crossfade time must be nonnegative') self._log.warning(u'crossfade is not implemented in bpd') self.crossfade = crossfade + self._send_event('options') def cmd_mixrampdb(self, conn, db): """Set the mixramp normalised max volume in dB.""" @@ -433,6 +443,7 @@ class BaseServer(object): raise BPDError(ERROR_ARG, u'mixrampdb time must be negative') self._log.warning('mixramp is not implemented in bpd') self.mixrampdb = db + self._send_event('options') def cmd_mixrampdelay(self, conn, delay): """Set the mixramp delay in seconds.""" @@ -441,6 +452,7 @@ class BaseServer(object): raise BPDError(ERROR_ARG, u'mixrampdelay time must be nonnegative') self._log.warning('mixramp is not implemented in bpd') self.mixrampdelay = delay + self._send_event('options') def cmd_replay_gain_mode(self, conn, mode): """Set the replay gain mode.""" @@ -448,6 +460,7 @@ class BaseServer(object): raise BPDError(ERROR_ARG, u'Unrecognised replay gain mode') self._log.warning('replay gain is not implemented in bpd') self.replay_gain_mode = mode + self._send_event('options') def cmd_replay_gain_status(self, conn): """Get the replaygain mode.""" @@ -458,6 +471,7 @@ class BaseServer(object): self.playlist = [] self.playlist_version += 1 self.cmd_stop(conn) + self._send_event('playlist') def cmd_delete(self, conn, index): """Remove the song at index from the playlist.""" @@ -473,6 +487,7 @@ class BaseServer(object): elif index < self.current_index: # Deleted before playing. # Shift playing index down. self.current_index -= 1 + self._send_event('playlist') def cmd_deleteid(self, conn, track_id): self.cmd_delete(conn, self._id_to_index(track_id)) @@ -496,6 +511,7 @@ class BaseServer(object): self.current_index += 1 self.playlist_version += 1 + self._send_event('playlist') def cmd_moveid(self, conn, idx_from, idx_to): idx_from = self._id_to_index(idx_from) @@ -521,6 +537,7 @@ class BaseServer(object): self.current_index = i self.playlist_version += 1 + self._send_event('playlist') def cmd_swapid(self, conn, i_id, j_id): i = self._id_to_index(i_id) @@ -578,6 +595,7 @@ class BaseServer(object): """Advance to the next song in the playlist.""" old_index = self.current_index self.current_index = self._succ_idx() + self._send_event('playlist') if self.consume: # TODO how does consume interact with single+repeat? self.playlist.pop(old_index) @@ -598,6 +616,7 @@ class BaseServer(object): """Step back to the last song.""" old_index = self.current_index self.current_index = self._prev_idx() + self._send_event('playlist') if self.consume: self.playlist.pop(old_index) if self.current_index < 0: @@ -613,6 +632,7 @@ class BaseServer(object): self.paused = not self.paused # Toggle. else: self.paused = cast_arg('intbool', state) + self._send_event('player') def cmd_play(self, conn, index=-1): """Begin playback, possibly at a specified playlist index.""" @@ -632,6 +652,7 @@ class BaseServer(object): self.current_index = index self.paused = False + self._send_event('player') def cmd_playid(self, conn, track_id=0): track_id = cast_arg(int, track_id) @@ -645,6 +666,7 @@ class BaseServer(object): """Stop playback.""" self.current_index = -1 self.paused = False + self._send_event('player') def cmd_seek(self, conn, index, pos): """Seek to a specified point in a specified song.""" @@ -652,6 +674,7 @@ class BaseServer(object): if index < 0 or index >= len(self.playlist): raise ArgumentIndexError() self.current_index = index + self._send_event('player') def cmd_seekid(self, conn, track_id, pos): index = self._id_to_index(track_id) @@ -1027,6 +1050,8 @@ class Server(BaseServer): self.tree = vfs.libtree(self.lib) self._log.debug(u'Finished building directory tree.') self.updated_time = time.time() + self._send_event('update') + self._send_event('database') # Path (directory tree) browsing. @@ -1136,6 +1161,7 @@ class Server(BaseServer): if send_id: yield u'Id: ' + six.text_type(item.id) self.playlist_version += 1 + self._send_event('playlist') def cmd_add(self, conn, path): """Adds a track or directory to the playlist, specified by a From d05ca2c2b081336d992d13efb461a121a672cc2b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 5 Apr 2019 11:18:42 +1100 Subject: [PATCH 168/339] bpd: add tests for idle command --- test/test_player.py | 58 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index baf1ddfdb..7a29610e3 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -23,6 +23,7 @@ from test.helper import TestHelper import os import sys import multiprocessing as mp +import threading import socket import time import yaml @@ -37,18 +38,19 @@ from beetsplug import bpd import mock import imp gstplayer = imp.new_module("beetsplug.bpd.gstplayer") -def _gstplayer_play(_): # noqa: 42 +def _gstplayer_play(*_): # noqa: 42 bpd.gstplayer._GstPlayer.playing = True return mock.DEFAULT gstplayer._GstPlayer = mock.MagicMock( spec_set=[ "time", "volume", "playing", "run", "play_file", "pause", "stop", - "seek" + "seek", "play" ], **{ 'playing': False, 'volume': 0, 'time.return_value': (0, 0), 'play_file.side_effect': _gstplayer_play, + 'play.side_effect': _gstplayer_play, }) gstplayer.GstPlayer = lambda _: gstplayer._GstPlayer sys.modules["beetsplug.bpd.gstplayer"] = gstplayer @@ -259,7 +261,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): @contextmanager def run_bpd(self, host='localhost', port=9876, password=None, - do_hello=True): + do_hello=True, second_client=False): """ Runs BPD in another process, configured with the same library database as we created in the setUp method. Exposes a client that is connected to the server, and kills the server at the end. @@ -290,7 +292,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): server.start() # Wait until the socket is connected: - sock = None + sock, sock2 = None, None for _ in range(20): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if sock.connect_ex((host, port)) == 0: @@ -302,9 +304,16 @@ class BPDTestHelper(unittest.TestCase, TestHelper): raise RuntimeError('Timed out waiting for the BPD server') try: - yield MPCClient(sock, do_hello) + if second_client: + sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock2.connect((host, port)) + yield MPCClient(sock, do_hello), MPCClient(sock2, do_hello) + else: + yield MPCClient(sock, do_hello) finally: sock.close() + if sock2: + sock2.close() server.terminate() server.join(timeout=0.2) @@ -375,8 +384,8 @@ class BPDTest(BPDTestHelper): class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ - 'clearerror', 'currentsong', 'idle', 'stats', - }, expectedFailure=True) + 'clearerror', 'currentsong', 'stats', + }) def test_cmd_status(self): with self.run_bpd() as client: @@ -397,6 +406,41 @@ class BPDQueryTest(BPDTestHelper): } self.assertEqual(fields_playing, set(responses[2].data.keys())) + def test_cmd_idle(self): + def _toggle(c): + for _ in range(3): + rs = c.send_commands(('play',), ('pause',)) + # time.sleep(0.05) # uncomment if test is flaky + if any(not r.ok for r in rs): + raise RuntimeError('Toggler failed') + with self.run_bpd(second_client=True) as (client, client2): + self._bpd_add(client, self.item1, self.item2) + toggler = threading.Thread(target=_toggle, args=(client2,)) + toggler.start() + # Idling will hang until the toggler thread changes the play state. + # Since the client sockets have a 1s timeout set at worst this will + # raise a socket.timeout and fail the test if the toggler thread + # manages to finish before the idle command is sent here. + response = client.send_command('idle', 'player') + toggler.join() + self._assert_ok(response) + + def test_cmd_idle_with_pending(self): + with self.run_bpd(second_client=True) as (client, client2): + response1 = client.send_command('random', '1') + response2 = client2.send_command('idle') + self._assert_ok(response1, response2) + self.assertEqual('options', response2.data['changed']) + + def test_cmd_noidle(self): + with self.run_bpd() as client: + # Manually send a command without reading a response. + request = client.serialise_command('idle') + client.sock.sendall(request) + time.sleep(0.01) + response = client.send_command('noidle') + self._assert_ok(response) + class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ From 6fbf3853f27e736f009343ff21f1251eaeaabc61 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 5 Apr 2019 11:20:00 +1100 Subject: [PATCH 169/339] bpd: bump protocol version to 0.14.0 --- beetsplug/bpd/__init__.py | 2 +- test/test_player.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 664eaa4bb..fdd78a11f 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -39,7 +39,7 @@ from beets import dbcore from beets.mediafile import MediaFile import six -PROTOCOL_VERSION = '0.13.0' +PROTOCOL_VERSION = '0.14.0' BUFSIZE = 1024 HELLO = u'OK MPD %s' % PROTOCOL_VERSION diff --git a/test/test_player.py b/test/test_player.py index 7a29610e3..3fd5910b4 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -354,7 +354,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): class BPDTest(BPDTestHelper): def test_server_hello(self): with self.run_bpd(do_hello=False) as client: - self.assertEqual(client.readline(), b'OK MPD 0.13.0\n') + self.assertEqual(client.readline(), b'OK MPD 0.14.0\n') def test_unknown_cmd(self): with self.run_bpd() as client: From 275301750a169d4f04981de70d822a961a6aed5a Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 5 Apr 2019 20:07:11 +1100 Subject: [PATCH 170/339] Changelog for #3205 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 43b6b20f6..e41d71bd2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -86,6 +86,9 @@ New features: new fields for ``status``. The bpd server now understands and ignores some additional commands. :bug:`3200` :bug:`800` +* :doc:`/plugins/bpd`: MPD protocol command ``idle`` is now supported, allowing + the MPD version to be bumped to 0.14. + :bug:`3205` :bug:`800` Changes: From e70b2134e4537d63a502d63c58e44c013769fd17 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 6 Apr 2019 13:34:06 +1100 Subject: [PATCH 171/339] bpd: update documentation --- docs/plugins/bpd.rst | 64 +++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index fc22846de..7757ba893 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -95,40 +95,42 @@ on-disk directory structure can. (Note that an obvious solution to this is just string matching on items' destination, but this requires examining the entire library Python-side for every query.) -We don't currently support versioned playlists. Many clients, however, use -plchanges instead of playlistinfo to get the current playlist, so plchanges -contains a dummy implementation that just calls playlistinfo. +BPD plays music using GStreamer's ``playbin`` player, which has a simple API +but doesn't support many advanced playback features. -The ``stats`` command always send zero for ``playtime``, which is supposed to -indicate the amount of time the server has spent playing music. BPD doesn't -currently keep track of this. +Differences from the real MPD +----------------------------- -The ``update`` command regenerates the directory tree from the beets database. - -Unimplemented Commands ----------------------- - -These are the commands from `the MPD protocol`_ that have not yet been -implemented in BPD. +BPD currently supports version 0.14 of `the MPD protocol`_, but several of the +commands and features are "pretend" implementations or have slightly different +behaviour to their MPD equivalents. BPD aims to look enough like MPD that it +can interact with the ecosystem of clients, but doesn't try to be +a fully-fledged MPD replacement in terms of its playback capabilities. .. _the MPD protocol: http://www.musicpd.org/doc/protocol/ -Saved playlists: +These are some of the known differences between BPD and MPD: -* playlistclear -* playlistdelete -* playlistmove -* playlistadd -* playlistsearch -* listplaylist -* listplaylistinfo -* playlistfind -* rm -* save -* load -* rename - -Deprecated: - -* playlist -* volume +* BPD doesn't currently support versioned playlists. Many clients, however, use + plchanges instead of playlistinfo to get the current playlist, so plchanges + contains a dummy implementation that just calls playlistinfo. +* Stored playlists aren't supported (BPD understands the commands though). +* The ``stats`` command always send zero for ``playtime``, which is supposed to + indicate the amount of time the server has spent playing music. BPD doesn't + currently keep track of this. +* The ``update`` command regenerates the directory tree from the beets database + synchronously, whereas MPD does this in the background. +* Advanced playback features like cross-fade, ReplayGain and MixRamp are not + supported due to BPD's simple audio player backend. +* Advanced query syntax is not currently supported. +* Not all tags (fields) are currently exposed to BPD. Clients also can't use + the ``tagtypes`` mask to hide fields. +* BPD's ``random`` mode is not deterministic and doesn't support priorities. +* Mounts and streams are not supported. BPD can only play files from disk. +* Stickers are not supported (although this is basically a flexattr in beets + nomenclature so this is feasible to add). +* There is only a single password, and is enabled it grants access to all + features rather than having permissions-based granularity. +* Partitions and alternative outputs are not supported; BPD can only play one + song at a time. +* Client channels are not implemented. From fa3813844c65199554756e03ec029614e3af5055 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 8 Apr 2019 13:03:44 +1000 Subject: [PATCH 172/339] bpd: reimplement idle without polling --- beetsplug/bpd/__init__.py | 85 +++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index fdd78a11f..229078e59 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -27,7 +27,6 @@ import random import time import math import inspect -import socket import beets from beets.plugins import BeetsPlugin @@ -230,6 +229,14 @@ class BaseServer(object): bluelet.run(bluelet.server(self.host, self.port, Connection.handler(self))) + def dispatch_events(self): + """If any clients have idle events ready, send them. + """ + # We need a copy of `self.connections` here since clients might + # disconnect once we try and send to them, changing `self.connections`. + for conn in list(self.connections): + yield bluelet.spawn(conn.send_notifications()) + def _send_event(self, event): """Notify subscribed connections of an event.""" for conn in self.connections: @@ -301,7 +308,7 @@ class BaseServer(object): if system not in SUBSYSTEMS: raise BPDError(ERROR_ARG, u'Unrecognised idle event: {}'.format(system)) - raise BPDIdle(subsystems) # put the connection to sleep + raise BPDIdle(subsystems) # put the connection into idle mode def cmd_kill(self, conn): """Exits the server process.""" @@ -709,6 +716,7 @@ class Connection(object): self.authenticated = False self.address = u'{}:{}'.format(*sock.sock.getpeername()) self.notifications = set() + self.idle_subscriptions = set() def send(self, lines): """Send lines, which which is either a single string or an @@ -744,50 +752,25 @@ class Connection(object): self.server.disconnect(self) self.server._log.debug(u'*[{}]: disconnected', self.address) - def poll_notifications(self, subsystems): - """Sleep until we have some notifications from the subsystems given. - In order to promptly detect if the client has disconnected while - idling, try reading a single byte from the socket. According to the MPD - protocol the client can send the special command `noidle` to cancel - idle mode, otherwise we're expecting either a timeout or a zero-byte - reply. When we have notifications, send them to the client. - """ - while True: - mpd_events = self.notifications.intersection(subsystems) - if mpd_events: - break - current_timeout = self.sock.sock.gettimeout() - try: - self.sock.sock.settimeout(0.01) - data = self.sock.sock.recv(1) - if data: # Client sent data when it was meant to by idle. - line = yield self.sock.readline() - command = (data + line).rstrip() - if command == b'noidle': - self.server._log.debug( - u'<[{}]: noidle'.format(self.address)) - break - err = BPDError( - ERROR_UNKNOWN, - u'Got command while idle: {}'.format( - command.decode('utf-8'))) - yield self.send(err.response()) - return - else: # The socket has been closed. - return - except socket.timeout: # The socket is still alive. - pass - finally: - self.sock.sock.settimeout(current_timeout) - yield bluelet.sleep(0.02) - self.notifications = self.notifications.difference(subsystems) - for event in mpd_events: - yield self.send(u'changed: {}'.format(event)) - yield self.send(RESP_OK) - def notify(self, event): + """Queue up an event for sending to this client. + """ self.notifications.add(event) + def send_notifications(self, force_close_idle=False): + """Send the client any queued events now. + """ + pending = self.notifications.intersection(self.idle_subscriptions) + try: + for event in pending: + yield self.send(u'changed: {}'.format(event)) + if pending or force_close_idle: + self.idle_subscriptions = set() + self.notifications = self.notifications.difference(pending) + yield self.send(RESP_OK) + except bluelet.SocketClosedError: + self.disconnect() # Client disappeared. + def run(self): """Send a greeting to the client and begin processing commands as they arrive. @@ -813,6 +796,17 @@ class Connection(object): for l in line.split(NEWLINE): self.server._log.debug('{}', session + l) + if self.idle_subscriptions: + # The connection is in idle mode. + if line == u'noidle': + yield bluelet.call(self.send_notifications(True)) + else: + err = BPDError(ERROR_UNKNOWN, + u'Got command while idle: {}'.format(line)) + yield self.send(err.response()) + break + continue + if clist is not None: # Command list already opened. if line == CLIST_END: @@ -835,7 +829,10 @@ class Connection(object): self.disconnect() # Client explicitly closed. return except BPDIdle as e: - yield bluelet.call(self.poll_notifications(e.subsystems)) + self.idle_subscriptions = e.subsystems + self.server._log.debug('z[{}]: awaiting: {}', self.address, + ' '.join(e.subsystems)) + yield bluelet.call(self.server.dispatch_events()) @classmethod def handler(cls, server): From d55f061f0b89919b6b641524f2cdf4571c16238f Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 9 Apr 2019 11:40:49 +1000 Subject: [PATCH 173/339] bpd: add control socket A new `ControlConnection` is created each time a client connects over a new control socket. This is used to forward events from the player, and also for debugging utilities that are not part of the real MPD protocol. This new feature reuses as much infrastructure from the normal protocol handling as possible (e.g. `Command` for parsing messages). While the normal connection delegates to server `cmd_*` methods which are string generators, the control connections delegate to `ctrl_*` methods defined on the connection itself that are full coroutines. --- beetsplug/bpd/__init__.py | 193 ++++++++++++++++++++++++++++---------- test/test_player.py | 2 +- 2 files changed, 146 insertions(+), 49 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 229078e59..5ebee7e46 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -27,6 +27,7 @@ import random import time import math import inspect +import socket import beets from beets.plugins import BeetsPlugin @@ -181,12 +182,14 @@ class BaseServer(object): This is a generic superclass and doesn't support many commands. """ - def __init__(self, host, port, password, log): + def __init__(self, host, port, password, ctrl_port, log, ctrl_host=None): """Create a new server bound to address `host` and listening on port `port`. If `password` is given, it is required to do anything significant on the server. """ self.host, self.port, self.password = host, port, password + self.ctrl_host, self.ctrl_port = ctrl_host or host, ctrl_port + self.ctrl_sock = None self._log = log # Default server values. @@ -226,8 +229,14 @@ class BaseServer(object): interrupt (^C) closes the server. """ self.startup_time = time.time() - bluelet.run(bluelet.server(self.host, self.port, - Connection.handler(self))) + + def start(): + yield bluelet.spawn( + bluelet.server(self.ctrl_host, self.ctrl_port, + ControlConnection.handler(self))) + yield bluelet.server(self.host, self.port, + MPDConnection.handler(self)) + bluelet.run(start()) def dispatch_events(self): """If any clients have idle events ready, send them. @@ -689,12 +698,6 @@ class BaseServer(object): # Debugging/testing commands that are not part of the MPD protocol. - def cmd_profile(self, conn): - """Memory profiling for debugging.""" - from guppy import hpy - heap = hpy().heap() - print(heap) - def cmd_crash_TypeError(self, conn): # noqa: N802 """Deliberately trigger a TypeError for testing purposes. We want to test that the server properly responds with ERROR_SYSTEM @@ -705,18 +708,21 @@ class BaseServer(object): class Connection(object): - """A connection between a client and the server. Handles input and - output from and to the client. + """A connection between a client and the server. + """ def __init__(self, server, sock): """Create a new connection for the accepted socket `client`. """ self.server = server self.sock = sock - self.authenticated = False self.address = u'{}:{}'.format(*sock.sock.getpeername()) - self.notifications = set() - self.idle_subscriptions = set() + + def debug(self, message, kind=' '): + self.server._log.debug(u'{}[{}]: {}', kind, self.address, message) + + def run(self): + pass def send(self, lines): """Send lines, which which is either a single string or an @@ -727,13 +733,32 @@ class Connection(object): if isinstance(lines, six.string_types): lines = [lines] out = NEWLINE.join(lines) + NEWLINE - session = u'>[{}]: '.format(self.address) for l in out.split(NEWLINE)[:-1]: - self.server._log.debug('{}', session + l) + self.debug(l, kind='>') if isinstance(out, six.text_type): out = out.encode('utf-8') return self.sock.sendall(out) + @classmethod + def handler(cls, server): + def _handle(sock): + """Creates a new `Connection` and runs it. + """ + return cls(server, sock).run() + return _handle + + +class MPDConnection(Connection): + """A connection that receives commands from an MPD-compatible client. + """ + def __init__(self, server, sock): + """Create a new connection for the accepted socket `client`. + """ + super(MPDConnection, self).__init__(server, sock) + self.authenticated = False + self.notifications = set() + self.idle_subscriptions = set() + def do_command(self, command): """A coroutine that runs the given command and sends an appropriate response.""" @@ -750,7 +775,7 @@ class Connection(object): """The connection has closed for any reason. """ self.server.disconnect(self) - self.server._log.debug(u'*[{}]: disconnected', self.address) + self.debug('disconnected', kind='*') def notify(self, event): """Queue up an event for sending to this client. @@ -775,11 +800,10 @@ class Connection(object): """Send a greeting to the client and begin processing commands as they arrive. """ - self.server._log.debug(u'*[{}]: connected', self.address) + self.debug('connected', kind='*') self.server.connect(self) yield self.send(HELLO) - session = u'<[{}]: '.format(self.address) clist = None # Initially, no command list is being constructed. while True: line = yield self.sock.readline() @@ -794,7 +818,7 @@ class Connection(object): break line = line.decode('utf8') # MPD protocol uses UTF-8. for l in line.split(NEWLINE): - self.server._log.debug('{}', session + l) + self.debug(l, kind='<') if self.idle_subscriptions: # The connection is in idle mode. @@ -830,17 +854,67 @@ class Connection(object): return except BPDIdle as e: self.idle_subscriptions = e.subsystems - self.server._log.debug('z[{}]: awaiting: {}', self.address, - ' '.join(e.subsystems)) + self.debug('awaiting: {}'.format(' '.join(e.subsystems)), + kind='z') yield bluelet.call(self.server.dispatch_events()) - @classmethod - def handler(cls, server): - def _handle(sock): - """Creates a new `Connection` and runs it. - """ - return cls(server, sock).run() - return _handle + +class ControlConnection(Connection): + """A connection used to control BPD for debugging and internal events. + """ + def __init__(self, server, sock): + """Create a new connection for the accepted socket `client`. + """ + super(ControlConnection, self).__init__(server, sock) + + def debug(self, message, kind=' '): + self.server._log.debug(u'CTRL {}[{}]: {}', kind, self.address, message) + + def run(self): + """Listen for control commands and delegate to `ctrl_*` methods. + """ + self.debug('connected', kind='*') + while True: + line = yield self.sock.readline() + if not line: + break # Client disappeared. + line = line.strip() + if not line: + break # Client sent a blank line. + line = line.decode('utf8') # Protocol uses UTF-8. + for l in line.split(NEWLINE): + self.debug(l, kind='<') + command = Command(line) + try: + func = command.delegate('ctrl_', self) + yield bluelet.call(func(*command.args)) + except (AttributeError, TypeError) as e: + yield self.send('ERROR: {}'.format(e.args[0])) + except Exception: + yield self.send(['ERROR: server error', + traceback.format_exc().rstrip()]) + + def ctrl_play_finished(self): + """Callback from the player signalling a song finished playing. + """ + yield bluelet.call(self.server.dispatch_events()) + + def ctrl_profile(self): + """Memory profiling for debugging. + """ + from guppy import hpy + heap = hpy().heap() + yield self.send(heap) + + def ctrl_nickname(self, oldlabel, newlabel): + """Rename a client in the log messages. + """ + for c in self.server.connections: + if c.address == oldlabel: + c.address = newlabel + break + else: + yield self.send(u'ERROR: no such client: {}'.format(oldlabel)) class Command(object): @@ -869,16 +943,17 @@ class Command(object): arg = match[1] self.args.append(arg) - def run(self, conn): - """A coroutine that executes the command on the given - connection. + def delegate(self, prefix, target, extra_args=0): + """Get the target method that corresponds to this command. + The `prefix` is prepended to the command name and then the resulting + name is used to search `target` for a method with a compatible number + of arguments. """ # Attempt to get correct command function. - func_name = 'cmd_' + self.name - if not hasattr(conn.server, func_name): - raise BPDError(ERROR_UNKNOWN, - u'unknown command "{}"'.format(self.name)) - func = getattr(conn.server, func_name) + func_name = prefix + self.name + if not hasattr(target, func_name): + raise AttributeError(u'unknown command "{}"'.format(self.name)) + func = getattr(target, func_name) if six.PY2: # caution: the fields of the namedtuple are slightly different @@ -889,8 +964,8 @@ class Command(object): # Check that `func` is able to handle the number of arguments sent # by the client (so we can raise ERROR_ARG instead of ERROR_SYSTEM). - # Maximum accepted arguments: argspec includes "self" and "conn". - max_args = len(argspec.args) - 2 + # Maximum accepted arguments: argspec includes "self". + max_args = len(argspec.args) - 1 - extra_args # Minimum accepted arguments: some arguments might be optional. min_args = max_args if argspec.defaults: @@ -898,10 +973,22 @@ class Command(object): wrong_num = (len(self.args) > max_args) or (len(self.args) < min_args) # If the command accepts a variable number of arguments skip the check. if wrong_num and not argspec.varargs: - raise BPDError(ERROR_ARG, - u'wrong number of arguments for "{}"' - .format(self.name), - self.name) + raise TypeError(u'wrong number of arguments for "{}"' + .format(self.name), self.name) + + return func + + def run(self, conn): + """A coroutine that executes the command on the given + connection. + """ + try: + # `conn` is an extra argument to all cmd handlers. + func = self.delegate('cmd_', conn.server, extra_args=1) + except AttributeError as e: + raise BPDError(ERROR_UNKNOWN, e.args[0]) + except TypeError as e: + raise BPDError(ERROR_ARG, e.args[0], self.name) # Ensure we have permission for this command. if conn.server.password and \ @@ -976,7 +1063,7 @@ class Server(BaseServer): to store its library. """ - def __init__(self, library, host, port, password, log): + def __init__(self, library, host, port, password, ctrl_port, log): try: from beetsplug.bpd import gstplayer except ImportError as e: @@ -986,7 +1073,7 @@ class Server(BaseServer): else: raise log.info(u'Starting server...') - super(Server, self).__init__(host, port, password, log) + super(Server, self).__init__(host, port, password, ctrl_port, log) self.lib = library self.player = gstplayer.GstPlayer(self.play_finished) self.cmd_update(None) @@ -1001,7 +1088,11 @@ class Server(BaseServer): """A callback invoked every time our player finishes a track. """ + if not self.ctrl_sock: + self.ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.ctrl_sock.connect((self.ctrl_host, self.ctrl_port)) self.cmd_next(None) + self.ctrl_sock.sendall(u'play_finished\n'.encode('utf-8')) # Metadata helper functions. @@ -1439,15 +1530,16 @@ class BPDPlugin(BeetsPlugin): self.config.add({ 'host': u'', 'port': 6600, + 'control_port': 6601, 'password': u'', 'volume': VOLUME_MAX, }) self.config['password'].redact = True - def start_bpd(self, lib, host, port, password, volume): + def start_bpd(self, lib, host, port, password, volume, ctrl_port): """Starts a BPD server.""" try: - server = Server(lib, host, port, password, self._log) + server = Server(lib, host, port, password, ctrl_port, self._log) server.cmd_setvol(None, volume) server.run() except NoGstreamerError: @@ -1464,11 +1556,16 @@ class BPDPlugin(BeetsPlugin): host = self.config['host'].as_str() host = args.pop(0) if args else host port = args.pop(0) if args else self.config['port'].get(int) + if args: + ctrl_port = args.pop(0) + else: + ctrl_port = self.config['control_port'].get(int) if args: raise beets.ui.UserError(u'too many arguments') password = self.config['password'].as_str() volume = self.config['volume'].get(int) - self.start_bpd(lib, host, int(port), password, volume) + self.start_bpd(lib, host, int(port), password, volume, + int(ctrl_port)) cmd.func = func return [cmd] diff --git a/test/test_player.py b/test/test_player.py index 3fd5910b4..6cc0869a7 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -270,7 +270,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): config = { 'pluginpath': [py3_path(self.temp_dir)], 'plugins': 'bpd', - 'bpd': {'host': host, 'port': port}, + 'bpd': {'host': host, 'port': port, 'control_port': port + 1}, } if password: config['bpd']['password'] = password From 826244777ee0d5f1143ffd9aeefd7ca605789a4d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 10 Apr 2019 15:52:47 +1000 Subject: [PATCH 174/339] bpd: minor control socket refactor --- beetsplug/bpd/__init__.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 5ebee7e46..11fec0890 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -186,6 +186,9 @@ class BaseServer(object): """Create a new server bound to address `host` and listening on port `port`. If `password` is given, it is required to do anything significant on the server. + A separate control socket is established listening to `ctrl_host` on + port `ctrl_port` which is used to forward notifications from the player + and can be sent debug commands (e.g. using netcat). """ self.host, self.port, self.password = host, port, password self.ctrl_host, self.ctrl_port = ctrl_host or host, ctrl_port @@ -246,6 +249,16 @@ class BaseServer(object): for conn in list(self.connections): yield bluelet.spawn(conn.send_notifications()) + def _ctrl_send(self, message): + """Send some data over the control socket. + If it's our first time, open the socket. The message should be a + string without a terminal newline. + """ + if not self.ctrl_sock: + self.ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.ctrl_sock.connect((self.ctrl_host, self.ctrl_port)) + self.ctrl_sock.sendall((message + u'\n').encode('utf-8')) + def _send_event(self, event): """Notify subscribed connections of an event.""" for conn in self.connections: @@ -696,7 +709,7 @@ class BaseServer(object): index = self._id_to_index(track_id) return self.cmd_seek(conn, index, pos) - # Debugging/testing commands that are not part of the MPD protocol. + # Additions to the MPD protocol. def cmd_crash_TypeError(self, conn): # noqa: N802 """Deliberately trigger a TypeError for testing purposes. @@ -709,7 +722,6 @@ class BaseServer(object): class Connection(object): """A connection between a client and the server. - """ def __init__(self, server, sock): """Create a new connection for the accepted socket `client`. @@ -719,6 +731,8 @@ class Connection(object): self.address = u'{}:{}'.format(*sock.sock.getpeername()) def debug(self, message, kind=' '): + """Log a debug message about this connection. + """ self.server._log.debug(u'{}[{}]: {}', kind, self.address, message) def run(self): @@ -836,6 +850,7 @@ class MPDConnection(Connection): if line == CLIST_END: yield bluelet.call(self.do_command(clist)) clist = None # Clear the command list. + yield bluelet.call(self.server.dispatch_events()) else: clist.append(Command(line)) @@ -856,7 +871,7 @@ class MPDConnection(Connection): self.idle_subscriptions = e.subsystems self.debug('awaiting: {}'.format(' '.join(e.subsystems)), kind='z') - yield bluelet.call(self.server.dispatch_events()) + yield bluelet.call(self.server.dispatch_events()) class ControlConnection(Connection): @@ -1085,14 +1100,10 @@ class Server(BaseServer): super(Server, self).run() def play_finished(self): - """A callback invoked every time our player finishes a - track. + """A callback invoked every time our player finishes a track. """ - if not self.ctrl_sock: - self.ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.ctrl_sock.connect((self.ctrl_host, self.ctrl_port)) self.cmd_next(None) - self.ctrl_sock.sendall(u'play_finished\n'.encode('utf-8')) + self._ctrl_send(u'play_finished') # Metadata helper functions. From 241e23eae8945eb45069b5b3587509ede4dd3f60 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 10 Apr 2019 15:54:41 +1000 Subject: [PATCH 175/339] bpd: document new control_port config --- docs/plugins/bpd.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index 7757ba893..ee36c040c 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -75,6 +75,8 @@ The available options are: Default: No password. - **volume**: Initial volume, as a percentage. Default: 100 +- **control_port**: Port for the internal control socket. + Default: 6601 Here's an example:: From 9182f18e6f98ff0bfd5e15d4ba92d4b68be36beb Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 16:25:09 +1000 Subject: [PATCH 176/339] bpd: support short form of list command for albums Some clients list the albums belonging to an artist by issuing the command `list album `. This change inserts the tag `artist` before the artist name so that this succeeds. Fixes #3007 --- beetsplug/bpd/__init__.py | 15 +++++++++++++++ test/test_player.py | 14 ++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 11fec0890..a70c6ecd6 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1404,16 +1404,31 @@ class Server(BaseServer): filtered by matching match_tag to match_term. """ show_tag_canon, show_key = self._tagtype_lookup(show_tag) + if len(kv) == 1: + if show_tag_canon == 'Album': + # If no tag was given, assume artist. This is because MPD + # supports a short version of this command for fetching the + # albums belonging to a particular artist, and some clients + # rely on this behaviour (e.g. MPDroid, M.A.L.P.). + kv = ('Artist', kv[0]) + else: + raise BPDError(ERROR_ARG, u'should be "Album" for 3 arguments') + elif len(kv) % 2 != 0: + raise BPDError(ERROR_ARG, u'Incorrect number of filter arguments') query = self._metadata_query(dbcore.query.MatchQuery, None, kv) clause, subvals = query.clause() statement = 'SELECT DISTINCT ' + show_key + \ ' FROM items WHERE ' + clause + \ ' ORDER BY ' + show_key + self._log.debug(statement) with self.lib.transaction() as tx: rows = tx.query(statement, subvals) for row in rows: + if not row[0]: + # Skip any empty values of the field. + continue yield show_tag_canon + u': ' + six.text_type(row[0]) def cmd_count(self, conn, tag, value): diff --git a/test/test_player.py b/test/test_player.py index 6cc0869a7..d7c9dba17 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -774,11 +774,21 @@ class BPDDatabaseTest(BPDTestHelper): with self.run_bpd() as client: responses = client.send_commands( ('list', 'album'), - ('list', 'track')) - self._assert_ok(*responses) + ('list', 'track'), + ('list', 'album', 'artist', 'Artist Name', 'track')) + self._assert_failed(responses, bpd.ERROR_ARG, pos=2) self.assertEqual('Album Title', responses[0].data['Album']) self.assertEqual(['1', '2'], responses[1].data['Track']) + def test_cmd_list_three_arg_form(self): + with self.run_bpd() as client: + responses = client.send_commands( + ('list', 'album', 'artist', 'Artist Name'), + ('list', 'album', 'Artist Name'), + ('list', 'track', 'Artist Name')) + self._assert_failed(responses, bpd.ERROR_ARG, pos=2) + self.assertEqual(responses[0].data, responses[1].data) + def test_cmd_lsinfo(self): with self.run_bpd() as client: response1 = client.send_command('lsinfo') From 7ddde2a10c05add1f9fce8583727dbfc5174e6ff Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 16:53:11 +1000 Subject: [PATCH 177/339] Changelog for #3215 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e41d71bd2..4ad0f057a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -200,6 +200,9 @@ Fixes: :bug:`3207` :bug:`2752` * Fix an unhandled exception when pruning empty directories. :bug:`1996` :bug:`3209` +* :doc:`/plugins/bpd`: Fix a crash triggered when certain clients tried to list + the albums belonging to a particular artist. + :bug:`3007` :bug:`3215` .. _python-itunes: https://github.com/ocelma/python-itunes From fc95fb86a10b12ddeda368c05c2de11627bb1909 Mon Sep 17 00:00:00 2001 From: rain0r Date: Wed, 17 Apr 2019 20:57:58 +0200 Subject: [PATCH 178/339] Update __init__.py Also fetch genres for single tracks via query. --- beetsplug/lastgenre/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index 3fd473db3..a340c1e19 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -402,7 +402,12 @@ class LastGenrePlugin(plugins.BeetsPlugin): if write: item.try_write() - + + for item in lib.items(ui.decargs(args)): + item.genre, src = self._get_genre(item) + self._log.debug(u'added last.fm item genre ({0}): {1}', src, + item.genre) + item.store() lastgenre_cmd.func = lastgenre_func return [lastgenre_cmd] From 0b2334f8e8fc1e5b4f0528546e6b7d9a63d8c7f8 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Thu, 18 Apr 2019 22:31:49 +0200 Subject: [PATCH 179/339] Added command line options to query explicit for albums and / or tracks. --- beetsplug/lastgenre/__init__.py | 53 ++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index a340c1e19..a26b1fd82 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -380,34 +380,45 @@ class LastGenrePlugin(plugins.BeetsPlugin): u'-s', u'--source', dest='source', type='string', help=u'genre source: artist, album, or track' ) + lastgenre_cmd.parser.add_option( + u'-A', u'--tracks', action='store_true', default=False, + help=u'match tracks instead of albums' + ) + lastgenre_cmd.parser.add_option( + u'-a', u'--albums', action='store_true', default=True, + help=u'match albums instead of tracks' + ) def lastgenre_func(lib, opts, args): write = ui.should_write() self.config.set_args(opts) - for album in lib.albums(ui.decargs(args)): - album.genre, src = self._get_genre(album) - self._log.info(u'genre for album {0} ({1}): {0.genre}', - album, src) - album.store() + if opts.albums: + for album in lib.albums(ui.decargs(args)): + album.genre, src = self._get_genre(album) + self._log.info(u'genre for album {0} ({1}): {0.genre}', + album, src) + album.store() - for item in album.items(): - # If we're using track-level sources, also look up each - # track on the album. - if 'track' in self.sources: - item.genre, src = self._get_genre(item) - item.store() - self._log.info(u'genre for track {0} ({1}): {0.genre}', - item, src) + for item in album.items(): + # If we're using track-level sources, also look up each + # track on the album. + if 'track' in self.sources: + item.genre, src = self._get_genre(item) + item.store() + self._log.info(u'genre for track {0} ({1}): {0.genre}', + item, src) + + if write: + item.try_write() + + if opts.tracks: + for item in lib.items(ui.decargs(args)): + item.genre, src = self._get_genre(item) + self._log.debug(u'added last.fm item genre ({0}): {1}', src, + item.genre) + item.store() - if write: - item.try_write() - - for item in lib.items(ui.decargs(args)): - item.genre, src = self._get_genre(item) - self._log.debug(u'added last.fm item genre ({0}): {1}', src, - item.genre) - item.store() lastgenre_cmd.func = lastgenre_func return [lastgenre_cmd] From 3da23167ca4aef838a7ffcab6b2036e03c8a2da2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 15:57:15 +1000 Subject: [PATCH 180/339] bpd: support decoders command This uses GStreamer APIs to extract a list of audio decoders and the relevant MIME types and file extensions. Some clients like ncmpcpp use this command to fetch a list of supported file extensions. --- beetsplug/bpd/__init__.py | 10 +++++++ beetsplug/bpd/gstplayer.py | 53 ++++++++++++++++++++++++++++++++++++++ test/test_player.py | 16 +++++++++--- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index a70c6ecd6..5c54c9eab 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1321,6 +1321,16 @@ class Server(BaseServer): u'db_update: ' + six.text_type(int(self.updated_time)), ) + def cmd_decoders(self, conn): + """Send list of supported decoders and formats.""" + decoders = self.player.get_decoders() + for name, (mimes, exts) in decoders.items(): + yield u'plugin: {}'.format(name) + for ext in exts: + yield u'suffix: {}'.format(ext) + for mime in mimes: + yield u'mime_type: {}'.format(mime) + # Searching. tagtype_map = { diff --git a/beetsplug/bpd/gstplayer.py b/beetsplug/bpd/gstplayer.py index fffa8a6ed..8d4e7c9ff 100644 --- a/beetsplug/bpd/gstplayer.py +++ b/beetsplug/bpd/gstplayer.py @@ -215,6 +215,59 @@ class GstPlayer(object): while self.playing: time.sleep(1) + def get_decoders(self): + return get_decoders() + + +def get_decoders(): + """Get supported audio decoders from GStreamer. + Returns a dict mapping decoder element names to the associated media types + and file extensions. + """ + # We only care about audio decoder elements. + filt = (Gst.ELEMENT_FACTORY_TYPE_DEPAYLOADER | + Gst.ELEMENT_FACTORY_TYPE_DEMUXER | + Gst.ELEMENT_FACTORY_TYPE_PARSER | + Gst.ELEMENT_FACTORY_TYPE_DECODER | + Gst.ELEMENT_FACTORY_TYPE_MEDIA_AUDIO) + + decoders = {} + mime_types = set() + for f in Gst.ElementFactory.list_get_elements(filt, Gst.Rank.NONE): + for pad in f.get_static_pad_templates(): + if pad.direction == Gst.PadDirection.SINK: + caps = pad.static_caps.get() + mimes = set() + for i in range(caps.get_size()): + struct = caps.get_structure(i) + mime = struct.get_name() + if mime == 'unknown/unknown': + continue + mimes.add(mime) + mime_types.add(mime) + if mimes: + decoders[f.get_name()] = (mimes, set()) + + # Check all the TypeFindFactory plugin features form the registry. If they + # are associated with an audio media type that we found above, get the list + # of corresponding file extensions. + mime_extensions = {mime: set() for mime in mime_types} + for feat in Gst.Registry.get().get_feature_list(Gst.TypeFindFactory): + caps = feat.get_caps() + if caps: + for i in range(caps.get_size()): + struct = caps.get_structure(i) + mime = struct.get_name() + if mime in mime_types: + mime_extensions[mime].update(feat.get_extensions()) + + # Fill in the slot we left for file extensions. + for name, (mimes, exts) in decoders.items(): + for mime in mimes: + exts.update(mime_extensions[mime]) + + return decoders + def play_simple(paths): """Play the files in paths in a straightforward way, without diff --git a/test/test_player.py b/test/test_player.py index d7c9dba17..102df1d7d 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -44,13 +44,14 @@ def _gstplayer_play(*_): # noqa: 42 gstplayer._GstPlayer = mock.MagicMock( spec_set=[ "time", "volume", "playing", "run", "play_file", "pause", "stop", - "seek", "play" + "seek", "play", "get_decoders", ], **{ 'playing': False, 'volume': 0, 'time.return_value': (0, 0), 'play_file.side_effect': _gstplayer_play, 'play.side_effect': _gstplayer_play, + 'get_decoders.return_value': {'default': ({'audio/mpeg'}, {'mp3'})}, }) gstplayer.GstPlayer = lambda _: gstplayer._GstPlayer sys.modules["beetsplug.bpd.gstplayer"] = gstplayer @@ -879,9 +880,16 @@ class BPDDeviceTest(BPDTestHelper): class BPDReflectionTest(BPDTestHelper): test_implements_reflection = implements({ - 'config', 'commands', 'notcommands', 'urlhandlers', - 'decoders', - }, expectedFailure=True) + 'config', 'commands', 'notcommands', 'urlhandlers', + }, expectedFailure=True) + + def test_cmd_decoders(self): + with self.run_bpd() as client: + response = client.send_command('decoders') + self._assert_ok(response) + self.assertEqual('default', response.data['plugin']) + self.assertEqual('mp3', response.data['suffix']) + self.assertEqual('audio/mpeg', response.data['mime_type']) class BPDPeersTest(BPDTestHelper): From c9327511f6df151ff93f4776a16e72212c16b5ba Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 16:21:52 +1000 Subject: [PATCH 181/339] Changelog for #3222 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ad0f057a..4b238c3ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -89,6 +89,8 @@ New features: * :doc:`/plugins/bpd`: MPD protocol command ``idle`` is now supported, allowing the MPD version to be bumped to 0.14. :bug:`3205` :bug:`800` +* :doc:`/plugins/bpd`: MPD protocol command ``decoders`` is now supported. + :bug:`3222` Changes: From dc5f110844c3977963d4d384b54e5a3be2c43bc2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 17:27:52 +1000 Subject: [PATCH 182/339] mpdstats: use MPD_PORT env variable --- beetsplug/mpdstats.py | 2 +- docs/plugins/mpdstats.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 876dcacdd..f232d87e9 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -326,7 +326,7 @@ class MPDStatsPlugin(plugins.BeetsPlugin): 'rating': True, 'rating_mix': 0.75, 'host': os.environ.get('MPD_HOST', u'localhost'), - 'port': 6600, + 'port': int(os.environ.get('MPD_PORT', 6600)), 'password': u'', }) mpd_config['password'].redact = True diff --git a/docs/plugins/mpdstats.rst b/docs/plugins/mpdstats.rst index 5472e7294..2e5e78c36 100644 --- a/docs/plugins/mpdstats.rst +++ b/docs/plugins/mpdstats.rst @@ -42,7 +42,8 @@ configuration file. The available options are: Default: The ``$MPD_HOST`` environment variable if set, falling back to ``localhost`` otherwise. - **port**: The MPD server port. - Default: 6600. + Default: The ``$MPD_PORT`` environment variable if set, + falling back to 6600 otherwise. - **password**: The MPD server password. Default: None. - **music_directory**: If your MPD library is at a different location from the From 21cba304bda60cd5f950bcbb2ab0175bdf389ca9 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 17:28:02 +1000 Subject: [PATCH 183/339] mpdupdate: use MPD_PORT env variable --- beetsplug/mpdupdate.py | 2 +- docs/plugins/mpdupdate.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/mpdupdate.py b/beetsplug/mpdupdate.py index 6ecc92131..72a98af0e 100644 --- a/beetsplug/mpdupdate.py +++ b/beetsplug/mpdupdate.py @@ -69,7 +69,7 @@ class MPDUpdatePlugin(BeetsPlugin): super(MPDUpdatePlugin, self).__init__() config['mpd'].add({ 'host': os.environ.get('MPD_HOST', u'localhost'), - 'port': 6600, + 'port': int(os.environ.get('MPD_PORT', 6600)), 'password': u'', }) config['mpd']['password'].redact = True diff --git a/docs/plugins/mpdupdate.rst b/docs/plugins/mpdupdate.rst index c846b917f..7ac647536 100644 --- a/docs/plugins/mpdupdate.rst +++ b/docs/plugins/mpdupdate.rst @@ -33,6 +33,7 @@ The available options under the ``mpd:`` section are: - **host**: The MPD server name. Default: The ``$MPD_HOST`` environment variable if set, falling back to ``localhost`` otherwise. - **port**: The MPD server port. - Default: 6600. + Default: The ``$MPD_PORT`` environment variable if set, falling back to 6600 + otherwise. - **password**: The MPD server password. Default: None. From 869d6b280fe9941192c0cfa4151c2bdf84d87ba3 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 17:35:27 +1000 Subject: [PATCH 184/339] Changelog for #3223 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ad0f057a..445847798 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -81,6 +81,9 @@ New features: to be displayed. Thanks to :user:`pprkut`. :bug:`3089` +* :doc:`/plugins/mpdstats`, :doc:`/plugins/mpdupdate`: Use the ``MPD_PORT`` + environment variable if no port is specified in the configuration file. + :bug:`3223` * :doc:`/plugins/bpd`: MPD protocol commands ``consume`` and ``single`` are now supported along with updated semantics for ``repeat`` and ``previous`` and new fields for ``status``. The bpd server now understands and ignores some From e4b2e7b4760e33e92d788ef93e41829c49987515 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Fri, 19 Apr 2019 20:58:13 +0200 Subject: [PATCH 185/339] Made -a and -A mutually exclusive. --- beetsplug/lastgenre/__init__.py | 33 +++++++++++++++++++++------------ docs/changelog.rst | 1 + docs/plugins/lastgenre.rst | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index a26b1fd82..2f90cc1c2 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -14,6 +14,8 @@ # included in all copies or substantial portions of the Software. from __future__ import division, absolute_import, print_function + +import ipdb import six """Gets genres for imported music based on Last.fm tags. @@ -381,19 +383,24 @@ class LastGenrePlugin(plugins.BeetsPlugin): help=u'genre source: artist, album, or track' ) lastgenre_cmd.parser.add_option( - u'-A', u'--tracks', action='store_true', default=False, - help=u'match tracks instead of albums' - ) + u'-A', u'--items', action='store_true', + help=u'match items instead of albums') lastgenre_cmd.parser.add_option( - u'-a', u'--albums', action='store_true', default=True, - help=u'match albums instead of tracks' - ) + u'-a', u'--albums', action='store_true', + help=u'match albums instead of items') + lastgenre_cmd.parser.set_defaults(query_type='albums') + def lastgenre_func(lib, opts, args): write = ui.should_write() self.config.set_args(opts) + if opts.albums and opts.items: + self._log.error(u'options -a and -A are mutually exclusive') + return + if opts.albums: + # Fetch genres for whole albums for album in lib.albums(ui.decargs(args)): album.genre, src = self._get_genre(album) self._log.info(u'genre for album {0} ({1}): {0.genre}', @@ -406,17 +413,19 @@ class LastGenrePlugin(plugins.BeetsPlugin): if 'track' in self.sources: item.genre, src = self._get_genre(item) item.store() - self._log.info(u'genre for track {0} ({1}): {0.genre}', - item, src) + self._log.info( + u'genre for track {0} ({1}): {0.genre}', + item, src) if write: item.try_write() - - if opts.tracks: + elif opts.items: + # Just query singletons, i.e. items that are not part of + # an album for item in lib.items(ui.decargs(args)): item.genre, src = self._get_genre(item) - self._log.debug(u'added last.fm item genre ({0}): {1}', src, - item.genre) + self._log.debug(u'added last.fm item genre ({0}): {1}', + src, item.genre) item.store() lastgenre_cmd.func = lastgenre_func diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ad0f057a..e0df4793c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* LastGenre can now be used to fetch genres for singletons. * The disambiguation string for identifying albums in the importer now shows the catalog number. Thanks to :user:`8h2a`. diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index 5e3235bd7..9604cf8aa 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -155,7 +155,7 @@ Running Manually In addition to running automatically on import, the plugin can also be run manually from the command line. Use the command ``beet lastgenre [QUERY]`` to fetch -genres for albums matching a certain query. +genres for albums or items matching a certain query. To disable automatic genre fetching on import, set the ``auto`` config option to false. From 5c643a8f164ff137954386c5f717a8ce77fd7708 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Fri, 19 Apr 2019 21:08:26 +0200 Subject: [PATCH 186/339] Removed ipdb import --- beetsplug/lastgenre/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index 2f90cc1c2..b29d7d01e 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -15,7 +15,6 @@ from __future__ import division, absolute_import, print_function -import ipdb import six """Gets genres for imported music based on Last.fm tags. From ddd7b4b3b4a30d060f127a4b6042008d8e655b76 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Fri, 19 Apr 2019 21:17:15 +0200 Subject: [PATCH 187/339] Removed empty line --- beetsplug/lastgenre/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index b29d7d01e..f13b62f1e 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -389,7 +389,6 @@ class LastGenrePlugin(plugins.BeetsPlugin): help=u'match albums instead of items') lastgenre_cmd.parser.set_defaults(query_type='albums') - def lastgenre_func(lib, opts, args): write = ui.should_write() self.config.set_args(opts) From 4d98088cc1f9e4663b5d1058abe2bd0d0179371f Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Sat, 20 Apr 2019 01:14:15 +0200 Subject: [PATCH 188/339] Replace more instances of unsafe calls to yaml.load --- beetsplug/edit.py | 2 +- test/test_config_command.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/edit.py b/beetsplug/edit.py index 631a1b584..9dbfcdd17 100644 --- a/beetsplug/edit.py +++ b/beetsplug/edit.py @@ -74,7 +74,7 @@ def load(s): """ try: out = [] - for d in yaml.load_all(s): + for d in yaml.safe_load_all(s): if not isinstance(d, dict): raise ParseError( u'each entry must be a dictionary; found {}'.format( diff --git a/test/test_config_command.py b/test/test_config_command.py index 35ba6ca0e..0d16dbf19 100644 --- a/test/test_config_command.py +++ b/test/test_config_command.py @@ -45,7 +45,7 @@ class ConfigCommandTest(unittest.TestCase, TestHelper): def _run_with_yaml_output(self, *args): output = self.run_with_output(*args) - return yaml.load(output) + return yaml.safe_load(output) def test_show_user_config(self): output = self._run_with_yaml_output('config', '-c') From bdc053d14b735dce63dc9cf6aca33fb4797652e5 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 20 Apr 2019 17:29:43 +1000 Subject: [PATCH 189/339] Changelog for #3225 --- docs/changelog.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ad0f057a..5398e0312 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -183,10 +183,10 @@ Fixes: * Fix several uses of deprecated standard-library features on Python 3.7. Thanks to :user:`arcresu`. :bug:`3197` -* :doc:`/plugins/lastgenre`: Avoid a deprecation warning from the YAML - library. - Thanks to :user:`translit`. - :bug:`3192` +* :doc:`/plugins/lastgenre`, :doc:`/plugins/edit`: Avoid a deprecation warnings + from the YAML library by switching to the safe loader. + Thanks to :user:`translit` and :user:`sbraz`. + :bug:`3192` :bug:`3225` * Fix compatibility with pre-release versions of Python 3.8. :bug:`3201` :bug:`3202` * :doc:`/plugins/bpd`: Fix crashes in the bpd server during exception handling. From d5507dc956e72defe3e67e7eba0f5b6f8c267cbe Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 20 Apr 2019 18:13:28 +1000 Subject: [PATCH 190/339] docs: remove reference to BeetsPlugin.listen This decorator was removed in 4578c4f0e1659daab8d25c79d1f466ae9838d8c7 and now `BeetsPlugin.register_listener` should be used instead. Fixes #2885. --- docs/dev/plugins.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index c9018c394..745c8340a 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -103,19 +103,18 @@ operation. For instance, a plugin could write a log message every time an album is successfully autotagged or update MPD's index whenever the database is changed. -You can "listen" for events using the ``BeetsPlugin.listen`` decorator. Here's +You can "listen" for events using ``BeetsPlugin.register_listener``. Here's an example:: from beets.plugins import BeetsPlugin - class SomePlugin(BeetsPlugin): - pass - - @SomePlugin.listen('pluginload') def loaded(): print 'Plugin loaded!' -Pass the name of the event in question to the ``listen`` decorator. + class SomePlugin(BeetsPlugin): + def __init__(self): + super(SomePlugin, self).__init__() + self.register_listener('pluginload', loaded) Note that if you want to access an attribute of your plugin (e.g. ``config`` or ``log``) you'll have to define a method and not a function. Here is the usual From b08db06c05435e23734d316cff8cd4778264426f Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sat, 20 Apr 2019 20:43:24 +0100 Subject: [PATCH 191/339] Add load_extension method for loading SQLite extensions --- beets/dbcore/db.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 43c044572..97a4a7ce3 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -850,16 +850,21 @@ class Database(object): """A container for Model objects that wraps an SQLite database as the backend. """ + _models = () """The Model subclasses representing tables in this database. """ + supports_extensions = hasattr(sqlite3.Connection, 'enable_load_extension') + """Whether or not the current version of SQLite supports extensions""" + def __init__(self, path, timeout=5.0): self.path = path self.timeout = timeout self._connections = {} self._tx_stacks = defaultdict(list) + self._extensions = [] # A lock to protect the _connections and _tx_stacks maps, which # both map thread IDs to private resources. @@ -909,6 +914,13 @@ class Database(object): py3_path(self.path), timeout=self.timeout ) + if self.supports_extensions: + conn.enable_load_extension(True) + + # Load any extension that are already loaded for other connections. + for path in self._extensions: + conn.load_extension(path) + # Access SELECT results like dictionaries. conn.row_factory = sqlite3.Row return conn @@ -937,6 +949,18 @@ class Database(object): """ return Transaction(self) + def load_extension(self, path): + """Load an SQLite extension into all open connections.""" + if not self.supports_extensions: + raise ValueError( + 'this sqlite3 installation does not support extensions') + + self._extensions.append(path) + + # Load the extension into every open connection. + for conn in self._connections.values(): + conn.load_extension(path) + # Schema setup and migration. def _make_table(self, table, fields): From f5f9aed641eff8702d6b72323b1680a6140650a1 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sat, 20 Apr 2019 01:42:32 +0100 Subject: [PATCH 192/339] Add loadext plugin --- beetsplug/loadext.py | 46 ++++++++++++++++++++++++++++++++++ docs/changelog.rst | 3 +++ docs/plugins/index.rst | 2 ++ docs/plugins/loadext.rst | 53 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 beetsplug/loadext.py create mode 100644 docs/plugins/loadext.rst diff --git a/beetsplug/loadext.py b/beetsplug/loadext.py new file mode 100644 index 000000000..5ab98bd59 --- /dev/null +++ b/beetsplug/loadext.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Jack Wilsdon +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Load SQLite extensions. +""" + +from __future__ import division, absolute_import, print_function + +from beets.dbcore import Database +from beets.plugins import BeetsPlugin +import sqlite3 + + +class LoadExtPlugin(BeetsPlugin): + def __init__(self): + super(LoadExtPlugin, self).__init__() + + if not Database.supports_extensions: + self._log.warn('loadext is enabled but the current SQLite ' + 'installation does not support extensions') + return + + self.register_listener('library_opened', self.library_opened) + + def library_opened(self, lib): + for v in self.config: + ext = v.as_filename() + + self._log.debug(u'loading extension {}', ext) + + try: + lib.load_extension(ext) + except sqlite3.OperationalError as e: + self._log.error(u'failed to load extension {}: {}', ext, e) diff --git a/docs/changelog.rst b/docs/changelog.rst index ffe8bdcac..2ccfbadb5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -94,6 +94,9 @@ New features: :bug:`3205` :bug:`800` * :doc:`/plugins/bpd`: MPD protocol command ``decoders`` is now supported. :bug:`3222` +* The new :doc:`/plugins/loadext` allows loading of SQLite extensions, primarily + for use with the ICU SQLite extension for internationalization. + :bug:`3160` :bug:`3226` Changes: diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 173aab5db..e885db39b 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -71,6 +71,7 @@ like this:: kodiupdate lastgenre lastimport + loadext lyrics mbcollection mbsubmit @@ -189,6 +190,7 @@ Miscellaneous * :doc:`hook`: Run a command when an event is emitted by beets. * :doc:`ihate`: Automatically skip albums and tracks during the import process. * :doc:`info`: Print music files' tags to the console. +* :doc:`loadext`: Load SQLite extensions. * :doc:`mbcollection`: Maintain your MusicBrainz collection list. * :doc:`mbsubmit`: Print an album's tracks in a MusicBrainz-friendly format. * :doc:`missing`: List missing tracks. diff --git a/docs/plugins/loadext.rst b/docs/plugins/loadext.rst new file mode 100644 index 000000000..5acd10ec7 --- /dev/null +++ b/docs/plugins/loadext.rst @@ -0,0 +1,53 @@ +Load Extension Plugin +===================== + +Beets uses an SQLite database to store and query library information, which +has support for extensions to extend its functionality. The ``loadext`` plugin +lets you enable these SQLite extensions within beets. + +One of the primary uses of this within beets is with the `"ICU" extension`_, +which adds support for case insensitive querying of non-ASCII characters. + +.. _"ICU" extension: https://www.sqlite.org/src/dir?ci=7461d2e120f21493&name=ext/icu + +Configuration +------------- + +To configure the plugin, make a ``loadext`` section in your configuration +file. The section must consist of a list of paths to extensions to load, which +looks like this: + +.. code-block:: yaml + + loadext: + - libicu + +If a relative path is specified, it is resolved relative to the beets +configuration directory. + +If no file extension is specified, the default dynamic library extension for +the current platform will be used. + +Building the ICU extension +-------------------------- +This section is for **advanced** users only, and is not an in-depth guide on +building the extension. + +To compile the ICU extension, you will need a few dependencies: + + - gcc + - icu-devtools + - libicu + - libicu-dev + - libsqlite3-dev + +Here's roughly how to download, build and install the extension (although the +specifics may vary from system to system): + +.. code-block:: shell + + $ wget https://sqlite.org/2019/sqlite-src-3280000.zip + $ unzip sqlite-src-3280000.zip + $ cd sqlite-src-3280000/ext/icu + $ gcc -shared -fPIC icu.c `icu-config --ldflags` -o libicu.so + $ cp libicu.so ~/.config/beets From b7d3ef62740aa2af90035fc9a6c7845541457e39 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Sun, 21 Apr 2019 18:32:41 +0200 Subject: [PATCH 193/339] - Improved doc and changelog - Cleaner implementation of mutual excursion of the command line arguments. --- beetsplug/lastgenre/__init__.py | 14 +++++--------- docs/changelog.rst | 2 +- docs/plugins/lastgenre.rst | 3 +++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/beetsplug/lastgenre/__init__.py b/beetsplug/lastgenre/__init__.py index f13b62f1e..cf90facbd 100644 --- a/beetsplug/lastgenre/__init__.py +++ b/beetsplug/lastgenre/__init__.py @@ -382,22 +382,18 @@ class LastGenrePlugin(plugins.BeetsPlugin): help=u'genre source: artist, album, or track' ) lastgenre_cmd.parser.add_option( - u'-A', u'--items', action='store_true', + u'-A', u'--items', action='store_false', dest='album', help=u'match items instead of albums') lastgenre_cmd.parser.add_option( - u'-a', u'--albums', action='store_true', + u'-a', u'--albums', action='store_true', dest='album', help=u'match albums instead of items') - lastgenre_cmd.parser.set_defaults(query_type='albums') + lastgenre_cmd.parser.set_defaults(album=True) def lastgenre_func(lib, opts, args): write = ui.should_write() self.config.set_args(opts) - if opts.albums and opts.items: - self._log.error(u'options -a and -A are mutually exclusive') - return - - if opts.albums: + if opts.album: # Fetch genres for whole albums for album in lib.albums(ui.decargs(args)): album.genre, src = self._get_genre(album) @@ -417,7 +413,7 @@ class LastGenrePlugin(plugins.BeetsPlugin): if write: item.try_write() - elif opts.items: + else: # Just query singletons, i.e. items that are not part of # an album for item in lib.items(ui.decargs(args)): diff --git a/docs/changelog.rst b/docs/changelog.rst index e0df4793c..d5e9a04df 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog New features: -* LastGenre can now be used to fetch genres for singletons. +* :doc:`/plugins/lastgenre`: Added option ``-A`` to match individual tracks and singletons. * The disambiguation string for identifying albums in the importer now shows the catalog number. Thanks to :user:`8h2a`. diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index 9604cf8aa..c7d04fe25 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -157,5 +157,8 @@ In addition to running automatically on import, the plugin can also be run manua from the command line. Use the command ``beet lastgenre [QUERY]`` to fetch genres for albums or items matching a certain query. +By default, ``beet lastgenre`` matches albums. If you would like to match +individual tracks or singletons, use the ``-A`` switch: ``beet lastgenre -A [QUERY]``. + To disable automatic genre fetching on import, set the ``auto`` config option to false. From 8b234117675a0f1a8011de392e611e2a9da80882 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 21 Apr 2019 20:40:18 -0400 Subject: [PATCH 194/339] Simplify docs for #3220 --- docs/plugins/lastgenre.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index c7d04fe25..c3d5f97ec 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -157,8 +157,9 @@ In addition to running automatically on import, the plugin can also be run manua from the command line. Use the command ``beet lastgenre [QUERY]`` to fetch genres for albums or items matching a certain query. -By default, ``beet lastgenre`` matches albums. If you would like to match -individual tracks or singletons, use the ``-A`` switch: ``beet lastgenre -A [QUERY]``. +By default, ``beet lastgenre`` matches albums. To match +individual tracks or singletons, use the ``-A`` switch: +``beet lastgenre -A [QUERY]``. To disable automatic genre fetching on import, set the ``auto`` config option to false. From 4d55e6dfbb6b3da5c731353bddbebf9423643104 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 21 Apr 2019 20:41:18 -0400 Subject: [PATCH 195/339] Add changelog link for #3220 (fix #3219) --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7dd455125..ab752b264 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,9 @@ Changelog New features: -* :doc:`/plugins/lastgenre`: Added option ``-A`` to match individual tracks and singletons. +* :doc:`/plugins/lastgenre`: Added option ``-A`` to match individual tracks + and singletons. + :bug:`3220` :bug:`3219` * The disambiguation string for identifying albums in the importer now shows the catalog number. Thanks to :user:`8h2a`. From a60935ed1c69993c0739c41d6c0afb3febbd03c5 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Mon, 22 Apr 2019 19:50:11 +0200 Subject: [PATCH 196/339] Prevent scientific notation --- beetsplug/acousticbrainz.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index f4960c301..bbbd098c7 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -17,9 +17,10 @@ """ from __future__ import division, absolute_import, print_function +from collections import defaultdict + import requests -from collections import defaultdict from beets import plugins, ui ACOUSTIC_BASE = "https://acousticbrainz.org/" @@ -101,6 +102,10 @@ ABSCHEME = { } } +FLOAT_FIELDS = ['danceable', 'mood_acoustic', 'mood_aggressive', + 'mood_electronic', 'mood_happy', 'mood_party', 'mood_relaxed', + 'mood_sad', 'tonal', 'average_loudness', 'chords_changes_rate', + 'chords_number_rate', 'key_strength'] class AcousticPlugin(plugins.BeetsPlugin): @@ -186,6 +191,8 @@ class AcousticPlugin(plugins.BeetsPlugin): if data: for attr, val in self._map_data_to_scheme(data, ABSCHEME): if not tags or attr in tags: + if attr in FLOAT_FIELDS: + val = '%f' % val self._log.debug(u'attribute {} of {} set to {}', attr, item, From 0a7c5cb163dc3419aca6b85b51bd8f410def4c93 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 23 Apr 2019 12:26:36 +1000 Subject: [PATCH 197/339] Prepare changelog for 1.4.8 --- docs/changelog.rst | 323 ++++++++++++++++++++++++--------------------- 1 file changed, 176 insertions(+), 147 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ab752b264..f9196f192 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,50 +4,8 @@ Changelog 1.4.8 (in development) ---------------------- -New features: +There are some new core features: -* :doc:`/plugins/lastgenre`: Added option ``-A`` to match individual tracks - and singletons. - :bug:`3220` :bug:`3219` -* The disambiguation string for identifying albums in the importer now shows - the catalog number. - Thanks to :user:`8h2a`. - :bug:`2951` -* :doc:`/plugins/play`: The plugin can now emit a UTF-8 BOM, fixing some - issues with foobar2000 and Winamp. - Thanks to :user:`mz2212`. - :bug:`2944` -* A new :doc:`/plugins/playlist` can query the beets library using - M3U playlists. - Thanks to :user:`Holzhaus` and :user:`Xenopathic`. - :bug:`123` :bug:`3145` -* Added whitespace padding to missing tracks dialog to improve readability. - Thanks to :user:`jams2`. - :bug:`2962` -* :bug:`/plugins/gmusic`: Add a new option to automatically upload to Google - Play Music library on track import. - Thanks to :user:`shuaiscott`. -* :doc:`/plugins/gmusic`: Add new options for Google Play Music - authentication. - Thanks to :user:`thetarkus`. - :bug:`3002` -* :doc:`/plugins/absubmit`: Analysis now works in parallel (on Python 3 only). - Thanks to :user:`bemeurer`. - :bug:`2442` :bug:`3003` -* :doc:`/plugins/replaygain`: albumpeak on large collections is calculated as - the average, not the maximum. - :bug:`3008` :bug:`3009` -* A new :doc:`/plugins/subsonicupdate` can automatically update your Subsonic library. - Thanks to :user:`maffo999`. - :bug:`3001` -* :doc:`/plugins/chroma`: Now optionally has a bias toward looking up more - relevant releases according to the :ref:`preferred` configuration options. - Thanks to :user:`archer4499`. - :bug:`3017` -* :doc:`/plugins/convert`: The plugin now has a ``id3v23`` option that allows - to override the global ``id3v23`` option. - Thanks to :user:`Holzhaus`. - :bug:`3104` * A new ``aunique`` configuration option allows setting default options for the :ref:`aunique` template function. * The ``albumdisambig`` field no longer includes the MusicBrainz release group @@ -57,15 +15,54 @@ New features: example, ``beet modify -a artist:beatles artpath!`` resets ``artpath`` attribute from matching albums back to the default value. :bug:`2497` -* Modify selection can now be applied early without selecting every item. - :bug:`3083` -* :doc:`/plugins/chroma`: Fingerprint values are now properly stored as - strings, which prevents strange repeated output when running ``beet write``. +* A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks + contained in data files :bug:`3021` + +There are some new plugins: + +* The :doc:`/plugins/playlist` can query the beets library using M3U playlists. + Thanks to :user:`Holzhaus` and :user:`Xenopathic`. + :bug:`123` :bug:`3145` +* The :doc:`/plugins/loadext` allows loading of SQLite extensions, primarily + for use with the ICU SQLite extension for internationalization. + :bug:`3160` :bug:`3226` +* The :doc:`/plugins/subsonicupdate` can automatically update your Subsonic + library. + Thanks to :user:`maffo999`. + :bug:`3001` + +And many improvements to existing plugins: + +* :doc:`/plugins/lastgenre`: Added option ``-A`` to match individual tracks + and singletons. + :bug:`3220` :bug:`3219` +* :doc:`/plugins/play`: The plugin can now emit a UTF-8 BOM, fixing some + issues with foobar2000 and Winamp. + Thanks to :user:`mz2212`. + :bug:`2944` +* :doc:`/plugins/gmusic`: + * Add a new option to automatically upload to Google Play Music library on + track import. + Thanks to :user:`shuaiscott`. + * Add new options for Google Play Music authentication. + Thanks to :user:`thetarkus`. + :bug:`3002` +* :doc:`/plugins/replaygain`: ``albumpeak`` on large collections is calculated + as the average, not the maximum. + :bug:`3008` :bug:`3009` +* :doc:`/plugins/chroma`: + * Now optionally has a bias toward looking up more relevant releases + according to the :ref:`preferred` configuration options. + Thanks to :user:`archer4499`. + :bug:`3017` + * Fingerprint values are now properly stored as strings, which prevents + strange repeated output when running ``beet write``. + Thanks to :user:`Holzhaus`. + :bug:`3097` :bug:`2942` +* :doc:`/plugins/convert`: The plugin now has a ``id3v23`` option that allows + youto override the global ``id3v23`` option. Thanks to :user:`Holzhaus`. - :bug:`3097` :bug:`2942` -* The ``move`` command now lists the number of items already in-place. - Thanks to :user:`RollingStar`. - :bug:`3117` + :bug:`3104` * :doc:`/plugins/spotify`: The plugin now uses OAuth for authentication to the Spotify API. Thanks to :user:`rhlahuja`. @@ -74,58 +71,142 @@ New features: provider: you can match tracks and albums using the Spotify database. Thanks to :user:`rhlahuja`. :bug:`3123` -* :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which passes that flag to ipfs. +* :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which + passes that flag to ipfs. Thanks to :user:`wildthyme`. * :doc:`/plugins/discogs`: The plugin has rate limiting for the discogs API now. :bug:`3081` -* The `badfiles` plugin now works in parallel (on Python 3 only). - Thanks to :user:`bemeurer`. -* Querying the library is now faster because we only convert fields that need - to be displayed. - Thanks to :user:`pprkut`. - :bug:`3089` -* :doc:`/plugins/mpdstats`, :doc:`/plugins/mpdupdate`: Use the ``MPD_PORT`` - environment variable if no port is specified in the configuration file. +* :doc:`/plugins/mpdstats`, :doc:`/plugins/mpdupdate`: These plugins now use + the ``MPD_PORT`` environment variable if no port is specified in the + configuration file. :bug:`3223` -* :doc:`/plugins/bpd`: MPD protocol commands ``consume`` and ``single`` are now - supported along with updated semantics for ``repeat`` and ``previous`` and - new fields for ``status``. The bpd server now understands and ignores some - additional commands. - :bug:`3200` :bug:`800` -* :doc:`/plugins/bpd`: MPD protocol command ``idle`` is now supported, allowing - the MPD version to be bumped to 0.14. - :bug:`3205` :bug:`800` -* :doc:`/plugins/bpd`: MPD protocol command ``decoders`` is now supported. - :bug:`3222` -* The new :doc:`/plugins/loadext` allows loading of SQLite extensions, primarily - for use with the ICU SQLite extension for internationalization. - :bug:`3160` :bug:`3226` - -Changes: - -* :doc:`/plugins/mbsync` no longer queries MusicBrainz when either the - ``mb_albumid`` or ``mb_trackid`` field is invalid +* :doc:`/plugins/bpd`: + * MPD protocol commands ``consume`` and ``single`` are now supported along + with updated semantics for ``repeat`` and ``previous`` and new fields for + ``status``. The bpd server now understands and ignores some additional + commands. + :bug:`3200` :bug:`800` + * MPD protocol command ``idle`` is now supported, allowing the MPD version + to be bumped to 0.14. + :bug:`3205` :bug:`800` + * MPD protocol command ``decoders`` is now supported. + :bug:`3222` + * The plugin now uses the main beets logging system. + The special-purpose ``--debug`` flag has been removed. + Thanks to :user:`arcresu`. + :bug:`3196` +* :doc:`/plugins/mbsync`: The plugin no longer queries MusicBrainz when either + the ``mb_albumid`` or ``mb_trackid`` field is invalid. See also the discussion on Google Groups_ Thanks to :user:`arogl`. -* :doc:`/plugins/export` now also exports ``path`` field if user explicitly - specifies it with ``-i`` parameter. Only works when exporting library fields. +* :doc:`/plugins/export`: The plugin now also exports ``path`` field if user + explicitly specifies it with ``-i`` parameter. This only works when exporting + library fields. :bug:`3084` .. _Groups: https://groups.google.com/forum/#!searchin/beets-users/mbsync|sort:date/beets-users/iwCF6bNdh9A/i1xl4Gx8BQAJ -Fixes: +Some improvements have been focused on improving beets' performance: + +* Querying the library is now faster because we only convert fields that need + to be displayed. + Thanks to :user:`pprkut`. + :bug:`3089` +* :doc:`/plugins/absubmit`, :doc:`/plugins/badfiles`: Analysis now works in + parallel (on Python 3 only). + Thanks to :user:`bemeurer`. + :bug:`2442` :bug:`3003` +* :doc:`/plugins/mpdstats`: Use the ``currentsong`` MPD command instead of + ``playlist`` to get the current song, improving performance when the playlist + is long. + Thanks to :user:`ray66`. + :bug:`3207` :bug:`2752` + +Several improvements are related to usability: + +* The disambiguation string for identifying albums in the importer now shows + the catalog number. + Thanks to :user:`8h2a`. + :bug:`2951` +* Added whitespace padding to missing tracks dialog to improve readability. + Thanks to :user:`jams2`. + :bug:`2962` +* The :ref:`move-cmd` command now lists the number of items already in-place. + Thanks to :user:`RollingStar`. + :bug:`3117` +* Modify selection can now be applied early without selecting every item. + :bug:`3083` +* Improve error reporting during startup if sqlite returns an error. The + sqlite error message is now attached to the beets message. + :bug:`3005` +* Fixed a confusing typo when the :doc:`/plugins/convert` plugin copies the art + covers. + :bug:`3063` + +Many fixes have been focused on issues where beets would previously crash: + +* Avoid a crash when archive extraction fails during import. + :bug:`3041` +* Missing album art file during an update no longer causes a fatal exception + (instead, an error is logged and the missing file path is removed from the + library). + :bug:`3030` +* When updating the database, beets no longer tries to move album art twice. + :bug:`3189` +* Fix an unhandled exception when pruning empty directories. + :bug:`1996` :bug:`3209` +* :doc:`/plugins/fetchart`: Added network connection error handling to backends + so that beets won't crash if a request fails. + Thanks to :user:`Holzhaus`. + :bug:`1579` +* :doc:`/plugins/badfiles`: Avoid a crash when the underlying tool emits + undecodable output. + :bug:`3165` +* :doc:`/plugins/beatport`: Avoid a crash when the server produces an error. + :bug:`3184` +* :doc:`/plugins/bpd`: Fix crashes in the bpd server during exception handling. + :bug:`3200` +* :doc:`/plugins/bpd`: Fix a crash triggered when certain clients tried to list + the albums belonging to a particular artist. + :bug:`3007` :bug:`3215` + +There are many fixes related to compatibility with our dependencies including +addressing changes interfaces: * On Python 2, pin the Jellyfish requirement to version 0.6.0 for compatibility. -* A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks - contained in data files :bug:`3021` +* Fix compatibility Python 3.7 and its change to a name in the ``re`` module. + :bug:`2978` +* Fix several uses of deprecated standard-library features on Python 3.7. + Thanks to :user:`arcresu`. + :bug:`3197` +* Fix compatibility with pre-release versions of Python 3.8. + :bug:`3201` :bug:`3202` +* :doc:`/plugins/web`: Fix an error when using more recent versions of Flask + with CORS enabled. + Thanks to :user:`rveachkc`. + :bug:`2979`: :bug:`2980` +* Avoid some deprecation warnings with certain versions of the MusicBrainz + library. + Thanks to :user:`zhelezov`. + :bug:`2826` :bug:`3092` * Restore iTunes Store album art source, and remove the dependency on python-itunes_, which had gone unmaintained and was not py3 compatible. Thanks to :user:`ocelma` for creating python-itunes_ in the first place. Thanks to :user:`nathdwek`. :bug:`2371` :bug:`2551` :bug:`2718` -* Fix compatibility Python 3.7 and its change to a name in the ``re`` module. - :bug:`2978` +* :doc:`/plugins/lastgenre`, :doc:`/plugins/edit`: Avoid a deprecation warnings + from the YAML library by switching to the safe loader. + Thanks to :user:`translit` and :user:`sbraz`. + :bug:`3192` :bug:`3225` +* Fix a problem when resizing images with PIL/Pillow on Python 3. + Thanks to :user:`architek`. + :bug:`2504` :bug:`3029` + +.. _python-itunes: https://github.com/ocelma/python-itunes + +And there are many other fixes: + * R128 normalization tags are now properly deleted from files when the values are missing. Thanks to :user:`autrimpo`. @@ -136,86 +217,30 @@ Fixes: * With the :ref:`from_scratch` configuration option set, only writable fields are cleared. Beets now no longer ignores the format your music is saved in. :bug:`2972` -* LastGenre: Allow to set the configuration option ``prefer_specific`` - without setting ``canonical``. - :bug:`2973` -* :doc:`/plugins/web`: Fix an error when using more recent versions of Flask - with CORS enabled. - Thanks to :user:`rveachkc`. - :bug:`2979`: :bug:`2980` -* Improve error reporting: during startup if sqlite returns an error the - sqlite error message is attached to the beets message. - :bug:`3005` -* Fix a problem when resizing images with PIL/Pillow on Python 3. - Thanks to :user:`architek`. - :bug:`2504` :bug:`3029` -* Avoid a crash when archive extraction fails during import. - :bug:`3041` * The ``%aunique`` template function now works correctly with the ``-f/--format`` option. :bug:`3043` -* Missing album art file during an update no longer causes a fatal exception - (instead, an error is logged and the missing file path is removed from the - library). :bug:`3030` * Fixed the ordering of items when manually selecting changes while updating tags Thanks to :user:`TaizoSimpson`. :bug:`3501` -* Confusing typo when the convert plugin copies the art covers. :bug:`3063` * The ``%title`` template function now works correctly with apostrophes. Thanks to :user:`GuilhermeHideki`. :bug:`3033` -* When updating the database, beets no longer tries to move album art twice. - :bug:`3189` -* :doc:`/plugins/fetchart`: Added network connection error handling to backends - so that beets won't crash if a request fails. - Thanks to :user:`Holzhaus`. - :bug:`1579` -* Fetchart now respects the ``ignore`` and ``ignore_hidden`` settings. :bug:`1632` -* :doc:`/plugins/badfiles`: Avoid a crash when the underlying tool emits - undecodable output. - :bug:`3165` +* :doc:`/plugins/lastgenre`: Allow to set the configuration option + ``prefer_specific`` without setting ``canonical``. + :bug:`2973` +* :doc:`/plugins/fetchart`: The plugin now respects the ``ignore`` and + ``ignore_hidden`` settings. + :bug:`1632` * :doc:`/plugins/hook`: Fix byte string interpolation in hook commands. :bug:`2967` :bug:`3167` -* Avoid some deprecation warnings with certain versions of the MusicBrainz - library. - Thanks to :user:`zhelezov`. - :bug:`2826` :bug:`3092` -* :doc:`/plugins/beatport`: Avoid a crash when the server produces an error. - :bug:`3184` * :doc:`/plugins/the`: Log a message when something has changed, not when it hasn't. Thanks to :user:`arcresu`. :bug:`3195` -* :doc:`/plugins/bpd`: The plugin now uses the main beets logging system. - The special-purpose ``--debug`` flag has been removed. - Thanks to :user:`arcresu`. - :bug:`3196` -* Fix several uses of deprecated standard-library features on Python 3.7. - Thanks to :user:`arcresu`. - :bug:`3197` -* :doc:`/plugins/lastgenre`, :doc:`/plugins/edit`: Avoid a deprecation warnings - from the YAML library by switching to the safe loader. - Thanks to :user:`translit` and :user:`sbraz`. - :bug:`3192` :bug:`3225` -* Fix compatibility with pre-release versions of Python 3.8. - :bug:`3201` :bug:`3202` -* :doc:`/plugins/bpd`: Fix crashes in the bpd server during exception handling. - :bug:`3200` * :doc:`/plugins/lastgenre`: The `force` config option now actually works. :bug:`2704` :bug:`3054` -* :doc:`/plugins/mpdstats`: Use the ``currentsong`` MPD command instead of - ``playlist`` to get the current song, improving performance when the playlist - is long. - Thanks to :user:`ray66`. - :bug:`3207` :bug:`2752` -* Fix an unhandled exception when pruning empty directories. - :bug:`1996` :bug:`3209` -* :doc:`/plugins/bpd`: Fix a crash triggered when certain clients tried to list - the albums belonging to a particular artist. - :bug:`3007` :bug:`3215` - -.. _python-itunes: https://github.com/ocelma/python-itunes For developers: @@ -225,6 +250,10 @@ For developers: ``playlist:name`` although there is no field named ``playlist``. See :ref:`extend-query` for details. +For packagers: + +* Note the changes to the dependencies on jellyfish and python-itunes. + 1.4.7 (May 29, 2018) -------------------- From 750ddc1129398e5c243e62b9a54a6f189f495ac8 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Tue, 23 Apr 2019 12:30:58 +1000 Subject: [PATCH 198/339] Update docs/changelog.rst Co-Authored-By: arcresu --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f9196f192..5947148ef 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -60,7 +60,7 @@ And many improvements to existing plugins: Thanks to :user:`Holzhaus`. :bug:`3097` :bug:`2942` * :doc:`/plugins/convert`: The plugin now has a ``id3v23`` option that allows - youto override the global ``id3v23`` option. + you to override the global ``id3v23`` option. Thanks to :user:`Holzhaus`. :bug:`3104` * :doc:`/plugins/spotify`: The plugin now uses OAuth for authentication to the From 2aeb0fa80dffbae3b1ef4ac98f4b9314629ccdf9 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 23 Apr 2019 12:35:41 +1000 Subject: [PATCH 199/339] Improve message for packagers --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5947148ef..db298561a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -252,7 +252,9 @@ For developers: For packagers: -* Note the changes to the dependencies on jellyfish and python-itunes. +* Note the changes to the dependencies on jellyfish and munkres. +* The optional python-itunes dependency has been removed. +* Python versions 3.7 and 3.8 are now supported. 1.4.7 (May 29, 2018) From ad9f256058fefed9df9b476a7314fd14929ed7d2 Mon Sep 17 00:00:00 2001 From: ababyduck Date: Mon, 22 Apr 2019 23:12:51 -0700 Subject: [PATCH 200/339] Update artresizer's ImageMagick commands to use the magick binary Updated artresizer's ImageMagick commands to use the magick binary added in ImageMagick 7.x, rather than the legacy utilities ('convert', 'identify', etc.) This resolves an issue where beets is failing to detect or use ImageMagick on Windows even when it is set correctly on the PATH, which in turn restores functionality to the fetchart and embedart plugins on Windows. Closes #2093 --- beets/util/artresizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index e58b356be..31554ceb9 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -93,7 +93,7 @@ def im_resize(maxwidth, path_in, path_out=None): # with regards to the height. try: util.command_output([ - 'convert', util.syspath(path_in, prefix=False), + 'magick', util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), util.syspath(path_out, prefix=False), ]) @@ -121,7 +121,7 @@ def pil_getsize(path_in): def im_getsize(path_in): - cmd = ['identify', '-format', '%w %h', + cmd = ['magick', 'identify', '-format', '%w %h', util.syspath(path_in, prefix=False)] try: out = util.command_output(cmd) @@ -235,7 +235,7 @@ def get_im_version(): Try invoking ImageMagick's "convert". """ try: - out = util.command_output(['convert', '--version']) + out = util.command_output(['magick', '-version']) if b'imagemagick' in out.lower(): pattern = br".+ (\d+)\.(\d+)\.(\d+).*" From 76e333c054c298acce89740a438909dbc8ccb8c6 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Tue, 23 Apr 2019 19:20:55 +0200 Subject: [PATCH 201/339] Added item_types --- beetsplug/acousticbrainz.py | 49 +++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index bbbd098c7..3a6db7d4e 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -22,6 +22,7 @@ from collections import defaultdict import requests from beets import plugins, ui +from beets.dbcore import types ACOUSTIC_BASE = "https://acousticbrainz.org/" LEVELS = ["/low-level", "/high-level"] @@ -102,13 +103,32 @@ ABSCHEME = { } } -FLOAT_FIELDS = ['danceable', 'mood_acoustic', 'mood_aggressive', - 'mood_electronic', 'mood_happy', 'mood_party', 'mood_relaxed', - 'mood_sad', 'tonal', 'average_loudness', 'chords_changes_rate', - 'chords_number_rate', 'key_strength'] class AcousticPlugin(plugins.BeetsPlugin): + item_types = { + 'average_loudness': types.FLOAT, + 'chords_changes_rate': types.FLOAT, + 'chords_key': types.STRING, + 'chords_number_rate': types.FLOAT, + 'chords_scale': types.STRING, + 'danceable': types.FLOAT, + 'gender': types.STRING, + 'genre_rosamerica': types.STRING, + 'initial_key': types.STRING, + 'key_strength': types.FLOAT, + 'mood_acoustic': types.FLOAT, + 'mood_aggressive': types.FLOAT, + 'mood_electronic': types.FLOAT, + 'mood_happy': types.FLOAT, + 'mood_party': types.FLOAT, + 'mood_relaxed': types.FLOAT, + 'mood_sad': types.FLOAT, + 'rhythm': types.FLOAT, + 'tonal': types.FLOAT, + 'voice_instrumental': types.STRING, + } + def __init__(self): super(AcousticPlugin, self).__init__() @@ -156,7 +176,7 @@ class AcousticPlugin(plugins.BeetsPlugin): return {} if res.status_code == 404: - self._log.info(u'recording ID {} not found', mbid) + self._log.info(u'recording ID \'{}\' not found', mbid) return {} try: @@ -179,28 +199,27 @@ class AcousticPlugin(plugins.BeetsPlugin): if not force: mood_str = item.get('mood_acoustic', u'') if mood_str: - self._log.info(u'data already present for: {}', item) + self._log.info(u'data already present for: \'{}\'', item) continue # We can only fetch data for tracks with MBIDs. if not item.mb_trackid: continue - self._log.info(u'getting data for: {}', item) + self._log.info(u'getting data for: \'{}\'', item) data = self._get_data(item.mb_trackid) if data: for attr, val in self._map_data_to_scheme(data, ABSCHEME): if not tags or attr in tags: - if attr in FLOAT_FIELDS: - val = '%f' % val - self._log.debug(u'attribute {} of {} set to {}', - attr, - item, - val) + self._log.debug( + u'attribute \'{}\' of \'{}\' set to \'{}\'', + attr, + item, + val) setattr(item, attr, val) else: - self._log.debug(u'skipping attribute {} of {}' - u' (value {}) due to config', + self._log.debug(u'skipping attribute \'{}\' of \'{}\'' + u' (value \'{}\') due to config', attr, item, val) From 8ba2ccdc638f0f41e6644285bbb9db488af49ffd Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 24 Apr 2019 10:44:46 +1000 Subject: [PATCH 202/339] Improve changelog text Co-Authored-By: arcresu --- docs/changelog.rst | 38 +++++++++++++++++++------------------- docs/conf.py | 2 ++ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index db298561a..babcf7ac3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,7 +16,7 @@ There are some new core features: attribute from matching albums back to the default value. :bug:`2497` * A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks - contained in data files :bug:`3021` + contained in data files. :bug:`3021` There are some new plugins: @@ -97,14 +97,14 @@ And many improvements to existing plugins: :bug:`3196` * :doc:`/plugins/mbsync`: The plugin no longer queries MusicBrainz when either the ``mb_albumid`` or ``mb_trackid`` field is invalid. - See also the discussion on Google Groups_ + See also the discussion on `Google Groups`_ Thanks to :user:`arogl`. -* :doc:`/plugins/export`: The plugin now also exports ``path`` field if user +* :doc:`/plugins/export`: The plugin now also exports ``path`` field if the user explicitly specifies it with ``-i`` parameter. This only works when exporting library fields. :bug:`3084` -.. _Groups: https://groups.google.com/forum/#!searchin/beets-users/mbsync|sort:date/beets-users/iwCF6bNdh9A/i1xl4Gx8BQAJ +.. _Google Groups: https://groups.google.com/forum/#!searchin/beets-users/mbsync|sort:date/beets-users/iwCF6bNdh9A/i1xl4Gx8BQAJ Some improvements have been focused on improving beets' performance: @@ -136,8 +136,8 @@ Several improvements are related to usability: :bug:`3117` * Modify selection can now be applied early without selecting every item. :bug:`3083` -* Improve error reporting during startup if sqlite returns an error. The - sqlite error message is now attached to the beets message. +* Beets now emits more useful messages during startup if SQLite returns an error. The + SQLite error message is now attached to the beets message. :bug:`3005` * Fixed a confusing typo when the :doc:`/plugins/convert` plugin copies the art covers. @@ -173,9 +173,10 @@ Many fixes have been focused on issues where beets would previously crash: There are many fixes related to compatibility with our dependencies including addressing changes interfaces: -* On Python 2, pin the Jellyfish requirement to version 0.6.0 for +* On Python 2, pin the :pypi:`jellyfish` requirement to version 0.6.0 for compatibility. -* Fix compatibility Python 3.7 and its change to a name in the ``re`` module. +* Fix compatibility with Python 3.7 and its change to a name in the + :stdlib:`re` module. :bug:`2978` * Fix several uses of deprecated standard-library features on Python 3.7. Thanks to :user:`arcresu`. @@ -191,20 +192,19 @@ addressing changes interfaces: Thanks to :user:`zhelezov`. :bug:`2826` :bug:`3092` * Restore iTunes Store album art source, and remove the dependency on - python-itunes_, which had gone unmaintained and was not py3 compatible. - Thanks to :user:`ocelma` for creating python-itunes_ in the first place. + :pypi:`python-itunes`, which had gone unmaintained and was not + Python-3-compatible. + Thanks to :user:`ocelma` for creating :pypi:`python-itunes` in the first place. Thanks to :user:`nathdwek`. :bug:`2371` :bug:`2551` :bug:`2718` * :doc:`/plugins/lastgenre`, :doc:`/plugins/edit`: Avoid a deprecation warnings - from the YAML library by switching to the safe loader. + from the :pypi:`PyYAML` library by switching to the safe loader. Thanks to :user:`translit` and :user:`sbraz`. :bug:`3192` :bug:`3225` -* Fix a problem when resizing images with PIL/Pillow on Python 3. +* Fix a problem when resizing images with :pypi:`PIL`/:pypi:`pillow` on Python 3. Thanks to :user:`architek`. :bug:`2504` :bug:`3029` -.. _python-itunes: https://github.com/ocelma/python-itunes - And there are many other fixes: * R128 normalization tags are now properly deleted from files when the values @@ -227,8 +227,8 @@ And there are many other fixes: * The ``%title`` template function now works correctly with apostrophes. Thanks to :user:`GuilhermeHideki`. :bug:`3033` -* :doc:`/plugins/lastgenre`: Allow to set the configuration option - ``prefer_specific`` without setting ``canonical``. +* :doc:`/plugins/lastgenre`: It's now possible to set the ``prefer_specific`` + option without also setting ``canonical``. :bug:`2973` * :doc:`/plugins/fetchart`: The plugin now respects the ``ignore`` and ``ignore_hidden`` settings. @@ -239,7 +239,7 @@ And there are many other fixes: hasn't. Thanks to :user:`arcresu`. :bug:`3195` -* :doc:`/plugins/lastgenre`: The `force` config option now actually works. +* :doc:`/plugins/lastgenre`: The ``force`` config option now actually works. :bug:`2704` :bug:`3054` For developers: @@ -252,8 +252,8 @@ For developers: For packagers: -* Note the changes to the dependencies on jellyfish and munkres. -* The optional python-itunes dependency has been removed. +* Note the changes to the dependencies on :pypi:`jellyfish` and :pypi:`munkres`. +* The optional :pypi:`python-itunes` dependency has been removed. * Python versions 3.7 and 3.8 are now supported. diff --git a/docs/conf.py b/docs/conf.py index c260ebb78..e9dcd0342 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,6 +24,8 @@ pygments_style = 'sphinx' extlinks = { 'bug': ('https://github.com/beetbox/beets/issues/%s', '#'), 'user': ('https://github.com/%s', ''), + 'pypi': ('https://pypi.org/project/%s/', ''), + 'stdlib': ('https://docs.python.org/3/library/%s.html', ''), } # Options for HTML output From f15f8a08f9b70ea0d5f092ee26003a18ae7d98dc Mon Sep 17 00:00:00 2001 From: ababyduck Date: Wed, 24 Apr 2019 05:30:17 -0700 Subject: [PATCH 203/339] Added fallback to ImageMagick's legacy utilities When the `magick` binary is not available, artresizer will fall back to the "legacy" binaries (`convert`, `identify`, etc.) --- beets/util/artresizer.py | 57 ++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 31554ceb9..a7e5ea170 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -81,7 +81,8 @@ def pil_resize(maxwidth, path_in, path_out=None): def im_resize(maxwidth, path_in, path_out=None): - """Resize using ImageMagick's ``convert`` tool. + """Resize using ImageMagick's ``magick`` tool + (or fall back to ``convert`` for older versions.) Return the output path of resized image. """ path_out = path_out or temp_file_for(path_in) @@ -92,11 +93,13 @@ def im_resize(maxwidth, path_in, path_out=None): # than the given width while maintaining the aspect ratio # with regards to the height. try: - util.command_output([ - 'magick', util.syspath(path_in, prefix=False), + cmds = (['magick'],['convert']) + cmd = cmds[0] if not im_legacy else cmds[1] + args = [util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), - util.syspath(path_out, prefix=False), - ]) + util.syspath(path_out, prefix=False)] + + util.command_output(cmd + args) except subprocess.CalledProcessError: log.warning(u'artresizer: IM convert failed for {0}', util.displayable_path(path_in)) @@ -121,10 +124,12 @@ def pil_getsize(path_in): def im_getsize(path_in): - cmd = ['magick', 'identify', '-format', '%w %h', - util.syspath(path_in, prefix=False)] + cmds = (['magick', 'identify'],['identify']) + cmd = cmds[0] if not im_legacy else cmds[1] + args = ['-format', '%w %h', util.syspath(path_in, prefix=False)] + try: - out = util.command_output(cmd) + out = util.command_output(cmd + args) except subprocess.CalledProcessError as exc: log.warning(u'ImageMagick size query failed') log.debug( @@ -229,26 +234,32 @@ class ArtResizer(six.with_metaclass(Shareable, object)): return WEBPROXY, (0) - +im_legacy = None def get_im_version(): """Return Image Magick version or None if it is unavailable - Try invoking ImageMagick's "convert". + Try invoking ImageMagick's "magick". If "magick" is unavailable, + as with older versions, fall back to "convert" """ - try: - out = util.command_output(['magick', '-version']) + cmds = ('magick','convert') + for isLegacy, cmd in enumerate(cmds): - if b'imagemagick' in out.lower(): - pattern = br".+ (\d+)\.(\d+)\.(\d+).*" - match = re.search(pattern, out) - if match: - return (int(match.group(1)), - int(match.group(2)), - int(match.group(3))) - return (0,) + try: + out = util.command_output([cmd, '--version']) - except (subprocess.CalledProcessError, OSError) as exc: - log.debug(u'ImageMagick check `convert --version` failed: {}', exc) - return None + if b'imagemagick' in out.lower(): + pattern = br".+ (\d+)\.(\d+)\.(\d+).*" + match = re.search(pattern, out) + if match: + im_legacy = bool(isLegacy) + return (int(match.group(1)), + int(match.group(2)), + int(match.group(3))) + + except (subprocess.CalledProcessError, OSError) as exc: + log.debug(u'ImageMagick version check failed: {}', exc) + return None + + return (0,) def get_pil_version(): From 1a6e0a7a29c21d4b62839a52c4c0e8b834cfcda0 Mon Sep 17 00:00:00 2001 From: ababyduck Date: Wed, 24 Apr 2019 06:18:43 -0700 Subject: [PATCH 204/339] Fix whitespace for flake8 compliance Fixed various whitespace issues and a global variable reference to comply with flake8 linting. --- beets/util/artresizer.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index a7e5ea170..dc5cff9aa 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -93,11 +93,11 @@ def im_resize(maxwidth, path_in, path_out=None): # than the given width while maintaining the aspect ratio # with regards to the height. try: - cmds = (['magick'],['convert']) + cmds = (['magick'], ['convert']) cmd = cmds[0] if not im_legacy else cmds[1] args = [util.syspath(path_in, prefix=False), - '-resize', '{0}x>'.format(maxwidth), - util.syspath(path_out, prefix=False)] + '-resize', '{0}x>'.format(maxwidth), + util.syspath(path_out, prefix=False)] util.command_output(cmd + args) except subprocess.CalledProcessError: @@ -124,7 +124,7 @@ def pil_getsize(path_in): def im_getsize(path_in): - cmds = (['magick', 'identify'],['identify']) + cmds = (['magick', 'identify'], ['identify']) cmd = cmds[0] if not im_legacy else cmds[1] args = ['-format', '%w %h', util.syspath(path_in, prefix=False)] @@ -234,13 +234,16 @@ class ArtResizer(six.with_metaclass(Shareable, object)): return WEBPROXY, (0) + im_legacy = None + + def get_im_version(): """Return Image Magick version or None if it is unavailable - Try invoking ImageMagick's "magick". If "magick" is unavailable, + Try invoking ImageMagick's "magick". If "magick" is unavailable, as with older versions, fall back to "convert" """ - cmds = ('magick','convert') + cmds = ('magick', 'convert') for isLegacy, cmd in enumerate(cmds): try: @@ -250,6 +253,7 @@ def get_im_version(): pattern = br".+ (\d+)\.(\d+)\.(\d+).*" match = re.search(pattern, out) if match: + global im_legacy im_legacy = bool(isLegacy) return (int(match.group(1)), int(match.group(2)), @@ -258,7 +262,7 @@ def get_im_version(): except (subprocess.CalledProcessError, OSError) as exc: log.debug(u'ImageMagick version check failed: {}', exc) return None - + return (0,) From 666790bd83be2afddb8395552d0da8c659d1068b Mon Sep 17 00:00:00 2001 From: ababyduck Date: Wed, 24 Apr 2019 09:07:38 -0700 Subject: [PATCH 205/339] Refactor to eliminate use of global variable `get_im_version` now returns an additional bool `isLegacy`, which indicates whether the the `magick` binary is accessible. It is stored in `self.im_legacy` on initialization of an `ArtResizer` object, and can be accessed via `ArtResizer.shared.im_legacy` --- beets/util/artresizer.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index dc5cff9aa..49dc21a93 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -94,7 +94,7 @@ def im_resize(maxwidth, path_in, path_out=None): # with regards to the height. try: cmds = (['magick'], ['convert']) - cmd = cmds[0] if not im_legacy else cmds[1] + cmd = cmds[0] if not ArtResizer.shared.im_legacy else cmds[1] args = [util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), util.syspath(path_out, prefix=False)] @@ -125,7 +125,7 @@ def pil_getsize(path_in): def im_getsize(path_in): cmds = (['magick', 'identify'], ['identify']) - cmd = cmds[0] if not im_legacy else cmds[1] + cmd = cmds[0] if not ArtResizer.shared.im_legacy else cmds[1] args = ['-format', '%w %h', util.syspath(path_in, prefix=False)] try: @@ -178,6 +178,9 @@ class ArtResizer(six.with_metaclass(Shareable, object)): log.debug(u"artresizer: method is {0}", self.method) self.can_compare = self._can_compare() + if self.method[0] == IMAGEMAGICK: + self.im_legacy = self.method[2] + def resize(self, maxwidth, path_in, path_out=None): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a @@ -224,9 +227,9 @@ class ArtResizer(six.with_metaclass(Shareable, object)): @staticmethod def _check_method(): """Return a tuple indicating an available method and its version.""" - version = get_im_version() + version, isLegacy = get_im_version() if version: - return IMAGEMAGICK, version + return IMAGEMAGICK, version, isLegacy version = get_pil_version() if version: @@ -235,13 +238,13 @@ class ArtResizer(six.with_metaclass(Shareable, object)): return WEBPROXY, (0) -im_legacy = None - - def get_im_version(): """Return Image Magick version or None if it is unavailable Try invoking ImageMagick's "magick". If "magick" is unavailable, as with older versions, fall back to "convert" + + Our iterator `isLegacy` will be non-zero when the first command + fails, and will be returned in a tuple along with the version """ cmds = ('magick', 'convert') for isLegacy, cmd in enumerate(cmds): @@ -253,11 +256,11 @@ def get_im_version(): pattern = br".+ (\d+)\.(\d+)\.(\d+).*" match = re.search(pattern, out) if match: - global im_legacy - im_legacy = bool(isLegacy) - return (int(match.group(1)), + return ((int(match.group(1)), int(match.group(2)), - int(match.group(3))) + int(match.group(3))), + bool(isLegacy) + ) except (subprocess.CalledProcessError, OSError) as exc: log.debug(u'ImageMagick version check failed: {}', exc) From e00640e7dd4d8feca8070ea1314e79d7f8f226c2 Mon Sep 17 00:00:00 2001 From: ababyduck Date: Wed, 24 Apr 2019 09:35:56 -0700 Subject: [PATCH 206/339] Handle TypeError exception when no ImageMagick install is present Fixes an error introduced in 1a6e0a7 where a TypeError exception was raised when calling `_check_method()` with no ImageMagick installation present. --- beets/util/artresizer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 49dc21a93..ad1521bcf 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -227,9 +227,12 @@ class ArtResizer(six.with_metaclass(Shareable, object)): @staticmethod def _check_method(): """Return a tuple indicating an available method and its version.""" - version, isLegacy = get_im_version() - if version: - return IMAGEMAGICK, version, isLegacy + try: + version, isLegacy = get_im_version() + if version: + return IMAGEMAGICK, version, isLegacy + except TypeError: + pass version = get_pil_version() if version: From a8f137bf9571c0671ce02e359c98c9c3aae90318 Mon Sep 17 00:00:00 2001 From: ababyduck Date: Wed, 24 Apr 2019 10:09:31 -0700 Subject: [PATCH 207/339] Rename `isLegacy` to `im_legacy` for consistency and flake8 compliance --- beets/util/artresizer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index ad1521bcf..5e3e98386 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -228,9 +228,9 @@ class ArtResizer(six.with_metaclass(Shareable, object)): def _check_method(): """Return a tuple indicating an available method and its version.""" try: - version, isLegacy = get_im_version() + version, im_legacy = get_im_version() if version: - return IMAGEMAGICK, version, isLegacy + return IMAGEMAGICK, version, im_legacy except TypeError: pass @@ -246,11 +246,11 @@ def get_im_version(): Try invoking ImageMagick's "magick". If "magick" is unavailable, as with older versions, fall back to "convert" - Our iterator `isLegacy` will be non-zero when the first command + Our iterator `im_legacy` will be non-zero when the first command fails, and will be returned in a tuple along with the version """ cmds = ('magick', 'convert') - for isLegacy, cmd in enumerate(cmds): + for im_legacy, cmd in enumerate(cmds): try: out = util.command_output([cmd, '--version']) @@ -262,7 +262,7 @@ def get_im_version(): return ((int(match.group(1)), int(match.group(2)), int(match.group(3))), - bool(isLegacy) + bool(im_legacy) ) except (subprocess.CalledProcessError, OSError) as exc: From 3f3b10288506de68c5a181f25c31fb8eb4ee7ee8 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Wed, 24 Apr 2019 19:53:01 +0200 Subject: [PATCH 208/339] added PaddingFloat --- beets/dbcore/types.py | 11 +++++++++++ beetsplug/acousticbrainz.py | 14 +++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index 935d03870..bc923f747 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -183,6 +183,17 @@ class Float(Type): return u'{0:.1f}'.format(value or 0.0) +class PaddedFloat(Integer): + """A float field that is formatted with a given number of digits, + padded with zeroes. + """ + def __init__(self, digits): + self.digits = digits + + def format(self, value): + return u'{0:0{1}d}'.format(value or 0, self.digits) + + class NullFloat(Float): """Same as `Float`, but does not normalize `None` to `0.0`. """ diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 3a6db7d4e..9c1eb7ce5 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -117,13 +117,13 @@ class AcousticPlugin(plugins.BeetsPlugin): 'genre_rosamerica': types.STRING, 'initial_key': types.STRING, 'key_strength': types.FLOAT, - 'mood_acoustic': types.FLOAT, - 'mood_aggressive': types.FLOAT, - 'mood_electronic': types.FLOAT, - 'mood_happy': types.FLOAT, - 'mood_party': types.FLOAT, - 'mood_relaxed': types.FLOAT, - 'mood_sad': types.FLOAT, + 'mood_acoustic': types.PaddedInt(6), + 'mood_aggressive': types.PaddedInt(6), + 'mood_electronic': types.PaddedInt(6), + 'mood_happy': types.PaddedInt(6), + 'mood_party': types.PaddedInt(6), + 'mood_relaxed': types.PaddedInt(6), + 'mood_sad': types.PaddedInt(6), 'rhythm': types.FLOAT, 'tonal': types.FLOAT, 'voice_instrumental': types.STRING, From 7676d2ae5a427d4fa1dd03c45a99bace78b1ee2e Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Wed, 24 Apr 2019 19:54:12 +0200 Subject: [PATCH 209/339] PaddingInt -> PaddingFloat --- beetsplug/acousticbrainz.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 9c1eb7ce5..5278416b9 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -117,13 +117,13 @@ class AcousticPlugin(plugins.BeetsPlugin): 'genre_rosamerica': types.STRING, 'initial_key': types.STRING, 'key_strength': types.FLOAT, - 'mood_acoustic': types.PaddedInt(6), - 'mood_aggressive': types.PaddedInt(6), - 'mood_electronic': types.PaddedInt(6), - 'mood_happy': types.PaddedInt(6), - 'mood_party': types.PaddedInt(6), - 'mood_relaxed': types.PaddedInt(6), - 'mood_sad': types.PaddedInt(6), + 'mood_acoustic': types.PaddedFloat(6), + 'mood_aggressive': types.PaddedFloat(6), + 'mood_electronic': types.PaddedFloat(6), + 'mood_happy': types.PaddedFloat(6), + 'mood_party': types.PaddedFloat(6), + 'mood_relaxed': types.PaddedFloat(6), + 'mood_sad': types.PaddedFloat(6), 'rhythm': types.FLOAT, 'tonal': types.FLOAT, 'voice_instrumental': types.STRING, From 07b617b307344259f5f62e4de81e6f8c20e71a16 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Wed, 24 Apr 2019 19:55:44 +0200 Subject: [PATCH 210/339] Converted all float types to PaddedFloat --- beetsplug/acousticbrainz.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 5278416b9..11a67eadd 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -107,16 +107,16 @@ ABSCHEME = { class AcousticPlugin(plugins.BeetsPlugin): item_types = { - 'average_loudness': types.FLOAT, - 'chords_changes_rate': types.FLOAT, + 'average_loudness': types.PaddedFloat(6), + 'chords_changes_rate': types.PaddedFloat(6), 'chords_key': types.STRING, - 'chords_number_rate': types.FLOAT, + 'chords_number_rate': types.PaddedFloat(6), 'chords_scale': types.STRING, - 'danceable': types.FLOAT, + 'danceable': types.PaddedFloat(6), 'gender': types.STRING, 'genre_rosamerica': types.STRING, 'initial_key': types.STRING, - 'key_strength': types.FLOAT, + 'key_strength': types.PaddedFloat(6), 'mood_acoustic': types.PaddedFloat(6), 'mood_aggressive': types.PaddedFloat(6), 'mood_electronic': types.PaddedFloat(6), @@ -124,8 +124,8 @@ class AcousticPlugin(plugins.BeetsPlugin): 'mood_party': types.PaddedFloat(6), 'mood_relaxed': types.PaddedFloat(6), 'mood_sad': types.PaddedFloat(6), - 'rhythm': types.FLOAT, - 'tonal': types.FLOAT, + 'rhythm': types.PaddedFloat(6), + 'tonal': types.PaddedFloat(6), 'voice_instrumental': types.STRING, } From 7240a826bc7e2a038501b4c5672442590cdc451f Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Wed, 24 Apr 2019 20:05:26 +0200 Subject: [PATCH 211/339] Fixed PaddedFloat-type and set precision to 8 --- beets/dbcore/types.py | 4 ++-- beetsplug/acousticbrainz.py | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index bc923f747..fe83f68b2 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -183,7 +183,7 @@ class Float(Type): return u'{0:.1f}'.format(value or 0.0) -class PaddedFloat(Integer): +class PaddedFloat(Float): """A float field that is formatted with a given number of digits, padded with zeroes. """ @@ -191,7 +191,7 @@ class PaddedFloat(Integer): self.digits = digits def format(self, value): - return u'{0:0{1}d}'.format(value or 0, self.digits) + return u'{0:.{1}f}'.format(value or 0, self.digits) class NullFloat(Float): diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 11a67eadd..cdd6a6b83 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -107,25 +107,25 @@ ABSCHEME = { class AcousticPlugin(plugins.BeetsPlugin): item_types = { - 'average_loudness': types.PaddedFloat(6), - 'chords_changes_rate': types.PaddedFloat(6), + 'average_loudness': types.PaddedFloat(8), + 'chords_changes_rate': types.PaddedFloat(8), 'chords_key': types.STRING, - 'chords_number_rate': types.PaddedFloat(6), + 'chords_number_rate': types.PaddedFloat(8), 'chords_scale': types.STRING, - 'danceable': types.PaddedFloat(6), + 'danceable': types.PaddedFloat(8), 'gender': types.STRING, 'genre_rosamerica': types.STRING, 'initial_key': types.STRING, - 'key_strength': types.PaddedFloat(6), - 'mood_acoustic': types.PaddedFloat(6), - 'mood_aggressive': types.PaddedFloat(6), - 'mood_electronic': types.PaddedFloat(6), - 'mood_happy': types.PaddedFloat(6), - 'mood_party': types.PaddedFloat(6), - 'mood_relaxed': types.PaddedFloat(6), - 'mood_sad': types.PaddedFloat(6), - 'rhythm': types.PaddedFloat(6), - 'tonal': types.PaddedFloat(6), + 'key_strength': types.PaddedFloat(8), + 'mood_acoustic': types.PaddedFloat(8), + 'mood_aggressive': types.PaddedFloat(8), + 'mood_electronic': types.PaddedFloat(8), + 'mood_happy': types.PaddedFloat(8), + 'mood_party': types.PaddedFloat(8), + 'mood_relaxed': types.PaddedFloat(8), + 'mood_sad': types.PaddedFloat(8), + 'rhythm': types.PaddedFloat(8), + 'tonal': types.PaddedFloat(8), 'voice_instrumental': types.STRING, } From 55fe077e549899743352f1528b17646034f41606 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Wed, 24 Apr 2019 20:39:05 +0200 Subject: [PATCH 212/339] Removed PaddedFloat in favour of adding a constructor parameter --- beets/dbcore/types.py | 14 +++----------- beetsplug/acousticbrainz.py | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index bc923f747..2a06c58fc 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -173,25 +173,17 @@ class Id(Integer): class Float(Type): - """A basic floating-point type. + """A basic floating-point type. Supports padding. """ sql = u'REAL' query = query.NumericQuery model_type = float - def format(self, value): - return u'{0:.1f}'.format(value or 0.0) - - -class PaddedFloat(Integer): - """A float field that is formatted with a given number of digits, - padded with zeroes. - """ - def __init__(self, digits): + def __init__(self, digits=1): self.digits = digits def format(self, value): - return u'{0:0{1}d}'.format(value or 0, self.digits) + return u'{0:0{1}f}'.format(value or 0, self.digits) class NullFloat(Float): diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 11a67eadd..4c4315b80 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -107,25 +107,25 @@ ABSCHEME = { class AcousticPlugin(plugins.BeetsPlugin): item_types = { - 'average_loudness': types.PaddedFloat(6), - 'chords_changes_rate': types.PaddedFloat(6), + 'average_loudness': types.Float(6), + 'chords_changes_rate': types.Float(6), 'chords_key': types.STRING, - 'chords_number_rate': types.PaddedFloat(6), + 'chords_number_rate': types.Float(6), 'chords_scale': types.STRING, - 'danceable': types.PaddedFloat(6), + 'danceable': types.Float(6), 'gender': types.STRING, 'genre_rosamerica': types.STRING, 'initial_key': types.STRING, - 'key_strength': types.PaddedFloat(6), - 'mood_acoustic': types.PaddedFloat(6), - 'mood_aggressive': types.PaddedFloat(6), - 'mood_electronic': types.PaddedFloat(6), - 'mood_happy': types.PaddedFloat(6), - 'mood_party': types.PaddedFloat(6), - 'mood_relaxed': types.PaddedFloat(6), - 'mood_sad': types.PaddedFloat(6), - 'rhythm': types.PaddedFloat(6), - 'tonal': types.PaddedFloat(6), + 'key_strength': types.Float(6), + 'mood_acoustic': types.Float(6), + 'mood_aggressive': types.Float(6), + 'mood_electronic': types.Float(6), + 'mood_happy': types.Float(6), + 'mood_party': types.Float(6), + 'mood_relaxed': types.Float(6), + 'mood_sad': types.Float(6), + 'rhythm': types.Float(6), + 'tonal': types.Float(6), 'voice_instrumental': types.STRING, } From 48be3a7eafd9534f210ff9c534d6b506127f3bc4 Mon Sep 17 00:00:00 2001 From: ababyduck Date: Wed, 24 Apr 2019 18:43:50 -0700 Subject: [PATCH 213/339] Make requested changes for PR 3236 - Refactored convert and identify command names to an ArtResizer member variable, set on ArtResizer init. Functions that use this info will now access it from there. - Changed the way `cmd` variables are written so that the command name and command args are assigned directly to `cmd`, rather than doing `command_output(cmd + args)` - `get_im_version()` will now always return a tuple containing two values: a tuple representing the version, and either a bool or None flag representing whether we should send legacy commands to ImageMagick - Improved readability of successful return value in `get_im_version()` --- beets/util/artresizer.py | 47 ++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 5e3e98386..427a5411d 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -93,13 +93,12 @@ def im_resize(maxwidth, path_in, path_out=None): # than the given width while maintaining the aspect ratio # with regards to the height. try: - cmds = (['magick'], ['convert']) - cmd = cmds[0] if not ArtResizer.shared.im_legacy else cmds[1] - args = [util.syspath(path_in, prefix=False), + cmd = ArtResizer.shared.im_convert_cmd + \ + [util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), util.syspath(path_out, prefix=False)] - util.command_output(cmd + args) + util.command_output(cmd) except subprocess.CalledProcessError: log.warning(u'artresizer: IM convert failed for {0}', util.displayable_path(path_in)) @@ -124,12 +123,11 @@ def pil_getsize(path_in): def im_getsize(path_in): - cmds = (['magick', 'identify'], ['identify']) - cmd = cmds[0] if not ArtResizer.shared.im_legacy else cmds[1] - args = ['-format', '%w %h', util.syspath(path_in, prefix=False)] - try: - out = util.command_output(cmd + args) + cmd = ArtResizer.shared.im_identify_cmd + \ + ['-format', '%w %h', util.syspath(path_in, prefix=False)] + + out = util.command_output(cmd) except subprocess.CalledProcessError as exc: log.warning(u'ImageMagick size query failed') log.debug( @@ -180,6 +178,12 @@ class ArtResizer(six.with_metaclass(Shareable, object)): if self.method[0] == IMAGEMAGICK: self.im_legacy = self.method[2] + if self.im_legacy: + self.im_convert_cmd = ['convert'] + self.im_identify_cmd = ['identify'] + else: + self.im_convert_cmd = ['magick'] + self.im_identify_cmd = ['magick', 'identify'] def resize(self, maxwidth, path_in, path_out=None): """Manipulate an image file according to the method, returning a @@ -246,30 +250,31 @@ def get_im_version(): Try invoking ImageMagick's "magick". If "magick" is unavailable, as with older versions, fall back to "convert" - Our iterator `im_legacy` will be non-zero when the first command - fails, and will be returned in a tuple along with the version + Our iterator will be non-zero when the first command fails, and will + be returned in a tuple along with the version. """ - cmds = ('magick', 'convert') - for im_legacy, cmd in enumerate(cmds): + cmd_names = (['magick'], + ['convert']) + for i, cmd_name in enumerate(cmd_names): try: - out = util.command_output([cmd, '--version']) + cmd = cmd_name + ['--version'] + out = util.command_output(cmd) if b'imagemagick' in out.lower(): pattern = br".+ (\d+)\.(\d+)\.(\d+).*" match = re.search(pattern, out) + version = (int(match.group(1)), + int(match.group(2)), + int(match.group(3))) + legacy = bool(i) if match: - return ((int(match.group(1)), - int(match.group(2)), - int(match.group(3))), - bool(im_legacy) - ) + return (version, legacy) except (subprocess.CalledProcessError, OSError) as exc: log.debug(u'ImageMagick version check failed: {}', exc) - return None - return (0,) + return (0, None) def get_pil_version(): From 09abd9802aa337f20ba83c3a0a6155cbf964b7c4 Mon Sep 17 00:00:00 2001 From: ababyduck Date: Thu, 25 Apr 2019 05:50:11 -0700 Subject: [PATCH 214/339] Make `get_im_version()` return same types across all conditions `get_im_version` should now always return a tuple containing: - index 0: a tuple representing the version - index 1: a bool or None, representing legacy status --- beets/util/artresizer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 427a5411d..7ae5fd63f 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -232,9 +232,9 @@ class ArtResizer(six.with_metaclass(Shareable, object)): def _check_method(): """Return a tuple indicating an available method and its version.""" try: - version, im_legacy = get_im_version() - if version: - return IMAGEMAGICK, version, im_legacy + version, legacy = get_im_version() + if version > (0, 0, 0): + return IMAGEMAGICK, version, legacy except TypeError: pass @@ -274,7 +274,7 @@ def get_im_version(): except (subprocess.CalledProcessError, OSError) as exc: log.debug(u'ImageMagick version check failed: {}', exc) - return (0, None) + return ((0,), None) def get_pil_version(): From 278d87f25aec3e59a197b6d1d5f8325707edab8e Mon Sep 17 00:00:00 2001 From: ababyduck Date: Thu, 25 Apr 2019 07:50:12 -0700 Subject: [PATCH 215/339] Make more requested changes for PR 3236 - Moved several variable assignments outside of try blocks - Added and clarified various comments and docstrings - Modified the command loop in `get_im_version()` to a slightly more readable approach - `get_im_version()` now returns None when ImageMagick is unavailable - Updated `ArtResizer._check_method` to handle our new returns in a way that is more readable - Fixed an issue where `get_im_version()` could crash if the regex search failed to find a match --- beets/util/artresizer.py | 67 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 7ae5fd63f..5a0036376 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -82,7 +82,7 @@ def pil_resize(maxwidth, path_in, path_out=None): def im_resize(maxwidth, path_in, path_out=None): """Resize using ImageMagick's ``magick`` tool - (or fall back to ``convert`` for older versions.) + (or fall back to ``convert`` for older versions). Return the output path of resized image. """ path_out = path_out or temp_file_for(path_in) @@ -92,17 +92,18 @@ def im_resize(maxwidth, path_in, path_out=None): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. - try: - cmd = ArtResizer.shared.im_convert_cmd + \ - [util.syspath(path_in, prefix=False), - '-resize', '{0}x>'.format(maxwidth), - util.syspath(path_out, prefix=False)] + cmd = ArtResizer.shared.im_convert_cmd + \ + [util.syspath(path_in, prefix=False), + '-resize', '{0}x>'.format(maxwidth), + util.syspath(path_out, prefix=False)] + try: util.command_output(cmd) except subprocess.CalledProcessError: log.warning(u'artresizer: IM convert failed for {0}', util.displayable_path(path_in)) return path_in + return path_out @@ -123,10 +124,10 @@ def pil_getsize(path_in): def im_getsize(path_in): - try: - cmd = ArtResizer.shared.im_identify_cmd + \ - ['-format', '%w %h', util.syspath(path_in, prefix=False)] + cmd = ArtResizer.shared.im_identify_cmd + \ + ['-format', '%w %h', util.syspath(path_in, prefix=False)] + try: out = util.command_output(cmd) except subprocess.CalledProcessError as exc: log.warning(u'ImageMagick size query failed') @@ -176,6 +177,9 @@ class ArtResizer(six.with_metaclass(Shareable, object)): log.debug(u"artresizer: method is {0}", self.method) self.can_compare = self._can_compare() + # Use ImageMagick's magick binary when it's available. If it's + # not, fall back to the older, separate convert and identify + # commands. if self.method[0] == IMAGEMAGICK: self.im_legacy = self.method[2] if self.im_legacy: @@ -230,13 +234,14 @@ class ArtResizer(six.with_metaclass(Shareable, object)): @staticmethod def _check_method(): - """Return a tuple indicating an available method and its version.""" - try: - version, legacy = get_im_version() - if version > (0, 0, 0): - return IMAGEMAGICK, version, legacy - except TypeError: - pass + """Return a tuple indicating an available method and its version. + If the method is ImageMagick, also return a bool indicating whether to + use the `magick` binary or legacy utils (`convert`, `identify`, etc.) + """ + version = get_im_version() + if version: + version, legacy = version + return IMAGEMAGICK, version, legacy version = get_pil_version() if version: @@ -246,40 +251,36 @@ class ArtResizer(six.with_metaclass(Shareable, object)): def get_im_version(): - """Return Image Magick version or None if it is unavailable - Try invoking ImageMagick's "magick". If "magick" is unavailable, - as with older versions, fall back to "convert" + """Return ImageMagick version/legacy-flag pair or None if the check fails. - Our iterator will be non-zero when the first command fails, and will - be returned in a tuple along with the version. + Try invoking ImageMagick's `magick` binary first, then `convert` if + `magick` is unavailable. """ - cmd_names = (['magick'], - ['convert']) - for i, cmd_name in enumerate(cmd_names): + for cmd_name, legacy in ((['magick'], False), (['convert'], True)): + cmd = cmd_name + ['--version'] try: - cmd = cmd_name + ['--version'] out = util.command_output(cmd) if b'imagemagick' in out.lower(): pattern = br".+ (\d+)\.(\d+)\.(\d+).*" match = re.search(pattern, out) - version = (int(match.group(1)), - int(match.group(2)), - int(match.group(3))) - legacy = bool(i) if match: - return (version, legacy) + version = (int(match.group(1)), + int(match.group(2)), + int(match.group(3))) + return version, legacy except (subprocess.CalledProcessError, OSError) as exc: log.debug(u'ImageMagick version check failed: {}', exc) - return ((0,), None) + return None def get_pil_version(): - """Return Image Magick version or None if it is unavailable - Try importing PIL.""" + """Return Pillow version or None if it is unavailable + Try importing PIL. + """ try: __import__('PIL', fromlist=[str('Image')]) return (0,) From 2e0230789f86a44adf04cad830224c208ca41a9f Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 27 Apr 2019 12:43:13 +1000 Subject: [PATCH 216/339] docs: fix some markup issues and typos --- docs/conf.py | 2 +- docs/plugins/absubmit.rst | 6 +++--- docs/plugins/badfiles.rst | 4 ++-- docs/plugins/bpd.rst | 2 +- docs/plugins/bucket.rst | 18 +++++++++--------- docs/plugins/convert.rst | 2 +- docs/plugins/embyupdate.rst | 2 +- docs/plugins/export.rst | 2 +- docs/plugins/fetchart.rst | 10 +++++----- docs/plugins/freedesktop.rst | 2 +- docs/plugins/gmusic.rst | 2 +- docs/plugins/index.rst | 14 +++++++------- docs/plugins/lyrics.rst | 4 ++-- docs/plugins/mbsync.rst | 2 +- docs/plugins/play.rst | 4 ++-- docs/plugins/replaygain.rst | 4 ++-- docs/plugins/smartplaylist.rst | 2 +- docs/plugins/thumbnails.rst | 2 +- 18 files changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e9dcd0342..7e52d283e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ release = '1.4.8' pygments_style = 'sphinx' -# External links to the bug tracker. +# External links to the bug tracker and other sites. extlinks = { 'bug': ('https://github.com/beetbox/beets/issues/%s', '#'), 'user': ('https://github.com/%s', ''), diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index feebc1642..30a77d4b0 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -1,15 +1,15 @@ AcousticBrainz Submit Plugin ============================ -The `absubmit` plugin lets you submit acoustic analysis results to the +The ``absubmit`` plugin lets you submit acoustic analysis results to the `AcousticBrainz`_ server. Installation ------------ -The `absubmit` plugin requires the `streaming_extractor_music`_ program to run. Its source can be found on `GitHub`_, and while it is possible to compile the extractor from source, AcousticBrainz would prefer if you used their binary (see the AcousticBrainz `FAQ`_). +The ``absubmit`` plugin requires the `streaming_extractor_music`_ program to run. Its source can be found on `GitHub`_, and while it is possible to compile the extractor from source, AcousticBrainz would prefer if you used their binary (see the AcousticBrainz `FAQ`_). -The `absubmit` also plugin requires `requests`_, which you can install using `pip`_ by typing:: +The ``absubmit`` plugin also requires `requests`_, which you can install using `pip`_ by typing:: pip install requests diff --git a/docs/plugins/badfiles.rst b/docs/plugins/badfiles.rst index 0a32f1a36..a59dbd0d1 100644 --- a/docs/plugins/badfiles.rst +++ b/docs/plugins/badfiles.rst @@ -48,11 +48,11 @@ Here is an example where the FLAC decoder signals a corrupt file:: 00.flac: ERROR while decoding data state = FLAC__STREAM_DECODER_READ_FRAME -Note that the default `mp3val` checker is a bit verbose and can output a lot +Note that the default ``mp3val`` checker is a bit verbose and can output a lot of "stream error" messages, even for files that play perfectly well. Generally, if more than one stream error happens, or if a stream error happens in the middle of a file, this is a bad sign. By default, only errors for the bad files will be shown. In order for the -results for all of the checked files to be seen, including the uncorrupted +results for all of the checked files to be seen, including the uncorrupted ones, use the ``-v`` or ``--verbose`` option. diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index ee36c040c..87c931793 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -20,7 +20,7 @@ with its Python bindings) on your system. gst-plugins-base pygobject3``. * On Linux, you need to install GStreamer 1.0 and the GObject bindings for - python. Under Ubuntu, they are called `python-gi` and `gstreamer1.0`. + python. Under Ubuntu, they are called ``python-gi`` and ``gstreamer1.0``. * On Windows, you may want to try `GStreamer WinBuilds`_ (caveat emptor: I haven't tried this). diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst index 99975968f..ee1857777 100644 --- a/docs/plugins/bucket.rst +++ b/docs/plugins/bucket.rst @@ -27,19 +27,19 @@ The ``bucket_year`` parameter is used for all substitutions occurring on the The definition of a range is somewhat loose, and multiple formats are allowed: - For alpha ranges: the range is defined by the lowest and highest (ASCII-wise) - alphanumeric characters in the string you provide. For example, *ABCD*, - *A-D*, *A->D*, and *[AD]* are all equivalent. + alphanumeric characters in the string you provide. For example, ``ABCD``, + ``A-D``, ``A->D``, and ``[AD]`` are all equivalent. - For year ranges: digits characters are extracted and the two extreme years - define the range. For example, *1975-77*, *1975,76,77* and *1975-1977* are + define the range. For example, ``1975-77``, ``1975,76,77`` and ``1975-1977`` are equivalent. If no upper bound is given, the range is extended to current year - (unless a later range is defined). For example, *1975* encompasses all years + (unless a later range is defined). For example, ``1975`` encompasses all years from 1975 until now. -The `%bucket` template function guesses whether to use alpha- or year-style +The ``%bucket`` template function guesses whether to use alpha- or year-style buckets depending on the text it receives. It can guess wrong if, for example, -an artist or album happens to begin with four digits. Provide `alpha` as the +an artist or album happens to begin with four digits. Provide ``alpha`` as the second argument to the template to avoid this automatic detection: for -example, use `%bucket{$artist,alpha}`. +example, use ``%bucket{$artist,alpha}``. Configuration @@ -56,7 +56,7 @@ The available options are: overrides original range definition. Default: none. - **bucket_year**: Ranges to use for all substitutions occurring on the - `$year` field. + ``$year`` field. Default: none. - **extrapolate**: Enable this if you want to group your files into multiple year ranges without enumerating them all. This option will generate year @@ -73,5 +73,5 @@ Here's an example:: 'A - D': ^[0-9a-dA-D…äÄ] This configuration creates five-year ranges for any input year. -The *A - D* bucket now matches also all artists starting with ä or Ä and 0 to 9 +The `A - D` bucket now matches also all artists starting with ä or Ä and 0 to 9 and … (ellipsis). The other alpha buckets work as ranges. diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 1a487cdee..92545af30 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -122,7 +122,7 @@ and select a command with the ``--format`` command-line option or the In this example ``beet convert`` will use the *speex* command by default. To convert the audio to `wav`, run ``beet convert -f wav``. -This will also use the format key (`wav`) as the file extension. +This will also use the format key (``wav``) as the file extension. Each entry in the ``formats`` map consists of a key (the name of the format) as well as the command and optionally the file extension. diff --git a/docs/plugins/embyupdate.rst b/docs/plugins/embyupdate.rst index 00373b98c..d820f5c6b 100644 --- a/docs/plugins/embyupdate.rst +++ b/docs/plugins/embyupdate.rst @@ -34,5 +34,5 @@ The available options under the ``emby:`` section are: - **password**: The password for the user. (This is only necessary if no API key is provided.) -You can choose to authenticate either with `apikey` or `password`, but only +You can choose to authenticate either with ``apikey`` or ``password``, but only one of those two is required. diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 4192f00c5..4326ccb16 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -42,7 +42,7 @@ Configuration To configure the plugin, make a ``export:`` section in your configuration file. Under the ``json`` key, these options are available: -- **ensure_ascii**: Escape non-ASCII characters with `\uXXXX` entities. +- **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. - **indent**: The number of spaces for indentation. diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 002471ec1..8af7f686a 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -73,18 +73,18 @@ or `Pillow`_. .. note:: - Previously, there was a `remote_priority` option to specify when to + Previously, there was a ``remote_priority`` option to specify when to look for art on the filesystem. This is still respected, but a deprecation message will be shown until you - replace this configuration with the new `filesystem` value in the - `sources` array. + replace this configuration with the new ``filesystem`` value in the + ``sources`` array. .. _beets custom search engine: https://cse.google.com.au:443/cse/publicurl?cx=001442825323518660753:hrh5ch1gjzm .. _Pillow: https://github.com/python-pillow/Pillow .. _ImageMagick: http://www.imagemagick.org/ -Here's an example that makes plugin select only images that contain *front* or -*back* keywords in their filenames and prioritizes the iTunes source over +Here's an example that makes plugin select only images that contain ``front`` or +``back`` keywords in their filenames and prioritizes the iTunes source over others:: fetchart: diff --git a/docs/plugins/freedesktop.rst b/docs/plugins/freedesktop.rst index 61943718e..0368cc5da 100644 --- a/docs/plugins/freedesktop.rst +++ b/docs/plugins/freedesktop.rst @@ -3,4 +3,4 @@ Freedesktop Plugin The ``freedesktop`` plugin created .directory files in your album folders. This plugin is now deprecated and replaced by the :doc:`/plugins/thumbnails` -with the `dolphin` option enabled. +with the ``dolphin`` option enabled. diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index a4f4c8e05..94ee2dae4 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -8,7 +8,7 @@ songs in your library. Installation ------------ -The plugin requires `gmusicapi`_. You can install it using `pip`:: +The plugin requires :pypi:`gmusicapi`. You can install it using ``pip``:: pip install gmusicapi diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index e885db39b..e75e2f810 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -13,11 +13,11 @@ Using Plugins ------------- To use one of the plugins included with beets (see the rest of this page for a -list), just use the `plugins` option in your :doc:`config.yaml ` file, like so:: +list), just use the ``plugins`` option in your :doc:`config.yaml ` file, like so:: plugins: inline convert web -The value for `plugins` can be a space-separated list of plugin names or a +The value for ``plugins`` can be a space-separated list of plugin names or a YAML list like ``[foo, bar]``. You can see which plugins are currently enabled by typing ``beet version``. @@ -30,7 +30,7 @@ Each plugin has its own set of options that can be defined in a section bearing Some plugins have special dependencies that you'll need to install. The documentation page for each plugin will list them in the setup instructions. -For some, you can use `pip`'s "extras" feature to install the dependencies, +For some, you can use ``pip``'s "extras" feature to install the dependencies, like this:: pip install beets[fetchart,lyrics,lastgenre] @@ -213,14 +213,14 @@ In addition to the plugins that come with beets, there are several plugins that are maintained by the beets community. To use an external plugin, there are two options for installation: -* Make sure it's in the Python path (known as `sys.path` to developers). This +* Make sure it's in the Python path (known as ``sys.path`` to developers). This just means the plugin has to be installed on your system (e.g., with a - `setup.py` script or a command like `pip` or `easy_install`). + ``setup.py`` script or a command like ``pip`` or ``easy_install``). -* Set the `pluginpath` config variable to point to the directory containing the +* Set the ``pluginpath`` config variable to point to the directory containing the plugin. (See :doc:`/reference/config`.) -Once the plugin is installed, enable it by placing its name on the `plugins` +Once the plugin is installed, enable it by placing its name on the ``plugins`` line in your config file. Here are a few of the plugins written by the beets community: diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 4131c85d9..799bd0325 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -60,9 +60,9 @@ configuration file. The available options are: - **sources**: List of sources to search for lyrics. An asterisk ``*`` expands to all available sources. Default: ``google lyricwiki musixmatch genius``, i.e., all the - available sources. The `google` source will be automatically + available sources. The ``google`` source will be automatically deactivated if no ``google_API_key`` is setup. - Both it and the `genius` source will only be enabled if BeautifulSoup is + Both it and the ``genius`` source will only be enabled if BeautifulSoup is installed. Here's an example of ``config.yaml``:: diff --git a/docs/plugins/mbsync.rst b/docs/plugins/mbsync.rst index a7633a500..1c8663dca 100644 --- a/docs/plugins/mbsync.rst +++ b/docs/plugins/mbsync.rst @@ -30,7 +30,7 @@ The command has a few command-line options: * By default, files will be moved (renamed) according to their metadata if they are inside your beets library directory. To disable this, use the ``-M`` (``--nomove``) command-line option. -* If you have the `import.write` configuration option enabled, then this +* If you have the ``import.write`` configuration option enabled, then this plugin will write new metadata to files' tags. To disable this, use the ``-W`` (``--nowrite``) option. * To customize the output of unrecognized items, use the ``-f`` diff --git a/docs/plugins/play.rst b/docs/plugins/play.rst index 86920c9ac..d72ec4e0d 100644 --- a/docs/plugins/play.rst +++ b/docs/plugins/play.rst @@ -32,8 +32,8 @@ command-line oriented, and you'll get its output in real time. Interactive Usage ----------------- -The `play` plugin can also be invoked during an import. If enabled, the plugin -adds a `plaY` option to the prompt, so pressing `y` will execute the configured +The ``play`` plugin can also be invoked during an import. If enabled, the plugin +adds a ``plaY`` option to the prompt, so pressing ``y`` will execute the configured command and play the items currently being imported. Once the configured command exits, you will be returned to the import diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index ad0e50e22..825f279e2 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -24,9 +24,9 @@ GStreamer To use `GStreamer`_ for ReplayGain analysis, you will of course need to install GStreamer and plugins for compatibility with your audio files. -You will need at least GStreamer 1.0 and `PyGObject 3.x`_ (a.k.a. python-gi). +You will need at least GStreamer 1.0 and `PyGObject 3.x`_ (a.k.a. ``python-gi``). -.. _PyGObject 3.x: https://wiki.gnome.org/action/show/Projects/PyGObject +.. _PyGObject 3.x: https://pygobject.readthedocs.io/en/latest/ .. _GStreamer: http://gstreamer.freedesktop.org/ Then, enable the ``replaygain`` plugin (see :ref:`using-plugins`) and specify diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index 2f691c4fe..8ccbd0091 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -36,7 +36,7 @@ For more advanced usage, you can use template syntax (see query: 'year::201(0|1)' This will query all the songs in 2010 and 2011 and generate the two playlist -files `ReleasedIn2010.m3u` and `ReleasedIn2011.m3u` using those songs. +files ``ReleasedIn2010.m3u`` and ``ReleasedIn2011.m3u`` using those songs. You can also gather the results of several queries by putting them in a list. (Items that match both queries are not included twice.) For example:: diff --git a/docs/plugins/thumbnails.rst b/docs/plugins/thumbnails.rst index 5753a9f7e..c2a28d091 100644 --- a/docs/plugins/thumbnails.rst +++ b/docs/plugins/thumbnails.rst @@ -7,7 +7,7 @@ Nautilus or Thunar, and is therefore POSIX-only. To use the ``thumbnails`` plugin, enable it (see :doc:`/plugins/index`) as well as the :doc:`/plugins/fetchart`. You'll need 2 additional python packages: -`pyxdg` and `pathlib`. +:pypi:`pyxdg` and :pypi:`pathlib`. ``thumbnails`` needs to resize the covers, and therefore requires either `ImageMagick`_ or `Pillow`_. From 9b0fcff216055cb15fc12272b7685d35e908dbe2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 27 Apr 2019 12:43:33 +1000 Subject: [PATCH 217/339] playlist: remove "smart" from title in docs --- docs/plugins/playlist.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index d9b400987..3622581db 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -1,5 +1,5 @@ -Smart Playlist Plugin -===================== +Playlist Plugin +=============== ``playlist`` is a plugin to use playlists in m3u format. From 75f8372a89f5ddb63a5aa86e1537f7c5e77565af Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 27 Apr 2019 12:43:50 +1000 Subject: [PATCH 218/339] Update optional dependencies in setup.py --- setup.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 24f9f389c..78e10a002 100755 --- a/setup.py +++ b/setup.py @@ -123,22 +123,37 @@ setup( # Plugin (optional) dependencies: extras_require={ 'absubmit': ['requests'], - 'fetchart': ['requests'], + 'fetchart': ['requests', 'Pillow'], + 'embedart': ['Pillow'], + 'embyupdate': ['requests'], 'chroma': ['pyacoustid'], + 'gmusic': ['gmusicapi'], 'discogs': ['discogs-client>=2.2.1'], 'beatport': ['requests-oauthlib>=0.6.1'], + 'kodiupdate': ['requests'], 'lastgenre': ['pylast'], + 'lastimport': ['pylast'], + 'lyrics': ['requests', 'beautifulsoup4', 'langdetect'], 'mpdstats': ['python-mpd2>=0.4.2'], + 'plexupdate': ['requests'], 'web': ['flask', 'flask-cors'], 'import': ['rarfile'], - 'thumbnails': ['pyxdg'] + + 'thumbnails': ['pyxdg', 'Pillow'] + (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], + 'sonosupdate': ['soco'], }, # Non-Python/non-PyPI plugin dependencies: - # convert: ffmpeg - # bpd: python-gi and GStreamer - # absubmit: extractor binary from http://acousticbrainz.org/download + # chroma: chromaprint or fpcalc + # convert: ffmpeg + # badfiles: mp3val and flac + # bpd: python-gi and GStreamer 1.0+ + # embedart: ImageMagick + # absubmit: extractor binary from http://acousticbrainz.org/download + # keyfinder: KeyFinder + # replaygain: python-gi and GStreamer 1.0+ or mp3gain/aacgain + # or Python Audio Tools + # ipfs: go-ipfs classifiers=[ 'Topic :: Multimedia :: Sound/Audio', From 86a0f04f26bd621ef8b8c1d4720a353fc67ad62e Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 27 Apr 2019 13:43:58 +1000 Subject: [PATCH 219/339] setup.py: add optional PyGObject dependency --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 78e10a002..1aacfc1c9 100755 --- a/setup.py +++ b/setup.py @@ -142,6 +142,8 @@ setup( (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], + 'bpd': ['PyGObject'], + 'replaygain': ['PyGObject'], }, # Non-Python/non-PyPI plugin dependencies: # chroma: chromaprint or fpcalc From df26f2e8f896fcd82d32fc4a93e398b0f6a57107 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 27 Apr 2019 14:30:04 +1000 Subject: [PATCH 220/339] Use new GitHub issue template format Inspired by examples at https://github.com/stevemao/github-issue-templates/blob/457e1d0ae0f7812c0ac0c0909840f446edc242fd/system/ISSUE_TEMPLATE/ --- .../bug-report.md} | 17 +++++++++--- .github/ISSUE_TEMPLATE/feature-request.md | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) rename .github/{ISSUE_TEMPLATE.md => ISSUE_TEMPLATE/bug-report.md} (57%) create mode 100644 .github/ISSUE_TEMPLATE/feature-request.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/bug-report.md similarity index 57% rename from .github/ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/bug-report.md index 0f5481f0c..646243812 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,6 +1,17 @@ -### Problem +--- +name: "\U0001F41B Bug report" +about: Report a problem with beets -(Describe your problem, feature request, or discussion topic here. If you're reporting a bug, please fill out this and the "Setup" section below. Otherwise, you can delete them.) +--- + + + +### Problem Running this command in verbose (`-vv`) mode: @@ -14,7 +25,7 @@ Led to this problem: (paste here) ``` -Here's a link to the music files that trigger the bug (if relevant): +Here's a link to the music files that trigger the bug (if relevant): ### Setup diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 000000000..5eb651aac --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,26 @@ +--- +name: "\U0001F680 Feature request" +about: Suggest a new idea for beets + +--- + +### Use case + +I'm trying to use beets to... + + +### Solution + + + +### Alternatives + From e10be98b66a79ac96b2089d18f0041d2df418938 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sat, 27 Apr 2019 15:44:56 +0100 Subject: [PATCH 221/339] Stop using beets.config['sort_case_insensitive'] in beets.dbcore --- beets/dbcore/queryparse.py | 20 +++++++++++--------- beets/library.py | 4 +++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index 1cb25a8c7..fee38afdd 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -20,7 +20,6 @@ from __future__ import division, absolute_import, print_function import re import itertools from . import query -import beets PARSE_QUERY_PART_REGEX = re.compile( # Non-capturing optional segment for the keyword. @@ -174,11 +173,13 @@ def query_from_strings(query_cls, model_cls, prefixes, query_parts): return query_cls(subqueries) -def construct_sort_part(model_cls, part): +def construct_sort_part(model_cls, part, case_insensitive=True): """Create a `Sort` from a single string criterion. `model_cls` is the `Model` being queried. `part` is a single string - ending in ``+`` or ``-`` indicating the sort. + ending in ``+`` or ``-`` indicating the sort. `case_insensitive` + indicates whether or not the sort should be performed in a case + sensitive manner. """ assert part, "part must be a field name and + or -" field = part[:-1] @@ -187,7 +188,6 @@ def construct_sort_part(model_cls, part): assert direction in ('+', '-'), "part must end with + or -" is_ascending = direction == '+' - case_insensitive = beets.config['sort_case_insensitive'].get(bool) if field in model_cls._sorts: sort = model_cls._sorts[field](model_cls, is_ascending, case_insensitive) @@ -199,21 +199,23 @@ def construct_sort_part(model_cls, part): return sort -def sort_from_strings(model_cls, sort_parts): +def sort_from_strings(model_cls, sort_parts, case_insensitive=True): """Create a `Sort` from a list of sort criteria (strings). """ if not sort_parts: sort = query.NullSort() elif len(sort_parts) == 1: - sort = construct_sort_part(model_cls, sort_parts[0]) + sort = construct_sort_part(model_cls, sort_parts[0], case_insensitive) else: sort = query.MultipleSort() for part in sort_parts: - sort.add_sort(construct_sort_part(model_cls, part)) + sort.add_sort(construct_sort_part(model_cls, part, + case_insensitive)) return sort -def parse_sorted_query(model_cls, parts, prefixes={}): +def parse_sorted_query(model_cls, parts, prefixes={}, + case_insensitive=True): """Given a list of strings, create the `Query` and `Sort` that they represent. """ @@ -248,5 +250,5 @@ def parse_sorted_query(model_cls, parts, prefixes={}): # Avoid needlessly wrapping single statements in an OR q = query.OrQuery(query_parts) if len(query_parts) > 1 else query_parts[0] - s = sort_from_strings(model_cls, sort_parts) + s = sort_from_strings(model_cls, sort_parts, case_insensitive) return q, s diff --git a/beets/library.py b/beets/library.py index 16db1e974..d49d67227 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1239,8 +1239,10 @@ def parse_query_parts(parts, model_cls): else: non_path_parts.append(s) + case_insensitive = beets.config['sort_case_insensitive'].get(bool) + query, sort = dbcore.parse_sorted_query( - model_cls, non_path_parts, prefixes + model_cls, non_path_parts, prefixes, case_insensitive ) # Add path queries to aggregate query. From 9467fc75572d0852b742e2dba074e9c0aa316fef Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sat, 27 Apr 2019 15:56:55 +0100 Subject: [PATCH 222/339] Revert "Travis: temporarily pin tox to <=3.8.1" This reverts commit 777cfbbf610522c0b655659f8f11d519714606b5. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6e4a59ac2..017ebcf57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ addons: # To install dependencies, tell tox to do everything but actually running the # test. install: - - travis_retry pip install 'tox<=3.8.1' sphinx + - travis_retry pip install tox sphinx # upgrade requests to satisfy sphinx linkcheck (for building man pages) - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then pip install -U requests; fi - travis_retry tox -e $TOX_ENV --notest From 36dc105dc9e9ac859b64fd4fabc6224462e4e539 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Sat, 27 Apr 2019 17:57:48 +0200 Subject: [PATCH 223/339] undid quotes in log messages --- beetsplug/acousticbrainz.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 4c4315b80..8075788aa 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -176,7 +176,7 @@ class AcousticPlugin(plugins.BeetsPlugin): return {} if res.status_code == 404: - self._log.info(u'recording ID \'{}\' not found', mbid) + self._log.info(u'recording ID {} not found', mbid) return {} try: @@ -199,27 +199,27 @@ class AcousticPlugin(plugins.BeetsPlugin): if not force: mood_str = item.get('mood_acoustic', u'') if mood_str: - self._log.info(u'data already present for: \'{}\'', item) + self._log.info(u'data already present for: {}', item) continue # We can only fetch data for tracks with MBIDs. if not item.mb_trackid: continue - self._log.info(u'getting data for: \'{}\'', item) + self._log.info(u'getting data for: {}', item) data = self._get_data(item.mb_trackid) if data: for attr, val in self._map_data_to_scheme(data, ABSCHEME): if not tags or attr in tags: self._log.debug( - u'attribute \'{}\' of \'{}\' set to \'{}\'', + u'attribute {} of {} set to {}', attr, item, val) setattr(item, attr, val) else: - self._log.debug(u'skipping attribute \'{}\' of \'{}\'' - u' (value \'{}\') due to config', + self._log.debug(u'skipping attribute {} of {}' + u' (value {}) due to config', attr, item, val) From 941dd6e48f989ab1da2773f1a71e7bab5b34a70e Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Sat, 27 Apr 2019 17:58:26 +0200 Subject: [PATCH 224/339] Formatting --- beetsplug/acousticbrainz.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 8075788aa..02be2d7e8 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -211,8 +211,7 @@ class AcousticPlugin(plugins.BeetsPlugin): if data: for attr, val in self._map_data_to_scheme(data, ABSCHEME): if not tags or attr in tags: - self._log.debug( - u'attribute {} of {} set to {}', + self._log.debug(u'attribute {} of {} set to {}', attr, item, val) From 62c1d37bcc44e86341388e595bdc5c2ed81a8d57 Mon Sep 17 00:00:00 2001 From: Rainer Hihn Date: Sat, 27 Apr 2019 17:58:48 +0200 Subject: [PATCH 225/339] Formatting --- beetsplug/acousticbrainz.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 02be2d7e8..01f3ac6ac 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -212,9 +212,9 @@ class AcousticPlugin(plugins.BeetsPlugin): for attr, val in self._map_data_to_scheme(data, ABSCHEME): if not tags or attr in tags: self._log.debug(u'attribute {} of {} set to {}', - attr, - item, - val) + attr, + item, + val) setattr(item, attr, val) else: self._log.debug(u'skipping attribute {} of {}' From 0bfe0e9a67cd8b3204d13c9f461955d0db63baee Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 30 Apr 2019 11:42:44 -0400 Subject: [PATCH 226/339] Link to the aunique config option --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index babcf7ac3..112afdea3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,8 +6,8 @@ Changelog There are some new core features: -* A new ``aunique`` configuration option allows setting default options - for the :ref:`aunique` template function. +* A new :ref:`config-aunique` configuration option allows setting default + options for the :ref:`aunique` template function. * The ``albumdisambig`` field no longer includes the MusicBrainz release group disambiguation comment. A new ``releasegroupdisambig`` field has been added. :bug:`3024` From fc084ae975a1ac06d23d5951309f2f8ec7af075c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 30 Apr 2019 11:47:26 -0400 Subject: [PATCH 227/339] More changelog proofreading --- docs/changelog.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 112afdea3..f3a6fe13c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -59,22 +59,22 @@ And many improvements to existing plugins: strange repeated output when running ``beet write``. Thanks to :user:`Holzhaus`. :bug:`3097` :bug:`2942` -* :doc:`/plugins/convert`: The plugin now has a ``id3v23`` option that allows +* :doc:`/plugins/convert`: The plugin now has an ``id3v23`` option that allows you to override the global ``id3v23`` option. Thanks to :user:`Holzhaus`. :bug:`3104` -* :doc:`/plugins/spotify`: The plugin now uses OAuth for authentication to the - Spotify API. - Thanks to :user:`rhlahuja`. - :bug:`2694` :bug:`3123` -* :doc:`/plugins/spotify`: The plugin now works as an import metadata - provider: you can match tracks and albums using the Spotify database. - Thanks to :user:`rhlahuja`. - :bug:`3123` +* :doc:`/plugins/spotify`: + * The plugin now uses OAuth for authentication to the Spotify API. + Thanks to :user:`rhlahuja`. + :bug:`2694` :bug:`3123` + * The plugin now works as an import metadata + provider: you can match tracks and albums using the Spotify database. + Thanks to :user:`rhlahuja`. + :bug:`3123` * :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which passes that flag to ipfs. Thanks to :user:`wildthyme`. -* :doc:`/plugins/discogs`: The plugin has rate limiting for the discogs API now. +* :doc:`/plugins/discogs`: The plugin now has rate limiting for the Discogs API. :bug:`3081` * :doc:`/plugins/mpdstats`, :doc:`/plugins/mpdupdate`: These plugins now use the ``MPD_PORT`` environment variable if no port is specified in the From 644c814695df6d7db26cfced5217f53d65b54758 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 30 Apr 2019 11:49:39 -0400 Subject: [PATCH 228/339] Clarify docstring for Float's digits parameters Introduced in #3238. --- beets/dbcore/types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index e08b417a7..c37def875 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -173,7 +173,8 @@ class Id(Integer): class Float(Type): - """A basic floating-point type. Supports padding. + """A basic floating-point type. The `digits` parameter specifies how + many decimal places to use in the human-readable representation. """ sql = u'REAL' query = query.NumericQuery From ff73a056f510911ac6c2954d25bb846298f4ba2d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 30 Apr 2019 11:52:29 -0400 Subject: [PATCH 229/339] Changelog for #3238 (fixes #2790) --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f3a6fe13c..54f39718c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -103,6 +103,11 @@ And many improvements to existing plugins: explicitly specifies it with ``-i`` parameter. This only works when exporting library fields. :bug:`3084` +* :doc:`/plugins/acousticbrainz`: The plugin now declares types for all its + fields, which enables easier querying and avoids a problem where very small + numbers would be stored as strings. + Thanks to :user:`rain0r`. + :bug:`2790` :bug:`3238` .. _Google Groups: https://groups.google.com/forum/#!searchin/beets-users/mbsync|sort:date/beets-users/iwCF6bNdh9A/i1xl4Gx8BQAJ From 0ef50ed2467430035dcf8c8e33f63d394bfe0c94 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 30 Apr 2019 16:53:54 -0400 Subject: [PATCH 230/339] Minor cleanup for #3236 --- beets/util/artresizer.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 5a0036376..3590dcf22 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -81,9 +81,10 @@ def pil_resize(maxwidth, path_in, path_out=None): def im_resize(maxwidth, path_in, path_out=None): - """Resize using ImageMagick's ``magick`` tool - (or fall back to ``convert`` for older versions). - Return the output path of resized image. + """Resize using ImageMagick. + + Use the ``magick`` program or ``convert`` on older versions. Return + the output path of resized image. """ path_out = path_out or temp_file_for(path_in) log.debug(u'artresizer: ImageMagick resizing {0} to {1}', @@ -235,8 +236,14 @@ class ArtResizer(six.with_metaclass(Shareable, object)): @staticmethod def _check_method(): """Return a tuple indicating an available method and its version. - If the method is ImageMagick, also return a bool indicating whether to - use the `magick` binary or legacy utils (`convert`, `identify`, etc.) + + The result has at least two elements: + - The method, eitehr WEBPROXY, PIL, or IMAGEMAGICK. + - The version. + + If the method is IMAGEMAGICK, there is also a third element: a + bool flag indicating whether to use the `magick` binary or + legacy single-purpose executables (`convert`, `identify`, etc.) """ version = get_im_version() if version: @@ -251,17 +258,17 @@ class ArtResizer(six.with_metaclass(Shareable, object)): def get_im_version(): - """Return ImageMagick version/legacy-flag pair or None if the check fails. - - Try invoking ImageMagick's `magick` binary first, then `convert` if - `magick` is unavailable. + """Get the ImageMagick version and legacy flag as a pair. Or return + None if ImageMagick is not available. """ for cmd_name, legacy in ((['magick'], False), (['convert'], True)): cmd = cmd_name + ['--version'] try: out = util.command_output(cmd) - + except (subprocess.CalledProcessError, OSError) as exc: + log.debug(u'ImageMagick version check failed: {}', exc) + else: if b'imagemagick' in out.lower(): pattern = br".+ (\d+)\.(\d+)\.(\d+).*" match = re.search(pattern, out) @@ -271,15 +278,11 @@ def get_im_version(): int(match.group(3))) return version, legacy - except (subprocess.CalledProcessError, OSError) as exc: - log.debug(u'ImageMagick version check failed: {}', exc) - return None def get_pil_version(): - """Return Pillow version or None if it is unavailable - Try importing PIL. + """Get the PIL/Pillow version, or None if it is unavailable. """ try: __import__('PIL', fromlist=[str('Image')]) From 6ec061b8a50d95591528b1a08e5ce7151075fdab Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 30 Apr 2019 16:57:27 -0400 Subject: [PATCH 231/339] Changelog for #3236 --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 54f39718c..035f8015a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -246,6 +246,11 @@ And there are many other fixes: :bug:`3195` * :doc:`/plugins/lastgenre`: The ``force`` config option now actually works. :bug:`2704` :bug:`3054` +* Resizing image files with ImageMagick now avoids problems on systems where + there is a ``convert`` command that is *not* ImageMagick's by using the + ``magick`` executable when it is available. + Thanks to :user:`ababyduck`. + :bug:`2093` :bug:`3236` For developers: From f09088df4355888641189f7019091a4d74b17bf7 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Wed, 1 May 2019 17:53:03 +0100 Subject: [PATCH 232/339] replaygain: Handle invalid XML output from bs1770gain --- beetsplug/replaygain.py | 9 +++++- test/test_replaygain.py | 62 ++++++++++++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 4168c61b9..4a0ea064f 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -251,7 +251,14 @@ class Bs1770gainBackend(Backend): state['gain'] = state['peak'] = None parser.StartElementHandler = start_element_handler parser.EndElementHandler = end_element_handler - parser.Parse(text, True) + + try: + parser.Parse(text, True) + except xml.parsers.expat.ExpatError: + raise ReplayGainError( + u'The bs1770gain tool produced malformed XML. ' + 'Using version >=0.4.10 may solve this problem.' + ) if len(per_file_gain) != len(path_list): raise ReplayGainError( diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 6ddee54da..f750f3012 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -19,7 +19,8 @@ from __future__ import division, absolute_import, print_function import unittest import six -from test.helper import TestHelper, has_program +from mock import patch +from test.helper import TestHelper, capture_log, has_program from beets import config from beets.mediafile import MediaFile @@ -44,6 +45,15 @@ else: LOUDNESS_PROG_AVAILABLE = False +def reset_replaygain(item): + item['rg_track_peak'] = None + item['rg_track_gain'] = None + item['rg_album_gain'] = None + item['rg_album_gain'] = None + item.write() + item.store() + + class ReplayGainCliTestBase(TestHelper): def setUp(self): @@ -68,20 +78,12 @@ class ReplayGainCliTestBase(TestHelper): album = self.add_album_fixture(2) for item in album.items(): - self._reset_replaygain(item) + reset_replaygain(item) def tearDown(self): self.teardown_beets() self.unload_plugins() - def _reset_replaygain(self, item): - item['rg_track_peak'] = None - item['rg_track_gain'] = None - item['rg_album_gain'] = None - item['rg_album_gain'] = None - item.write() - item.store() - def test_cli_saves_track_gain(self): for item in self.lib.items(): self.assertIsNone(item.rg_track_peak) @@ -166,6 +168,46 @@ class ReplayGainLdnsCliTest(ReplayGainCliTestBase, unittest.TestCase): backend = u'bs1770gain' +class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): + + @patch('beetsplug.replaygain.call') + def setUp(self, call_patch): + self.setup_beets() + self.config['replaygain']['backend'] = 'bs1770gain' + + # Patch call to return nothing, bypassing the bs1770gain installation + # check. + call_patch.return_value = None + self.load_plugins('replaygain') + + for item in self.add_album_fixture(2).items(): + reset_replaygain(item) + + @patch('beetsplug.replaygain.call') + def test_malformed_output(self, call_patch): + # Return malformed XML (the ampersand should be &) + call_patch.return_value = """ + + + + + + + """ + + with capture_log('beets.replaygain') as logs: + self.run_command('replaygain') + + # Count how many lines match the expected error. + matching = [line for line in logs if + line == 'replaygain: ReplayGain error: bs1770gain ' + 'returned malformed XML - this is a bug in ' + 'versions prior to v0.4.10, please ensure that ' + 'your version is up to date'] + + self.assertEqual(len(matching), 2) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 83ac5ed022f3bc4b5fdbf4e411c308bfb3c1d1f6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 2 May 2019 09:35:14 -0400 Subject: [PATCH 233/339] Changelog for #3247 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 035f8015a..d88bf0b12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -174,6 +174,9 @@ Many fixes have been focused on issues where beets would previously crash: * :doc:`/plugins/bpd`: Fix a crash triggered when certain clients tried to list the albums belonging to a particular artist. :bug:`3007` :bug:`3215` +* :doc:`/plugins/replaygain`: Avoid a crash when the ``bs1770gain`` tool emits + malformed XML. + :bug:`2983` :bug:`3247` There are many fixes related to compatibility with our dependencies including addressing changes interfaces: From 8bef21a4d5f80a2bd1468b543fcf596f89140630 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 2 May 2019 09:38:11 -0400 Subject: [PATCH 234/339] More permissive log check for #3247 --- test/test_replaygain.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index f750f3012..81da7139e 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -200,10 +200,7 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): # Count how many lines match the expected error. matching = [line for line in logs if - line == 'replaygain: ReplayGain error: bs1770gain ' - 'returned malformed XML - this is a bug in ' - 'versions prior to v0.4.10, please ensure that ' - 'your version is up to date'] + 'malformed XML' in line] self.assertEqual(len(matching), 2) From c5075b28551d8552d2dd35fd8f3d08969f24906c Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 18:24:59 +0200 Subject: [PATCH 235/339] Create a cached template() function We were previously doing calls to Template() directly, sometimes in a loop. This caused the same template to be recompiled over and over. This commit introduces a function template() which caches the results, so that multiple calls with the same template string does not require recompilation. --- beets/dbcore/db.py | 4 ++-- beets/library.py | 6 +++--- beets/ui/__init__.py | 4 ++-- beets/util/functemplate.py | 4 ++++ docs/changelog.rst | 3 +++ 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 97a4a7ce3..3195b52c9 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -25,7 +25,7 @@ import sqlite3 import contextlib import beets -from beets.util.functemplate import Template +from beets.util import functemplate from beets.util import py3_path from beets.dbcore import types from .query import MatchQuery, NullSort, TrueQuery @@ -597,7 +597,7 @@ class Model(object): """ # Perform substitution. if isinstance(template, six.string_types): - template = Template(template) + template = functemplate.template(template) return template.substitute(self.formatted(for_path), self._template_funcs()) diff --git a/beets/library.py b/beets/library.py index d49d67227..5786ce9d9 100644 --- a/beets/library.py +++ b/beets/library.py @@ -31,7 +31,7 @@ from beets import plugins from beets import util from beets.util import bytestring_path, syspath, normpath, samefile, \ MoveOperation -from beets.util.functemplate import Template +from beets.util.functemplate import template, Template from beets import dbcore from beets.dbcore import types import beets @@ -855,7 +855,7 @@ class Item(LibModel): if isinstance(path_format, Template): subpath_tmpl = path_format else: - subpath_tmpl = Template(path_format) + subpath_tmpl = template(path_format) # Evaluate the selected template. subpath = self.evaluate_template(subpath_tmpl, True) @@ -1134,7 +1134,7 @@ class Album(LibModel): image = bytestring_path(image) item_dir = item_dir or self.item_dir() - filename_tmpl = Template( + filename_tmpl = template( beets.config['art_filename'].as_str()) subpath = self.evaluate_template(filename_tmpl, True) if beets.config['asciify_paths']: diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 327db6b04..622a1e7f0 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -36,7 +36,7 @@ from beets import logging from beets import library from beets import plugins from beets import util -from beets.util.functemplate import Template +from beets.util.functemplate import template from beets import config from beets.util import confit, as_string from beets.autotag import mb @@ -616,7 +616,7 @@ def get_path_formats(subview=None): subview = subview or config['paths'] for query, view in subview.items(): query = PF_KEY_QUERIES.get(query, query) # Expand common queries. - path_formats.append((query, Template(view.as_str()))) + path_formats.append((query, template(view.as_str()))) return path_formats diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 6a34a3bb3..57ea1a394 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -35,6 +35,7 @@ import dis import types import sys import six +import functools SYMBOL_DELIM = u'$' FUNC_DELIM = u'%' @@ -552,6 +553,9 @@ def _parse(template): parts.append(remainder) return Expression(parts) +@functools.lru_cache(maxsize=128) +def template(fmt): + return Template(fmt) # External interface. diff --git a/docs/changelog.rst b/docs/changelog.rst index d88bf0b12..085345353 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -117,6 +117,9 @@ Some improvements have been focused on improving beets' performance: to be displayed. Thanks to :user:`pprkut`. :bug:`3089` +* Querying the library was further improved by reusing compiled teamplates + instead of compiling them over and over again. + Thanks to :user:`SimonPersson`. * :doc:`/plugins/absubmit`, :doc:`/plugins/badfiles`: Analysis now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. From b9768e6c9b3591c5d512d32f8fc56e2cdd148cdd Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 18:31:51 +0200 Subject: [PATCH 236/339] Lazily fetch album in FormattedItemMapping, as it's not needed in most cases. --- beets/library.py | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/beets/library.py b/beets/library.py index d49d67227..27c657347 100644 --- a/beets/library.py +++ b/beets/library.py @@ -376,24 +376,41 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def __init__(self, item, for_path=False): super(FormattedItemMapping, self).__init__(item, for_path) - self.album = item.get_album() - self.album_keys = [] - if self.album: - for key in self.album.keys(True): - if key in Album.item_keys or key not in item._fields.keys(): - self.album_keys.append(key) - self.all_keys = set(self.model_keys).union(self.album_keys) + self.album = None + self.album_keys = None + self.all_keys = None + self.item = item + + def _all_keys(self): + if not self.all_keys: + self.all_keys = set(self.model_keys).union(self._album_keys()) + return self.all_keys + + def _album_keys(self): + if not self.album_keys: + album = self._album() + self.album_keys = [] + if album: + for key in album.keys(True): + if key in Album.item_keys or key not in self.item._fields.keys(): + self.album_keys.append(key) + return self.album_keys + + def _album(self): + if not self.album: + self.album = self.item.get_album() + return self.album def _get(self, key): """Get the value for a key, either from the album or the item. Raise a KeyError for invalid keys. """ - if self.for_path and key in self.album_keys: - return self._get_formatted(self.album, key) + if self.for_path and key in self._album_keys(): + return self._get_formatted(self._album(), key) elif key in self.model_keys: return self._get_formatted(self.model, key) - elif key in self.album_keys: - return self._get_formatted(self.album, key) + elif key in self._album_keys(): + return self._get_formatted(self._album(), key) else: raise KeyError(key) @@ -413,10 +430,10 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): return value def __iter__(self): - return iter(self.all_keys) + return iter(self._all_keys()) def __len__(self): - return len(self.all_keys) + return len(self._all_keys()) class Item(LibModel): From b1f8fe963f0210490937b87588d2644e5b59ccc9 Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 19:02:39 +0200 Subject: [PATCH 237/339] Fix flake8 errors. --- beets/library.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 27c657347..6493c2fb6 100644 --- a/beets/library.py +++ b/beets/library.py @@ -392,7 +392,8 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): self.album_keys = [] if album: for key in album.keys(True): - if key in Album.item_keys or key not in self.item._fields.keys(): + if key in Album.item_keys \ + or key not in self.item._fields.keys(): self.album_keys.append(key) return self.album_keys From 7df4e23b134c20fa292dbe4c35b333002e5f33f0 Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 19:27:31 +0200 Subject: [PATCH 238/339] Fix formatting, and add python2 support. --- beets/util/functemplate.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 57ea1a394..9c625b7f9 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -553,12 +553,21 @@ def _parse(template): parts.append(remainder) return Expression(parts) -@functools.lru_cache(maxsize=128) + +# Decorator that enables lru_cache on py3, and no caching on py2. +def cached(func): + if six.PY2: + # Sorry python2 users, no caching for you :( + return func + return functools.lru_cache(maxsize=128)(func) + + +@cached def template(fmt): return Template(fmt) -# External interface. +# External interface. class Template(object): """A string template, including text, Symbols, and Calls. """ From 0d190e7fad17d335b9bba17741167b2b3a9fc77f Mon Sep 17 00:00:00 2001 From: David Logie Date: Thu, 9 May 2019 19:08:43 +0100 Subject: [PATCH 239/339] Use NullPaddedInt for all r128_album_gain fields. --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index d49d67227..e3e2a6876 100644 --- a/beets/library.py +++ b/beets/library.py @@ -935,7 +935,7 @@ class Album(LibModel): 'releasegroupdisambig': types.STRING, 'rg_album_gain': types.NULL_FLOAT, 'rg_album_peak': types.NULL_FLOAT, - 'r128_album_gain': types.PaddedInt(6), + 'r128_album_gain': types.NullPaddedInt(6), 'original_year': types.PaddedInt(4), 'original_month': types.PaddedInt(2), 'original_day': types.PaddedInt(2), From d236e1edff56255e5632c3dbbf4c9e4b2cb3a16c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 9 May 2019 14:17:11 -0400 Subject: [PATCH 240/339] Changelog refinement for #3258 --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 085345353..8608f1b6f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -117,9 +117,10 @@ Some improvements have been focused on improving beets' performance: to be displayed. Thanks to :user:`pprkut`. :bug:`3089` -* Querying the library was further improved by reusing compiled teamplates - instead of compiling them over and over again. +* Another query optimization works by compiling templates once and reusing + them instead of recompiling them to print out each matching object. Thanks to :user:`SimonPersson`. + :bug:`3258` * :doc:`/plugins/absubmit`, :doc:`/plugins/badfiles`: Analysis now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. From ff1d43ddf92f5c5a85d72766e01208566bfc1d1c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 9 May 2019 14:21:38 -0400 Subject: [PATCH 241/339] Refine @cached decorator from #3258 Don't restrict to Python 2 precisely. --- beets/util/functemplate.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 9c625b7f9..5d9900f0b 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -554,12 +554,15 @@ def _parse(template): return Expression(parts) -# Decorator that enables lru_cache on py3, and no caching on py2. def cached(func): - if six.PY2: - # Sorry python2 users, no caching for you :( + """Like the `functools.lru_cache` decorator, but works (as a no-op) + on Python < 3.2. + """ + if hasattr(functools, 'lru_cache'): + return functools.lru_cache(maxsize=128)(func) + else: + # Do nothing when lru_cache is not available. return func - return functools.lru_cache(maxsize=128)(func) @cached From 23da057cebf35e82193cdf4b9e56064ae49b6f41 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 9 May 2019 14:22:48 -0400 Subject: [PATCH 242/339] tox: Don't use Python 3.8 by default 3.8.0 final is not released yet. This default set is meant to be a reasonable list for quick iteration during development. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e3250bd6b..8736f0f3c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py38-test, py27-flake8, docs +envlist = py27-test, py37-test, py27-flake8, docs # The exhaustive list of environments is: # envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs From 7043ed59474de417c240fa01664e460104b96c97 Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 20:48:20 +0200 Subject: [PATCH 243/339] Introduce @lazy_property for better readability. --- beets/library.py | 65 ++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/beets/library.py b/beets/library.py index 6493c2fb6..839f0096f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -24,6 +24,7 @@ import time import re import six import string +import functools from beets import logging from beets.mediafile import MediaFile, UnreadableFileError @@ -368,6 +369,22 @@ class LibModel(dbcore.Model): return self.__str__().encode('utf-8') +def lazy_property(func): + field_name = '_' + func.__name__ + + @property + @functools.wraps(func) + def wrapper(self): + if hasattr(self, field_name): + return getattr(self, field_name) + + value = func(self) + setattr(self, field_name, value) + return value + + return wrapper + + class FormattedItemMapping(dbcore.db.FormattedMapping): """Add lookup for album-level fields. @@ -376,42 +393,36 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def __init__(self, item, for_path=False): super(FormattedItemMapping, self).__init__(item, for_path) - self.album = None - self.album_keys = None - self.all_keys = None self.item = item - def _all_keys(self): - if not self.all_keys: - self.all_keys = set(self.model_keys).union(self._album_keys()) - return self.all_keys + @lazy_property + def all_keys(self): + return set(self.model_keys).union(self.album_keys) - def _album_keys(self): - if not self.album_keys: - album = self._album() - self.album_keys = [] - if album: - for key in album.keys(True): - if key in Album.item_keys \ - or key not in self.item._fields.keys(): - self.album_keys.append(key) - return self.album_keys + @lazy_property + def album_keys(self): + album_keys = [] + if self.album: + for key in self.album.keys(True): + if key in Album.item_keys \ + or key not in self.item._fields.keys(): + album_keys.append(key) + return album_keys - def _album(self): - if not self.album: - self.album = self.item.get_album() - return self.album + @lazy_property + def album(self): + return self.item.get_album() def _get(self, key): """Get the value for a key, either from the album or the item. Raise a KeyError for invalid keys. """ - if self.for_path and key in self._album_keys(): - return self._get_formatted(self._album(), key) + if self.for_path and key in self.album_keys: + return self._get_formatted(self.album, key) elif key in self.model_keys: return self._get_formatted(self.model, key) - elif key in self._album_keys(): - return self._get_formatted(self._album(), key) + elif key in self.album_keys: + return self._get_formatted(self.album, key) else: raise KeyError(key) @@ -431,10 +442,10 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): return value def __iter__(self): - return iter(self._all_keys()) + return iter(self.all_keys) def __len__(self): - return len(self._all_keys()) + return len(self.all_keys) class Item(LibModel): From 337b6bc4c90c1692638df7de62df05867b968011 Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 21:00:59 +0200 Subject: [PATCH 244/339] Formatting fix. --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 839f0096f..0ca718ee0 100644 --- a/beets/library.py +++ b/beets/library.py @@ -405,7 +405,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): if self.album: for key in self.album.keys(True): if key in Album.item_keys \ - or key not in self.item._fields.keys(): + or key not in self.item._fields.keys(): album_keys.append(key) return album_keys From 96d83ad1db48a6ef0584691e75963cb3e326fdc3 Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Fri, 10 May 2019 19:29:51 +0200 Subject: [PATCH 245/339] Move lazy_property to util package. --- beets/library.py | 19 +------------------ beets/util/__init__.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/beets/library.py b/beets/library.py index 0ca718ee0..103054fd3 100644 --- a/beets/library.py +++ b/beets/library.py @@ -24,14 +24,13 @@ import time import re import six import string -import functools from beets import logging from beets.mediafile import MediaFile, UnreadableFileError from beets import plugins from beets import util from beets.util import bytestring_path, syspath, normpath, samefile, \ - MoveOperation + MoveOperation, lazy_property from beets.util.functemplate import Template from beets import dbcore from beets.dbcore import types @@ -369,22 +368,6 @@ class LibModel(dbcore.Model): return self.__str__().encode('utf-8') -def lazy_property(func): - field_name = '_' + func.__name__ - - @property - @functools.wraps(func) - def wrapper(self): - if hasattr(self, field_name): - return getattr(self, field_name) - - value = func(self) - setattr(self, field_name, value) - return value - - return wrapper - - class FormattedItemMapping(dbcore.db.FormattedMapping): """Add lookup for album-level fields. diff --git a/beets/util/__init__.py b/beets/util/__init__.py index f5ad2da22..e2348cf6e 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -23,6 +23,7 @@ import locale import re import shutil import fnmatch +import functools from collections import Counter from multiprocessing.pool import ThreadPool import traceback @@ -1031,3 +1032,26 @@ def par_map(transform, items): pool.map(transform, items) pool.close() pool.join() + + +def lazy_property(func): + """A decorator that creates a lazily evaluated property. On first access, + the property is assigned the return value of `func`. This first value is + stored, so that future accesses do not have to evaluate `func` again. + + This behaviour is useful when `func` is expensive to evaluate, and it is + not certain that the result will be needed. + """ + field_name = '_' + func.__name__ + + @property + @functools.wraps(func) + def wrapper(self): + if hasattr(self, field_name): + return getattr(self, field_name) + + value = func(self) + setattr(self, field_name, value) + return value + + return wrapper From 0754940465fad936feea596df00ca289e5dd7aed Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Fri, 10 May 2019 19:41:11 +0200 Subject: [PATCH 246/339] Add changelog entry. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d88bf0b12..145eed330 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -126,6 +126,10 @@ Some improvements have been focused on improving beets' performance: is long. Thanks to :user:`ray66`. :bug:`3207` :bug:`2752` +* Querying the library for items is now faster, for all queries that do not need + to access album level properties. This was implemented by lazily fetching the + album only when needed. + Thanks to :user:`SimonPersson`. Several improvements are related to usability: From 909fd1eb272691d5cccaa0426db74871c770a8bb Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 12 May 2019 11:21:19 -0400 Subject: [PATCH 247/339] Reorganize changelog for performance improvements --- docs/changelog.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 53be0befd..a0f77b065 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -113,14 +113,19 @@ And many improvements to existing plugins: Some improvements have been focused on improving beets' performance: -* Querying the library is now faster because we only convert fields that need - to be displayed. - Thanks to :user:`pprkut`. - :bug:`3089` -* Another query optimization works by compiling templates once and reusing - them instead of recompiling them to print out each matching object. - Thanks to :user:`SimonPersson`. - :bug:`3258` +* Querying the library is now faster: + * We only convert fields that need to be displayed. + Thanks to :user:`pprkut`. + :bug:`3089` + * We now compile templates once and reuse them instead of recompiling them + to print out each matching object. + Thanks to :user:`SimonPersson`. + :bug:`3258` + * Querying the library for items is now faster, for all queries that do not + need to access album level properties. This was implemented by lazily + fetching the album only when needed. + Thanks to :user:`SimonPersson`. + :bug:`3260` * :doc:`/plugins/absubmit`, :doc:`/plugins/badfiles`: Analysis now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. @@ -130,10 +135,6 @@ Some improvements have been focused on improving beets' performance: is long. Thanks to :user:`ray66`. :bug:`3207` :bug:`2752` -* Querying the library for items is now faster, for all queries that do not need - to access album level properties. This was implemented by lazily fetching the - album only when needed. - Thanks to :user:`SimonPersson`. Several improvements are related to usability: From c3794189671d37025560f0e7f07d5683477cf36c Mon Sep 17 00:00:00 2001 From: arogl Date: Mon, 13 May 2019 18:08:29 +1000 Subject: [PATCH 248/339] Update setup.py to install to a directory rather than egg file fixes #3264 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1aacfc1c9..8e9828223 100755 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ setup( platforms='ALL', long_description=_read('README.rst'), test_suite='test.testall.suite', + zip_safe=False, include_package_data=True, # Install plugin resources. packages=[ From 967c08cf47e55818947173d453f68b29b40b8005 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 16 May 2019 16:39:51 -0400 Subject: [PATCH 249/339] Changelog summary --- docs/changelog.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a0f77b065..8baa96c98 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,14 @@ Changelog 1.4.8 (in development) ---------------------- -There are some new core features: +This release is far too long in coming, but it's a good one. There is the +usual torrent of new features and a ridiculously long line of fixes, but there +are also some crucial maintenance changes. +We officially support Python 3.7 and 3.8, and some performance optimizations +can (anecdotally) make listing your library more than three times faster than +in the previous version. + +The new core features are: * A new :ref:`config-aunique` configuration option allows setting default options for the :ref:`aunique` template function. From 217915bd69a9e5bab97185bb9ba79dfd648bf506 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 16 May 2019 16:42:15 -0400 Subject: [PATCH 250/339] Slightly longer changelog section headings --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8baa96c98..a63ade9e9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -271,7 +271,7 @@ And there are many other fixes: Thanks to :user:`ababyduck`. :bug:`2093` :bug:`3236` -For developers: +There is one new thing for plugin developers to know about: * In addition to prefix-based field queries, plugins can now define *named queries* that are not associated with any specific field. @@ -279,7 +279,7 @@ For developers: ``playlist:name`` although there is no field named ``playlist``. See :ref:`extend-query` for details. -For packagers: +And some messages for packagers: * Note the changes to the dependencies on :pypi:`jellyfish` and :pypi:`munkres`. * The optional :pypi:`python-itunes` dependency has been removed. From 2f9ce7e43c82d35fd6f867e45e67770dcb864573 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 16 May 2019 16:42:42 -0400 Subject: [PATCH 251/339] Date for 1.4.8 release :scream: --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a63ade9e9..720925c8e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,8 +1,8 @@ Changelog ========= -1.4.8 (in development) ----------------------- +1.4.8 (May 16, 2019) +-------------------- This release is far too long in coming, but it's a good one. There is the usual torrent of new features and a ridiculously long line of fixes, but there From c566a74bca206730d54e80b829945678be2e62c1 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 16 May 2019 17:39:18 -0400 Subject: [PATCH 252/339] Travis: temporarily disable Python 3.8 There seems to be a compatibility error in Werkzeug on the prerelease of 3.8. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 017ebcf57..46018cd4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ matrix: - python: 3.7 env: {TOX_ENV: py37-test} dist: xenial - - python: 3.8-dev - env: {TOX_ENV: py38-test} - dist: xenial + # - python: 3.8-dev + # env: {TOX_ENV: py38-test} + # dist: xenial # - python: pypy # - env: {TOX_ENV: pypy-test} - python: 3.4 From a34f19e01c849d0779b8ae6b9899b83bb18c4c0c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 16 May 2019 20:44:38 -0400 Subject: [PATCH 253/339] Version bump: 1.4.9 Given what we have in the pipeline, we may need to change this to 1.5.0 instead... --- beets/__init__.py | 2 +- docs/changelog.rst | 6 ++++++ docs/conf.py | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index c76f633d2..92afa2c87 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -19,7 +19,7 @@ import os from beets.util import confit -__version__ = u'1.4.8' +__version__ = u'1.4.9' __author__ = u'Adrian Sampson ' diff --git a/docs/changelog.rst b/docs/changelog.rst index 720925c8e..b073249e4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +1.4.9 (in development) +---------------------- + +Changelog goes here! + + 1.4.8 (May 16, 2019) -------------------- diff --git a/docs/conf.py b/docs/conf.py index 7e52d283e..0459c85a0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ project = u'beets' copyright = u'2016, Adrian Sampson' version = '1.4' -release = '1.4.8' +release = '1.4.9' pygments_style = 'sphinx' diff --git a/setup.py b/setup.py index 1aacfc1c9..5a705b982 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if 'sdist' in sys.argv: setup( name='beets', - version='1.4.8', + version='1.4.9', description='music tagger and library organizer', author='Adrian Sampson', author_email='adrian@radbox.org', From 299cb53e44efdbccd2ad623434edcdbd4073547a Mon Sep 17 00:00:00 2001 From: Filipe Fortes Date: Fri, 17 May 2019 15:02:18 -0400 Subject: [PATCH 254/339] Fix ImageMagick Detection The `return` statement was at the wrong indent level, so we never checked for the legacy `convert` executable --- beets/util/artresizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 3590dcf22..1ee3e560d 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -278,7 +278,7 @@ def get_im_version(): int(match.group(3))) return version, legacy - return None + return None def get_pil_version(): From 3399154adf30c72ab3d1c22570764c72c120cc89 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 17 May 2019 19:37:03 -0400 Subject: [PATCH 255/339] Changelog for #3269 --- docs/changelog.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b073249e4..029942716 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,11 @@ Changelog 1.4.9 (in development) ---------------------- -Changelog goes here! +Fixes: + +* Fix a regression in the last release that made the image resizer fail to + detect older versions of ImageMagick. + :bug:`3269` 1.4.8 (May 16, 2019) From d6dc1b7c4e9690a27727a8cfc03fcad9a7ebc383 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 18 May 2019 16:27:13 -0400 Subject: [PATCH 256/339] gmusic: Use as_filename for oauth_file (#3270) --- beetsplug/gmusic.py | 2 +- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index c2fda19d4..a10577520 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -62,7 +62,7 @@ class Gmusic(BeetsPlugin): return # Checks for OAuth2 credentials, # if they don't exist - performs authorization - oauth_file = self.config['oauth_file'].as_str() + oauth_file = self.config['oauth_file'].as_filename() if os.path.isfile(oauth_file): uploader_id = self.config['uploader_id'] uploader_name = self.config['uploader_name'] diff --git a/docs/changelog.rst b/docs/changelog.rst index 029942716..79aa53d3c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,9 @@ Fixes: * Fix a regression in the last release that made the image resizer fail to detect older versions of ImageMagick. :bug:`3269` +* :doc:`/plugins/gmusic`: The ``oauth_file`` config option now supports more + flexible path values, including ``~`` for the home directory. + :bug:`3270` 1.4.8 (May 16, 2019) From d77a13eb343a2a41e60c50e754694896d354e633 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 18 May 2019 16:32:56 -0400 Subject: [PATCH 257/339] gmusic: Fix compatibility with gmusicapi>=12.0.0 Fixes #3270. --- beetsplug/gmusic.py | 9 ++++++++- docs/changelog.rst | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index a10577520..f548d1944 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -31,12 +31,19 @@ class Gmusic(BeetsPlugin): def __init__(self): super(Gmusic, self).__init__() self.m = Musicmanager() + + # OAUTH_FILEPATH was moved in gmusicapi 12.0.0. + if hasattr(Musicmanager, 'OAUTH_FILEPATH'): + oauth_file = Musicmanager.OAUTH_FILEPATH + else: + oauth_file = gmusicapi.clients.OAUTH_FILEPATH + self.config.add({ u'auto': False, u'uploader_id': '', u'uploader_name': '', u'device_id': '', - u'oauth_file': gmusicapi.clients.OAUTH_FILEPATH, + u'oauth_file': oauth_file, }) if self.config['auto']: self.import_stages = [self.autoupload] diff --git a/docs/changelog.rst b/docs/changelog.rst index 79aa53d3c..8bd64e7e4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,9 @@ Fixes: * :doc:`/plugins/gmusic`: The ``oauth_file`` config option now supports more flexible path values, including ``~`` for the home directory. :bug:`3270` +* :doc:`/plugins/gmusic`: Fix a crash when using version 12.0.0 or later of + the ``gmusicapi`` module. + :bug:`3270` 1.4.8 (May 16, 2019) From b193f250ba6e6864e0e6873d1771602db9de5cce Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 00:18:24 +0200 Subject: [PATCH 258/339] add work, work-disambig and work_id tags --- beets/autotag/hooks.py | 9 ++++++++- beets/autotag/mb.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index ec7047b7c..942b8bd7f 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -159,6 +159,9 @@ class TrackInfo(object): - ``composer_sort``: individual track composer sort name - ``arranger`: individual track arranger name - ``track_alt``: alternative track number (tape, vinyl, etc.) + - ``work`: individual track work title + - ``work_id`: individual track work id + - ``work_disambig`: individual track work diambiguation Only ``title`` and ``track_id`` are required. The rest of the fields may be None. The indices ``index``, ``medium``, and ``medium_index`` @@ -169,7 +172,8 @@ class TrackInfo(object): medium_index=None, medium_total=None, artist_sort=None, disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, - composer_sort=None, arranger=None, track_alt=None): + composer_sort=None, arranger=None, track_alt=None, + work=None, work_id=None, work_disambig=None): self.title = title self.track_id = track_id self.release_track_id = release_track_id @@ -191,6 +195,9 @@ class TrackInfo(object): self.composer_sort = composer_sort self.arranger = arranger self.track_alt = track_alt + self.work = work + self.work_id = work_id + self.work_disambig = work_disambig # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 4ea56af7f..23ee58b5a 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -210,9 +210,18 @@ def track_info(recording, index=None, medium=None, medium_index=None, lyricist = [] composer = [] composer_sort = [] + work = [] + work_id = [] + work_disambig = [] for work_relation in recording.get('work-relation-list', ()): if work_relation['type'] != 'performance': continue + work.append(work_relation['work']['title']) + work_id.append(work_relation['work']['id']) + if 'disambiguation' in work_relation['work']: + work_disambig.append(work_relation['work']['disambiguation']) + else: + work_disambig.append('') for artist_relation in work_relation['work'].get( 'artist-relation-list', ()): if 'type' in artist_relation: @@ -237,6 +246,11 @@ def track_info(recording, index=None, medium=None, medium_index=None, arranger.append(artist_relation['artist']['name']) if arranger: info.arranger = u', '.join(arranger) + if work: + info.work = u', '.join(work) + info.work_id = u', '.join(work_id) + if all(dis for dis in work_disambig): + info.work_disambig = u', '.join(work_disambig) info.decode() return info From 4c197e6f19250c54bc0ee8b2bc0353ef41497a95 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 00:38:38 +0200 Subject: [PATCH 259/339] completed library and test files --- beets/mediafile.py | 6 ++++++ test/_common.py | 3 +++ test/test_mediafile.py | 1 + 3 files changed, 10 insertions(+) diff --git a/beets/mediafile.py b/beets/mediafile.py index 32a32fe1d..d1508b762 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1643,6 +1643,12 @@ class MediaFile(object): StorageStyle('COMPOSERSORT'), ASFStorageStyle('WM/Composersortorder'), ) + work = MediaField( + MP3DescStorageStyle(u'Work'), + MP4StorageStyle('----:com.apple.iTunes:Work'), + StorageStyle('WORK'), + ASFStorageStyle('beets/Work'), + ) arranger = MediaField( MP3PeopleStorageStyle('TIPL', involvement='arranger'), MP4StorageStyle('----:com.apple.iTunes:Arranger'), diff --git a/test/_common.py b/test/_common.py index 99f2e968f..26add0a18 100644 --- a/test/_common.py +++ b/test/_common.py @@ -70,6 +70,9 @@ def item(lib=None): composer=u'the composer', arranger=u'the arranger', grouping=u'the grouping', + work=u'the work title', + work_id=u'the work musicbrainz id', + work_disambig=u'the work disambiguation', year=1, month=2, day=3, diff --git a/test/test_mediafile.py b/test/test_mediafile.py index 36a2c53ac..9af4b428f 100644 --- a/test/test_mediafile.py +++ b/test/test_mediafile.py @@ -352,6 +352,7 @@ class ReadWriteTestBase(ArtTestMixin, GenreListTestMixin, 'lyricist', 'composer', 'composer_sort', + 'work', 'arranger', 'grouping', 'year', From 5fe92730daf492c4ad6ae560e83be56f10fe5319 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 00:40:13 +0200 Subject: [PATCH 260/339] completed autotag/__int__ --- beets/autotag/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index a71b9b0a6..2f1033369 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -54,6 +54,12 @@ def apply_item_metadata(item, track_info): item.composer_sort = track_info.composer_sort if track_info.arranger is not None: item.arranger = track_info.arranger + if track_info.work is not None: + item.work = track_info.work + if track_info.work_id is not None: + item.work_id = track_info.work_id + if track_info.work_disambig is not None: + item.work_disambig = track_info.work_disambig # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? From 42d10318f274269618a391235786ad7a474f6644 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 11:56:38 +0200 Subject: [PATCH 261/339] removed useless checks for disambiguation --- beets/autotag/mb.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 23ee58b5a..fb2878dba 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -57,6 +57,7 @@ class MusicBrainzAPIError(util.HumanReadableException): self._reasonstr(), self.verb, repr(self.query) ) + log = logging.getLogger('beets') RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups', @@ -220,8 +221,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, work_id.append(work_relation['work']['id']) if 'disambiguation' in work_relation['work']: work_disambig.append(work_relation['work']['disambiguation']) - else: - work_disambig.append('') + for artist_relation in work_relation['work'].get( 'artist-relation-list', ()): if 'type' in artist_relation: @@ -249,8 +249,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, if work: info.work = u', '.join(work) info.work_id = u', '.join(work_id) - if all(dis for dis in work_disambig): - info.work_disambig = u', '.join(work_disambig) + info.work_disambig = u', '.join(work_disambig) info.decode() return info From 2b64fd45fea59eecc090cd05d58cf59789a5206a Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 13:59:20 +0200 Subject: [PATCH 262/339] new changes since my first try --- beets/autotag/__init__.py | 3 +++ beets/library.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 2f1033369..e6153478a 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -173,6 +173,9 @@ def apply_metadata(album_info, mapping): 'composer', 'composer_sort', 'arranger', + 'work', + 'work_id', + 'work_disambig', ) } diff --git a/beets/library.py b/beets/library.py index 9a9d95256..36af5adf4 100644 --- a/beets/library.py +++ b/beets/library.py @@ -451,6 +451,9 @@ class Item(LibModel): 'lyricist': types.STRING, 'composer': types.STRING, 'composer_sort': types.STRING, + 'work': types.STRING, + 'work_id': types.STRING, + 'work_disambig': types.STRING, 'arranger': types.STRING, 'grouping': types.STRING, 'year': types.PaddedInt(4), From 1d809aa433e29f2161e196c17b58f2f8e6a393b6 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 15:03:23 +0200 Subject: [PATCH 263/339] update mediafile, small typos --- beets/autotag/mb.py | 1 - beets/library.py | 2 +- beets/mediafile.py | 12 +++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index fb2878dba..177411034 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -57,7 +57,6 @@ class MusicBrainzAPIError(util.HumanReadableException): self._reasonstr(), self.verb, repr(self.query) ) - log = logging.getLogger('beets') RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups', diff --git a/beets/library.py b/beets/library.py index 36af5adf4..6d143ef16 100644 --- a/beets/library.py +++ b/beets/library.py @@ -452,7 +452,7 @@ class Item(LibModel): 'composer': types.STRING, 'composer_sort': types.STRING, 'work': types.STRING, - 'work_id': types.STRING, + 'mb_workid': types.STRING, 'work_disambig': types.STRING, 'arranger': types.STRING, 'grouping': types.STRING, diff --git a/beets/mediafile.py b/beets/mediafile.py index d1508b762..1c07d64f8 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1644,10 +1644,16 @@ class MediaFile(object): ASFStorageStyle('WM/Composersortorder'), ) work = MediaField( - MP3DescStorageStyle(u'Work'), - MP4StorageStyle('----:com.apple.iTunes:Work'), + MP3StorageStyle('TIT1'), + MP4StorageStyle('\xa9wrk'), StorageStyle('WORK'), - ASFStorageStyle('beets/Work'), + ASFStorageStyle('WM/Work'), + ) + mb_workid = MediaField( + MP3StorageStyle('TXXX:MusicBrainz Work Id'), + MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Work Id'), + StorageStyle('MUSICBRAINZ_WORKID '), + ASFStorageStyle('MusicBrainz/Work Id'), ) arranger = MediaField( MP3PeopleStorageStyle('TIPL', involvement='arranger'), From f7205c09c3e6e231b483ef69b4c0c8c4767ba9ac Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 15:07:24 +0200 Subject: [PATCH 264/339] update mediafile: MP3 tag for mb_workid --- beets/mediafile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 1c07d64f8..d2a66379a 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1650,7 +1650,7 @@ class MediaFile(object): ASFStorageStyle('WM/Work'), ) mb_workid = MediaField( - MP3StorageStyle('TXXX:MusicBrainz Work Id'), + MP3DescStorageStyle(u'MusicBrainz Work Id'), MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Work Id'), StorageStyle('MUSICBRAINZ_WORKID '), ASFStorageStyle('MusicBrainz/Work Id'), From 0131d253ee676a5491f80952931e6bd34950acea Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 15:10:18 +0200 Subject: [PATCH 265/339] replace work_id by mb_workid everywhere --- beets/autotag/__init__.py | 6 +++--- beets/autotag/hooks.py | 6 +++--- beets/autotag/mb.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index e6153478a..ede4fbe12 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -56,8 +56,8 @@ def apply_item_metadata(item, track_info): item.arranger = track_info.arranger if track_info.work is not None: item.work = track_info.work - if track_info.work_id is not None: - item.work_id = track_info.work_id + if track_info.mb_workid is not None: + item.mb_workid = track_info.mb_workid if track_info.work_disambig is not None: item.work_disambig = track_info.work_disambig @@ -174,7 +174,7 @@ def apply_metadata(album_info, mapping): 'composer_sort', 'arranger', 'work', - 'work_id', + 'mb_workid', 'work_disambig', ) } diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 942b8bd7f..57cd1c309 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -160,7 +160,7 @@ class TrackInfo(object): - ``arranger`: individual track arranger name - ``track_alt``: alternative track number (tape, vinyl, etc.) - ``work`: individual track work title - - ``work_id`: individual track work id + - ``mb_workid`: individual track work id - ``work_disambig`: individual track work diambiguation Only ``title`` and ``track_id`` are required. The rest of the fields @@ -173,7 +173,7 @@ class TrackInfo(object): disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, track_alt=None, - work=None, work_id=None, work_disambig=None): + work=None, mb_workid=None, work_disambig=None): self.title = title self.track_id = track_id self.release_track_id = release_track_id @@ -196,7 +196,7 @@ class TrackInfo(object): self.arranger = arranger self.track_alt = track_alt self.work = work - self.work_id = work_id + self.mb_workid = mb_workid self.work_disambig = work_disambig # As above, work around a bug in python-musicbrainz-ngs. diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 177411034..a7bb6566d 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -211,13 +211,13 @@ def track_info(recording, index=None, medium=None, medium_index=None, composer = [] composer_sort = [] work = [] - work_id = [] + mb_workid = [] work_disambig = [] for work_relation in recording.get('work-relation-list', ()): if work_relation['type'] != 'performance': continue work.append(work_relation['work']['title']) - work_id.append(work_relation['work']['id']) + mb_workid.append(work_relation['work']['id']) if 'disambiguation' in work_relation['work']: work_disambig.append(work_relation['work']['disambiguation']) @@ -247,7 +247,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, info.arranger = u', '.join(arranger) if work: info.work = u', '.join(work) - info.work_id = u', '.join(work_id) + info.mb_workid = u', '.join(mb_workid) info.work_disambig = u', '.join(work_disambig) info.decode() From 94ef7012cd696020a2577fc0c2be52c08c3ebd7f Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sun, 26 May 2019 15:21:49 +0200 Subject: [PATCH 266/339] update tag mapping to picard 2.1, tag conflict between work and grouping resolved --- beets/mediafile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index d2a66379a..9443cada2 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1663,7 +1663,7 @@ class MediaFile(object): ) grouping = MediaField( - MP3StorageStyle('TIT1'), + MP3StorageStyle('GRP1'), MP4StorageStyle('\xa9grp'), StorageStyle('GROUPING'), ASFStorageStyle('WM/ContentGroupDescription'), From 7ebcda0c3f80848560e579245238d93a02123c25 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 27 May 2019 10:04:54 +0200 Subject: [PATCH 267/339] revert changes to mediafile --- beets/mediafile.py | 15 +-------------- test/test_mediafile.py | 1 - 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 9443cada2..941a2f8f1 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1643,27 +1643,14 @@ class MediaFile(object): StorageStyle('COMPOSERSORT'), ASFStorageStyle('WM/Composersortorder'), ) - work = MediaField( - MP3StorageStyle('TIT1'), - MP4StorageStyle('\xa9wrk'), - StorageStyle('WORK'), - ASFStorageStyle('WM/Work'), - ) - mb_workid = MediaField( - MP3DescStorageStyle(u'MusicBrainz Work Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Work Id'), - StorageStyle('MUSICBRAINZ_WORKID '), - ASFStorageStyle('MusicBrainz/Work Id'), - ) arranger = MediaField( MP3PeopleStorageStyle('TIPL', involvement='arranger'), MP4StorageStyle('----:com.apple.iTunes:Arranger'), StorageStyle('ARRANGER'), ASFStorageStyle('beets/Arranger'), ) - grouping = MediaField( - MP3StorageStyle('GRP1'), + MP3StorageStyle('TIT1'), MP4StorageStyle('\xa9grp'), StorageStyle('GROUPING'), ASFStorageStyle('WM/ContentGroupDescription'), diff --git a/test/test_mediafile.py b/test/test_mediafile.py index 9af4b428f..36a2c53ac 100644 --- a/test/test_mediafile.py +++ b/test/test_mediafile.py @@ -352,7 +352,6 @@ class ReadWriteTestBase(ArtTestMixin, GenreListTestMixin, 'lyricist', 'composer', 'composer_sort', - 'work', 'arranger', 'grouping', 'year', From f72ef0d563376024bb08c76c192e880b5994d04c Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 27 May 2019 10:55:12 +0200 Subject: [PATCH 268/339] newline snuck in there --- beets/mediafile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/mediafile.py b/beets/mediafile.py index 941a2f8f1..111b4d73b 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1649,6 +1649,7 @@ class MediaFile(object): StorageStyle('ARRANGER'), ASFStorageStyle('beets/Arranger'), ) + grouping = MediaField( MP3StorageStyle('TIT1'), MP4StorageStyle('\xa9grp'), From 76e754de41a7f9c0be49d62b9d4e192823e3cac7 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 27 May 2019 11:03:25 +0200 Subject: [PATCH 269/339] flake8 --- beets/mediafile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 111b4d73b..32a32fe1d 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1649,7 +1649,7 @@ class MediaFile(object): StorageStyle('ARRANGER'), ASFStorageStyle('beets/Arranger'), ) - + grouping = MediaField( MP3StorageStyle('TIT1'), MP4StorageStyle('\xa9grp'), From f0d96dcadd103d7e821a79e0b13ef988f4702ef1 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 27 May 2019 11:59:11 +0200 Subject: [PATCH 270/339] replace work_id by mb_workid --- test/_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/_common.py b/test/_common.py index 26add0a18..7a3ece4d8 100644 --- a/test/_common.py +++ b/test/_common.py @@ -71,7 +71,7 @@ def item(lib=None): arranger=u'the arranger', grouping=u'the grouping', work=u'the work title', - work_id=u'the work musicbrainz id', + mb_workid=u'the work musicbrainz id', work_disambig=u'the work disambiguation', year=1, month=2, From 0f74173ac20b76b98e38fa96f0daf615b9f55dc1 Mon Sep 17 00:00:00 2001 From: David Logie Date: Tue, 28 May 2019 11:57:50 +0100 Subject: [PATCH 271/339] Add support for NO_COLOR. --- beets/ui/__init__.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 622a1e7f0..5b8f9680a 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -529,22 +529,22 @@ def colorize(color_name, text): """Colorize text if colored output is enabled. (Like _colorize but conditional.) """ - if config['ui']['color']: - global COLORS - if not COLORS: - COLORS = dict((name, - config['ui']['colors'][name].as_str()) - for name in COLOR_NAMES) - # In case a 3rd party plugin is still passing the actual color ('red') - # instead of the abstract color name ('text_error') - color = COLORS.get(color_name) - if not color: - log.debug(u'Invalid color_name: {0}', color_name) - color = color_name - return _colorize(color, text) - else: + if not config['ui']['color'] or 'NO_COLOR' in os.environ.keys(): return text + global COLORS + if not COLORS: + COLORS = dict((name, + config['ui']['colors'][name].as_str()) + for name in COLOR_NAMES) + # In case a 3rd party plugin is still passing the actual color ('red') + # instead of the abstract color name ('text_error') + color = COLORS.get(color_name) + if not color: + log.debug(u'Invalid color_name: {0}', color_name) + color = color_name + return _colorize(color, text) + def _colordiff(a, b, highlight='text_highlight', minor_highlight='text_highlight_minor'): From 899d09fc2d8bdb295fa475164adf374cf6887d6c Mon Sep 17 00:00:00 2001 From: David Logie Date: Tue, 28 May 2019 12:12:50 +0100 Subject: [PATCH 272/339] Add changelog entry for #3273. --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8bd64e7e4..21b0ae0bc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,13 @@ Fixes: the ``gmusicapi`` module. :bug:`3270` +New features: + +* Support for the `NO_COLOR`_ environment variable. + :bug:`3273` + +.. _NO_COLOR: https://no-color.org + 1.4.8 (May 16, 2019) -------------------- From 8519f02335b89d84be65cb4d5bfcc0e8c471b34f Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 23 Apr 2019 17:28:25 +1000 Subject: [PATCH 273/339] mediafile: replace with a wrapper around mediafile --- beets/mediafile.py | 2096 +------------------------------------------- setup.py | 1 + 2 files changed, 9 insertions(+), 2088 deletions(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 32a32fe1d..373642b42 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -13,2096 +13,16 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Handles low-level interfacing for files' tags. Wraps Mutagen to -automatically detect file types and provide a unified interface for a -useful subset of music files' tags. - -Usage: - - >>> f = MediaFile('Lucy.mp3') - >>> f.title - u'Lucy in the Sky with Diamonds' - >>> f.artist = 'The Beatles' - >>> f.save() - -A field will always return a reasonable value of the correct type, even -if no tag is present. If no value is available, the value will be false -(e.g., zero or the empty string). - -Internally ``MediaFile`` uses ``MediaField`` descriptors to access the -data from the tags. In turn ``MediaField`` uses a number of -``StorageStyle`` strategies to handle format specific logic. -""" from __future__ import division, absolute_import, print_function -import mutagen -import mutagen.id3 -import mutagen.mp4 -import mutagen.flac -import mutagen.asf +import mediafile -import codecs -import datetime -import re -import base64 -import binascii -import math -import struct -import imghdr -import os -import traceback -import enum -import logging -import six +import warnings +warnings.warn("beets.mediafile is deprecated; use mediafile instead") +# Import everything from the mediafile module into this module. +for key, value in mediafile.__dict__.items(): + if key not in ['__name__']: + globals()[key] = value -__all__ = ['UnreadableFileError', 'FileTypeError', 'MediaFile'] - -log = logging.getLogger(__name__) - -# Human-readable type names. -TYPES = { - 'mp3': 'MP3', - 'aac': 'AAC', - 'alac': 'ALAC', - 'ogg': 'OGG', - 'opus': 'Opus', - 'flac': 'FLAC', - 'ape': 'APE', - 'wv': 'WavPack', - 'mpc': 'Musepack', - 'asf': 'Windows Media', - 'aiff': 'AIFF', - 'dsf': 'DSD Stream File', -} - -PREFERRED_IMAGE_EXTENSIONS = {'jpeg': 'jpg'} - - -# Exceptions. - -class UnreadableFileError(Exception): - """Mutagen is not able to extract information from the file. - """ - def __init__(self, path, msg): - Exception.__init__(self, msg if msg else repr(path)) - - -class FileTypeError(UnreadableFileError): - """Reading this type of file is not supported. - - If passed the `mutagen_type` argument this indicates that the - mutagen type is not supported by `Mediafile`. - """ - def __init__(self, path, mutagen_type=None): - if mutagen_type is None: - msg = u'{0!r}: not in a recognized format'.format(path) - else: - msg = u'{0}: of mutagen type {1}'.format(repr(path), mutagen_type) - Exception.__init__(self, msg) - - -class MutagenError(UnreadableFileError): - """Raised when Mutagen fails unexpectedly---probably due to a bug. - """ - def __init__(self, path, mutagen_exc): - msg = u'{0}: {1}'.format(repr(path), mutagen_exc) - Exception.__init__(self, msg) - - -# Interacting with Mutagen. - -def mutagen_call(action, path, func, *args, **kwargs): - """Call a Mutagen function with appropriate error handling. - - `action` is a string describing what the function is trying to do, - and `path` is the relevant filename. The rest of the arguments - describe the callable to invoke. - - We require at least Mutagen 1.33, where `IOError` is *never* used, - neither for internal parsing errors *nor* for ordinary IO error - conditions such as a bad filename. Mutagen-specific parsing errors and IO - errors are reraised as `UnreadableFileError`. Other exceptions - raised inside Mutagen---i.e., bugs---are reraised as `MutagenError`. - """ - try: - return func(*args, **kwargs) - except mutagen.MutagenError as exc: - log.debug(u'%s failed: %s', action, six.text_type(exc)) - raise UnreadableFileError(path, six.text_type(exc)) - except Exception as exc: - # Isolate bugs in Mutagen. - log.debug(u'%s', traceback.format_exc()) - log.error(u'uncaught Mutagen exception in %s: %s', action, exc) - raise MutagenError(path, exc) - - -# Utility. - -def _safe_cast(out_type, val): - """Try to covert val to out_type but never raise an exception. If - the value can't be converted, then a sensible default value is - returned. out_type should be bool, int, or unicode; otherwise, the - value is just passed through. - """ - if val is None: - return None - - if out_type == int: - if isinstance(val, int) or isinstance(val, float): - # Just a number. - return int(val) - else: - # Process any other type as a string. - if isinstance(val, bytes): - val = val.decode('utf-8', 'ignore') - elif not isinstance(val, six.string_types): - val = six.text_type(val) - # Get a number from the front of the string. - match = re.match(r'[\+-]?[0-9]+', val.strip()) - return int(match.group(0)) if match else 0 - - elif out_type == bool: - try: - # Should work for strings, bools, ints: - return bool(int(val)) - except ValueError: - return False - - elif out_type == six.text_type: - if isinstance(val, bytes): - return val.decode('utf-8', 'ignore') - elif isinstance(val, six.text_type): - return val - else: - return six.text_type(val) - - elif out_type == float: - if isinstance(val, int) or isinstance(val, float): - return float(val) - else: - if isinstance(val, bytes): - val = val.decode('utf-8', 'ignore') - else: - val = six.text_type(val) - match = re.match(r'[\+-]?([0-9]+\.?[0-9]*|[0-9]*\.[0-9]+)', - val.strip()) - if match: - val = match.group(0) - if val: - return float(val) - return 0.0 - - else: - return val - - -# Image coding for ASF/WMA. - -def _unpack_asf_image(data): - """Unpack image data from a WM/Picture tag. Return a tuple - containing the MIME type, the raw image data, a type indicator, and - the image's description. - - This function is treated as "untrusted" and could throw all manner - of exceptions (out-of-bounds, etc.). We should clean this up - sometime so that the failure modes are well-defined. - """ - type, size = struct.unpack_from(' 0: - gain = math.log10(maxgain / 1000.0) * -10 - else: - # Invalid gain value found. - gain = 0.0 - - # SoundCheck stores peak values as the actual value of the sample, - # and again separately for the left and right channels. We need to - # convert this to a percentage of full scale, which is 32768 for a - # 16 bit sample. Once again, we play it safe by using the larger of - # the two values. - peak = max(soundcheck[6:8]) / 32768.0 - - return round(gain, 2), round(peak, 6) - - -def _sc_encode(gain, peak): - """Encode ReplayGain gain/peak values as a Sound Check string. - """ - # SoundCheck stores the peak value as the actual value of the - # sample, rather than the percentage of full scale that RG uses, so - # we do a simple conversion assuming 16 bit samples. - peak *= 32768.0 - - # SoundCheck stores absolute RMS values in some unknown units rather - # than the dB values RG uses. We can calculate these absolute values - # from the gain ratio using a reference value of 1000 units. We also - # enforce the maximum value here, which is equivalent to about - # -18.2dB. - g1 = int(min(round((10 ** (gain / -10)) * 1000), 65534)) - # Same as above, except our reference level is 2500 units. - g2 = int(min(round((10 ** (gain / -10)) * 2500), 65534)) - - # The purpose of these values are unknown, but they also seem to be - # unused so we just use zero. - uk = 0 - values = (g1, g1, g2, g2, uk, uk, int(peak), int(peak), uk, uk) - return (u' %08X' * 10) % values - - -# Cover art and other images. -def _imghdr_what_wrapper(data): - """A wrapper around imghdr.what to account for jpeg files that can only be - identified as such using their magic bytes - See #1545 - See https://github.com/file/file/blob/master/magic/Magdir/jpeg#L12 - """ - # imghdr.what returns none for jpegs with only the magic bytes, so - # _wider_test_jpeg is run in that case. It still returns None if it didn't - # match such a jpeg file. - return imghdr.what(None, h=data) or _wider_test_jpeg(data) - - -def _wider_test_jpeg(data): - """Test for a jpeg file following the UNIX file implementation which - uses the magic bytes rather than just looking for the bytes that - represent 'JFIF' or 'EXIF' at a fixed position. - """ - if data[:2] == b'\xff\xd8': - return 'jpeg' - - -def image_mime_type(data): - """Return the MIME type of the image data (a bytestring). - """ - # This checks for a jpeg file with only the magic bytes (unrecognized by - # imghdr.what). imghdr.what returns none for that type of file, so - # _wider_test_jpeg is run in that case. It still returns None if it didn't - # match such a jpeg file. - kind = _imghdr_what_wrapper(data) - if kind in ['gif', 'jpeg', 'png', 'tiff', 'bmp']: - return 'image/{0}'.format(kind) - elif kind == 'pgm': - return 'image/x-portable-graymap' - elif kind == 'pbm': - return 'image/x-portable-bitmap' - elif kind == 'ppm': - return 'image/x-portable-pixmap' - elif kind == 'xbm': - return 'image/x-xbitmap' - else: - return 'image/x-{0}'.format(kind) - - -def image_extension(data): - ext = _imghdr_what_wrapper(data) - return PREFERRED_IMAGE_EXTENSIONS.get(ext, ext) - - -class ImageType(enum.Enum): - """Indicates the kind of an `Image` stored in a file's tag. - """ - other = 0 - icon = 1 - other_icon = 2 - front = 3 - back = 4 - leaflet = 5 - media = 6 - lead_artist = 7 - artist = 8 - conductor = 9 - group = 10 - composer = 11 - lyricist = 12 - recording_location = 13 - recording_session = 14 - performance = 15 - screen_capture = 16 - fish = 17 - illustration = 18 - artist_logo = 19 - publisher_logo = 20 - - -class Image(object): - """Structure representing image data and metadata that can be - stored and retrieved from tags. - - The structure has four properties. - * ``data`` The binary data of the image - * ``desc`` An optional description of the image - * ``type`` An instance of `ImageType` indicating the kind of image - * ``mime_type`` Read-only property that contains the mime type of - the binary data - """ - def __init__(self, data, desc=None, type=None): - assert isinstance(data, bytes) - if desc is not None: - assert isinstance(desc, six.text_type) - self.data = data - self.desc = desc - if isinstance(type, int): - try: - type = list(ImageType)[type] - except IndexError: - log.debug(u"ignoring unknown image type index %s", type) - type = ImageType.other - self.type = type - - @property - def mime_type(self): - if self.data: - return image_mime_type(self.data) - - @property - def type_index(self): - if self.type is None: - # This method is used when a tag format requires the type - # index to be set, so we return "other" as the default value. - return 0 - return self.type.value - - -# StorageStyle classes describe strategies for accessing values in -# Mutagen file objects. - -class StorageStyle(object): - """A strategy for storing a value for a certain tag format (or set - of tag formats). This basic StorageStyle describes simple 1:1 - mapping from raw values to keys in a Mutagen file object; subclasses - describe more sophisticated translations or format-specific access - strategies. - - MediaFile uses a StorageStyle via three methods: ``get()``, - ``set()``, and ``delete()``. It passes a Mutagen file object to - each. - - Internally, the StorageStyle implements ``get()`` and ``set()`` - using two steps that may be overridden by subtypes. To get a value, - the StorageStyle first calls ``fetch()`` to retrieve the value - corresponding to a key and then ``deserialize()`` to convert the raw - Mutagen value to a consumable Python value. Similarly, to set a - field, we call ``serialize()`` to encode the value and then - ``store()`` to assign the result into the Mutagen object. - - Each StorageStyle type has a class-level `formats` attribute that is - a list of strings indicating the formats that the style applies to. - MediaFile only uses StorageStyles that apply to the correct type for - a given audio file. - """ - - formats = ['FLAC', 'OggOpus', 'OggTheora', 'OggSpeex', 'OggVorbis', - 'OggFlac', 'APEv2File', 'WavPack', 'Musepack', 'MonkeysAudio'] - """List of mutagen classes the StorageStyle can handle. - """ - - def __init__(self, key, as_type=six.text_type, suffix=None, - float_places=2): - """Create a basic storage strategy. Parameters: - - - `key`: The key on the Mutagen file object used to access the - field's data. - - `as_type`: The Python type that the value is stored as - internally (`unicode`, `int`, `bool`, or `bytes`). - - `suffix`: When `as_type` is a string type, append this before - storing the value. - - `float_places`: When the value is a floating-point number and - encoded as a string, the number of digits to store after the - decimal point. - """ - self.key = key - self.as_type = as_type - self.suffix = suffix - self.float_places = float_places - - # Convert suffix to correct string type. - if self.suffix and self.as_type is six.text_type \ - and not isinstance(self.suffix, six.text_type): - self.suffix = self.suffix.decode('utf-8') - - # Getter. - - def get(self, mutagen_file): - """Get the value for the field using this style. - """ - return self.deserialize(self.fetch(mutagen_file)) - - def fetch(self, mutagen_file): - """Retrieve the raw value of for this tag from the Mutagen file - object. - """ - try: - return mutagen_file[self.key][0] - except (KeyError, IndexError): - return None - - def deserialize(self, mutagen_value): - """Given a raw value stored on a Mutagen object, decode and - return the represented value. - """ - if self.suffix and isinstance(mutagen_value, six.text_type) \ - and mutagen_value.endswith(self.suffix): - return mutagen_value[:-len(self.suffix)] - else: - return mutagen_value - - # Setter. - - def set(self, mutagen_file, value): - """Assign the value for the field using this style. - """ - self.store(mutagen_file, self.serialize(value)) - - def store(self, mutagen_file, value): - """Store a serialized value in the Mutagen file object. - """ - mutagen_file[self.key] = [value] - - def serialize(self, value): - """Convert the external Python value to a type that is suitable for - storing in a Mutagen file object. - """ - if isinstance(value, float) and self.as_type is six.text_type: - value = u'{0:.{1}f}'.format(value, self.float_places) - value = self.as_type(value) - elif self.as_type is six.text_type: - if isinstance(value, bool): - # Store bools as 1/0 instead of True/False. - value = six.text_type(int(bool(value))) - elif isinstance(value, bytes): - value = value.decode('utf-8', 'ignore') - else: - value = six.text_type(value) - else: - value = self.as_type(value) - - if self.suffix: - value += self.suffix - - return value - - def delete(self, mutagen_file): - """Remove the tag from the file. - """ - if self.key in mutagen_file: - del mutagen_file[self.key] - - -class ListStorageStyle(StorageStyle): - """Abstract storage style that provides access to lists. - - The ListMediaField descriptor uses a ListStorageStyle via two - methods: ``get_list()`` and ``set_list()``. It passes a Mutagen file - object to each. - - Subclasses may overwrite ``fetch`` and ``store``. ``fetch`` must - return a (possibly empty) list and ``store`` receives a serialized - list of values as the second argument. - - The `serialize` and `deserialize` methods (from the base - `StorageStyle`) are still called with individual values. This class - handles packing and unpacking the values into lists. - """ - def get(self, mutagen_file): - """Get the first value in the field's value list. - """ - try: - return self.get_list(mutagen_file)[0] - except IndexError: - return None - - def get_list(self, mutagen_file): - """Get a list of all values for the field using this style. - """ - return [self.deserialize(item) for item in self.fetch(mutagen_file)] - - def fetch(self, mutagen_file): - """Get the list of raw (serialized) values. - """ - try: - return mutagen_file[self.key] - except KeyError: - return [] - - def set(self, mutagen_file, value): - """Set an individual value as the only value for the field using - this style. - """ - self.set_list(mutagen_file, [value]) - - def set_list(self, mutagen_file, values): - """Set all values for the field using this style. `values` - should be an iterable. - """ - self.store(mutagen_file, [self.serialize(value) for value in values]) - - def store(self, mutagen_file, values): - """Set the list of all raw (serialized) values for this field. - """ - mutagen_file[self.key] = values - - -class SoundCheckStorageStyleMixin(object): - """A mixin for storage styles that read and write iTunes SoundCheck - analysis values. The object must have an `index` field that - indicates which half of the gain/peak pair---0 or 1---the field - represents. - """ - def get(self, mutagen_file): - data = self.fetch(mutagen_file) - if data is not None: - return _sc_decode(data)[self.index] - - def set(self, mutagen_file, value): - data = self.fetch(mutagen_file) - if data is None: - gain_peak = [0, 0] - else: - gain_peak = list(_sc_decode(data)) - gain_peak[self.index] = value or 0 - data = self.serialize(_sc_encode(*gain_peak)) - self.store(mutagen_file, data) - - -class ASFStorageStyle(ListStorageStyle): - """A general storage style for Windows Media/ASF files. - """ - formats = ['ASF'] - - def deserialize(self, data): - if isinstance(data, mutagen.asf.ASFBaseAttribute): - data = data.value - return data - - -class MP4StorageStyle(StorageStyle): - """A general storage style for MPEG-4 tags. - """ - formats = ['MP4'] - - def serialize(self, value): - value = super(MP4StorageStyle, self).serialize(value) - if self.key.startswith('----:') and isinstance(value, six.text_type): - value = value.encode('utf-8') - return value - - -class MP4TupleStorageStyle(MP4StorageStyle): - """A style for storing values as part of a pair of numbers in an - MPEG-4 file. - """ - def __init__(self, key, index=0, **kwargs): - super(MP4TupleStorageStyle, self).__init__(key, **kwargs) - self.index = index - - def deserialize(self, mutagen_value): - items = mutagen_value or [] - packing_length = 2 - return list(items) + [0] * (packing_length - len(items)) - - def get(self, mutagen_file): - value = super(MP4TupleStorageStyle, self).get(mutagen_file)[self.index] - if value == 0: - # The values are always present and saved as integers. So we - # assume that "0" indicates it is not set. - return None - else: - return value - - def set(self, mutagen_file, value): - if value is None: - value = 0 - items = self.deserialize(self.fetch(mutagen_file)) - items[self.index] = int(value) - self.store(mutagen_file, items) - - def delete(self, mutagen_file): - if self.index == 0: - super(MP4TupleStorageStyle, self).delete(mutagen_file) - else: - self.set(mutagen_file, None) - - -class MP4ListStorageStyle(ListStorageStyle, MP4StorageStyle): - pass - - -class MP4SoundCheckStorageStyle(SoundCheckStorageStyleMixin, MP4StorageStyle): - def __init__(self, key, index=0, **kwargs): - super(MP4SoundCheckStorageStyle, self).__init__(key, **kwargs) - self.index = index - - -class MP4BoolStorageStyle(MP4StorageStyle): - """A style for booleans in MPEG-4 files. (MPEG-4 has an atom type - specifically for representing booleans.) - """ - def get(self, mutagen_file): - try: - return mutagen_file[self.key] - except KeyError: - return None - - def get_list(self, mutagen_file): - raise NotImplementedError(u'MP4 bool storage does not support lists') - - def set(self, mutagen_file, value): - mutagen_file[self.key] = value - - def set_list(self, mutagen_file, values): - raise NotImplementedError(u'MP4 bool storage does not support lists') - - -class MP4ImageStorageStyle(MP4ListStorageStyle): - """Store images as MPEG-4 image atoms. Values are `Image` objects. - """ - def __init__(self, **kwargs): - super(MP4ImageStorageStyle, self).__init__(key='covr', **kwargs) - - def deserialize(self, data): - return Image(data) - - def serialize(self, image): - if image.mime_type == 'image/png': - kind = mutagen.mp4.MP4Cover.FORMAT_PNG - elif image.mime_type == 'image/jpeg': - kind = mutagen.mp4.MP4Cover.FORMAT_JPEG - else: - raise ValueError(u'MP4 files only supports PNG and JPEG images') - return mutagen.mp4.MP4Cover(image.data, kind) - - -class MP3StorageStyle(StorageStyle): - """Store data in ID3 frames. - """ - formats = ['MP3', 'AIFF', 'DSF'] - - def __init__(self, key, id3_lang=None, **kwargs): - """Create a new ID3 storage style. `id3_lang` is the value for - the language field of newly created frames. - """ - self.id3_lang = id3_lang - super(MP3StorageStyle, self).__init__(key, **kwargs) - - def fetch(self, mutagen_file): - try: - return mutagen_file[self.key].text[0] - except (KeyError, IndexError): - return None - - def store(self, mutagen_file, value): - frame = mutagen.id3.Frames[self.key](encoding=3, text=[value]) - mutagen_file.tags.setall(self.key, [frame]) - - -class MP3PeopleStorageStyle(MP3StorageStyle): - """Store list of people in ID3 frames. - """ - def __init__(self, key, involvement='', **kwargs): - self.involvement = involvement - super(MP3PeopleStorageStyle, self).__init__(key, **kwargs) - - def store(self, mutagen_file, value): - frames = mutagen_file.tags.getall(self.key) - - # Try modifying in place. - found = False - for frame in frames: - if frame.encoding == mutagen.id3.Encoding.UTF8: - for pair in frame.people: - if pair[0].lower() == self.involvement.lower(): - pair[1] = value - found = True - - # Try creating a new frame. - if not found: - frame = mutagen.id3.Frames[self.key]( - encoding=mutagen.id3.Encoding.UTF8, - people=[[self.involvement, value]] - ) - mutagen_file.tags.add(frame) - - def fetch(self, mutagen_file): - for frame in mutagen_file.tags.getall(self.key): - for pair in frame.people: - if pair[0].lower() == self.involvement.lower(): - try: - return pair[1] - except IndexError: - return None - - -class MP3ListStorageStyle(ListStorageStyle, MP3StorageStyle): - """Store lists of data in multiple ID3 frames. - """ - def fetch(self, mutagen_file): - try: - return mutagen_file[self.key].text - except KeyError: - return [] - - def store(self, mutagen_file, values): - frame = mutagen.id3.Frames[self.key](encoding=3, text=values) - mutagen_file.tags.setall(self.key, [frame]) - - -class MP3UFIDStorageStyle(MP3StorageStyle): - """Store string data in a UFID ID3 frame with a particular owner. - """ - def __init__(self, owner, **kwargs): - self.owner = owner - super(MP3UFIDStorageStyle, self).__init__('UFID:' + owner, **kwargs) - - def fetch(self, mutagen_file): - try: - return mutagen_file[self.key].data - except KeyError: - return None - - def store(self, mutagen_file, value): - # This field type stores text data as encoded data. - assert isinstance(value, six.text_type) - value = value.encode('utf-8') - - frames = mutagen_file.tags.getall(self.key) - for frame in frames: - # Replace existing frame data. - if frame.owner == self.owner: - frame.data = value - else: - # New frame. - frame = mutagen.id3.UFID(owner=self.owner, data=value) - mutagen_file.tags.setall(self.key, [frame]) - - -class MP3DescStorageStyle(MP3StorageStyle): - """Store data in a TXXX (or similar) ID3 frame. The frame is - selected based its ``desc`` field. - """ - def __init__(self, desc=u'', key='TXXX', **kwargs): - assert isinstance(desc, six.text_type) - self.description = desc - super(MP3DescStorageStyle, self).__init__(key=key, **kwargs) - - def store(self, mutagen_file, value): - frames = mutagen_file.tags.getall(self.key) - if self.key != 'USLT': - value = [value] - - # Try modifying in place. - found = False - for frame in frames: - if frame.desc.lower() == self.description.lower(): - frame.text = value - frame.encoding = mutagen.id3.Encoding.UTF8 - found = True - - # Try creating a new frame. - if not found: - frame = mutagen.id3.Frames[self.key]( - desc=self.description, - text=value, - encoding=mutagen.id3.Encoding.UTF8, - ) - if self.id3_lang: - frame.lang = self.id3_lang - mutagen_file.tags.add(frame) - - def fetch(self, mutagen_file): - for frame in mutagen_file.tags.getall(self.key): - if frame.desc.lower() == self.description.lower(): - if self.key == 'USLT': - return frame.text - try: - return frame.text[0] - except IndexError: - return None - - def delete(self, mutagen_file): - found_frame = None - for frame in mutagen_file.tags.getall(self.key): - if frame.desc.lower() == self.description.lower(): - found_frame = frame - break - if found_frame is not None: - del mutagen_file[frame.HashKey] - - -class MP3SlashPackStorageStyle(MP3StorageStyle): - """Store value as part of pair that is serialized as a slash- - separated string. - """ - def __init__(self, key, pack_pos=0, **kwargs): - super(MP3SlashPackStorageStyle, self).__init__(key, **kwargs) - self.pack_pos = pack_pos - - def _fetch_unpacked(self, mutagen_file): - data = self.fetch(mutagen_file) - if data: - items = six.text_type(data).split('/') - else: - items = [] - packing_length = 2 - return list(items) + [None] * (packing_length - len(items)) - - def get(self, mutagen_file): - return self._fetch_unpacked(mutagen_file)[self.pack_pos] - - def set(self, mutagen_file, value): - items = self._fetch_unpacked(mutagen_file) - items[self.pack_pos] = value - if items[0] is None: - items[0] = '' - if items[1] is None: - items.pop() # Do not store last value - self.store(mutagen_file, '/'.join(map(six.text_type, items))) - - def delete(self, mutagen_file): - if self.pack_pos == 0: - super(MP3SlashPackStorageStyle, self).delete(mutagen_file) - else: - self.set(mutagen_file, None) - - -class MP3ImageStorageStyle(ListStorageStyle, MP3StorageStyle): - """Converts between APIC frames and ``Image`` instances. - - The `get_list` method inherited from ``ListStorageStyle`` returns a - list of ``Image``s. Similarly, the `set_list` method accepts a - list of ``Image``s as its ``values`` argument. - """ - def __init__(self): - super(MP3ImageStorageStyle, self).__init__(key='APIC') - self.as_type = bytes - - def deserialize(self, apic_frame): - """Convert APIC frame into Image.""" - return Image(data=apic_frame.data, desc=apic_frame.desc, - type=apic_frame.type) - - def fetch(self, mutagen_file): - return mutagen_file.tags.getall(self.key) - - def store(self, mutagen_file, frames): - mutagen_file.tags.setall(self.key, frames) - - def delete(self, mutagen_file): - mutagen_file.tags.delall(self.key) - - def serialize(self, image): - """Return an APIC frame populated with data from ``image``. - """ - assert isinstance(image, Image) - frame = mutagen.id3.Frames[self.key]() - frame.data = image.data - frame.mime = image.mime_type - frame.desc = image.desc or u'' - - # For compatibility with OS X/iTunes prefer latin-1 if possible. - # See issue #899 - try: - frame.desc.encode("latin-1") - except UnicodeEncodeError: - frame.encoding = mutagen.id3.Encoding.UTF16 - else: - frame.encoding = mutagen.id3.Encoding.LATIN1 - - frame.type = image.type_index - return frame - - -class MP3SoundCheckStorageStyle(SoundCheckStorageStyleMixin, - MP3DescStorageStyle): - def __init__(self, index=0, **kwargs): - super(MP3SoundCheckStorageStyle, self).__init__(**kwargs) - self.index = index - - -class ASFImageStorageStyle(ListStorageStyle): - """Store images packed into Windows Media/ASF byte array attributes. - Values are `Image` objects. - """ - formats = ['ASF'] - - def __init__(self): - super(ASFImageStorageStyle, self).__init__(key='WM/Picture') - - def deserialize(self, asf_picture): - mime, data, type, desc = _unpack_asf_image(asf_picture.value) - return Image(data, desc=desc, type=type) - - def serialize(self, image): - pic = mutagen.asf.ASFByteArrayAttribute() - pic.value = _pack_asf_image(image.mime_type, image.data, - type=image.type_index, - description=image.desc or u'') - return pic - - -class VorbisImageStorageStyle(ListStorageStyle): - """Store images in Vorbis comments. Both legacy COVERART fields and - modern METADATA_BLOCK_PICTURE tags are supported. Data is - base64-encoded. Values are `Image` objects. - """ - formats = ['OggOpus', 'OggTheora', 'OggSpeex', 'OggVorbis', - 'OggFlac'] - - def __init__(self): - super(VorbisImageStorageStyle, self).__init__( - key='metadata_block_picture' - ) - self.as_type = bytes - - def fetch(self, mutagen_file): - images = [] - if 'metadata_block_picture' not in mutagen_file: - # Try legacy COVERART tags. - if 'coverart' in mutagen_file: - for data in mutagen_file['coverart']: - images.append(Image(base64.b64decode(data))) - return images - for data in mutagen_file["metadata_block_picture"]: - try: - pic = mutagen.flac.Picture(base64.b64decode(data)) - except (TypeError, AttributeError): - continue - images.append(Image(data=pic.data, desc=pic.desc, - type=pic.type)) - return images - - def store(self, mutagen_file, image_data): - # Strip all art, including legacy COVERART. - if 'coverart' in mutagen_file: - del mutagen_file['coverart'] - if 'coverartmime' in mutagen_file: - del mutagen_file['coverartmime'] - super(VorbisImageStorageStyle, self).store(mutagen_file, image_data) - - def serialize(self, image): - """Turn a Image into a base64 encoded FLAC picture block. - """ - pic = mutagen.flac.Picture() - pic.data = image.data - pic.type = image.type_index - pic.mime = image.mime_type - pic.desc = image.desc or u'' - - # Encoding with base64 returns bytes on both Python 2 and 3. - # Mutagen requires the data to be a Unicode string, so we decode - # it before passing it along. - return base64.b64encode(pic.write()).decode('ascii') - - -class FlacImageStorageStyle(ListStorageStyle): - """Converts between ``mutagen.flac.Picture`` and ``Image`` instances. - """ - formats = ['FLAC'] - - def __init__(self): - super(FlacImageStorageStyle, self).__init__(key='') - - def fetch(self, mutagen_file): - return mutagen_file.pictures - - def deserialize(self, flac_picture): - return Image(data=flac_picture.data, desc=flac_picture.desc, - type=flac_picture.type) - - def store(self, mutagen_file, pictures): - """``pictures`` is a list of mutagen.flac.Picture instances. - """ - mutagen_file.clear_pictures() - for pic in pictures: - mutagen_file.add_picture(pic) - - def serialize(self, image): - """Turn a Image into a mutagen.flac.Picture. - """ - pic = mutagen.flac.Picture() - pic.data = image.data - pic.type = image.type_index - pic.mime = image.mime_type - pic.desc = image.desc or u'' - return pic - - def delete(self, mutagen_file): - """Remove all images from the file. - """ - mutagen_file.clear_pictures() - - -class APEv2ImageStorageStyle(ListStorageStyle): - """Store images in APEv2 tags. Values are `Image` objects. - """ - formats = ['APEv2File', 'WavPack', 'Musepack', 'MonkeysAudio', 'OptimFROG'] - - TAG_NAMES = { - ImageType.other: 'Cover Art (other)', - ImageType.icon: 'Cover Art (icon)', - ImageType.other_icon: 'Cover Art (other icon)', - ImageType.front: 'Cover Art (front)', - ImageType.back: 'Cover Art (back)', - ImageType.leaflet: 'Cover Art (leaflet)', - ImageType.media: 'Cover Art (media)', - ImageType.lead_artist: 'Cover Art (lead)', - ImageType.artist: 'Cover Art (artist)', - ImageType.conductor: 'Cover Art (conductor)', - ImageType.group: 'Cover Art (band)', - ImageType.composer: 'Cover Art (composer)', - ImageType.lyricist: 'Cover Art (lyricist)', - ImageType.recording_location: 'Cover Art (studio)', - ImageType.recording_session: 'Cover Art (recording)', - ImageType.performance: 'Cover Art (performance)', - ImageType.screen_capture: 'Cover Art (movie scene)', - ImageType.fish: 'Cover Art (colored fish)', - ImageType.illustration: 'Cover Art (illustration)', - ImageType.artist_logo: 'Cover Art (band logo)', - ImageType.publisher_logo: 'Cover Art (publisher logo)', - } - - def __init__(self): - super(APEv2ImageStorageStyle, self).__init__(key='') - - def fetch(self, mutagen_file): - images = [] - for cover_type, cover_tag in self.TAG_NAMES.items(): - try: - frame = mutagen_file[cover_tag] - text_delimiter_index = frame.value.find(b'\x00') - if text_delimiter_index > 0: - comment = frame.value[0:text_delimiter_index] - comment = comment.decode('utf-8', 'replace') - else: - comment = None - image_data = frame.value[text_delimiter_index + 1:] - images.append(Image(data=image_data, type=cover_type, - desc=comment)) - except KeyError: - pass - - return images - - def set_list(self, mutagen_file, values): - self.delete(mutagen_file) - - for image in values: - image_type = image.type or ImageType.other - comment = image.desc or '' - image_data = comment.encode('utf-8') + b'\x00' + image.data - cover_tag = self.TAG_NAMES[image_type] - mutagen_file[cover_tag] = image_data - - def delete(self, mutagen_file): - """Remove all images from the file. - """ - for cover_tag in self.TAG_NAMES.values(): - try: - del mutagen_file[cover_tag] - except KeyError: - pass - - -# MediaField is a descriptor that represents a single logical field. It -# aggregates several StorageStyles describing how to access the data for -# each file type. - -class MediaField(object): - """A descriptor providing access to a particular (abstract) metadata - field. - """ - def __init__(self, *styles, **kwargs): - """Creates a new MediaField. - - :param styles: `StorageStyle` instances that describe the strategy - for reading and writing the field in particular - formats. There must be at least one style for - each possible file format. - - :param out_type: the type of the value that should be returned when - getting this property. - - """ - self.out_type = kwargs.get('out_type', six.text_type) - self._styles = styles - - def styles(self, mutagen_file): - """Yields the list of storage styles of this field that can - handle the MediaFile's format. - """ - for style in self._styles: - if mutagen_file.__class__.__name__ in style.formats: - yield style - - def __get__(self, mediafile, owner=None): - out = None - for style in self.styles(mediafile.mgfile): - out = style.get(mediafile.mgfile) - if out: - break - return _safe_cast(self.out_type, out) - - def __set__(self, mediafile, value): - if value is None: - value = self._none_value() - for style in self.styles(mediafile.mgfile): - style.set(mediafile.mgfile, value) - - def __delete__(self, mediafile): - for style in self.styles(mediafile.mgfile): - style.delete(mediafile.mgfile) - - def _none_value(self): - """Get an appropriate "null" value for this field's type. This - is used internally when setting the field to None. - """ - if self.out_type == int: - return 0 - elif self.out_type == float: - return 0.0 - elif self.out_type == bool: - return False - elif self.out_type == six.text_type: - return u'' - - -class ListMediaField(MediaField): - """Property descriptor that retrieves a list of multiple values from - a tag. - - Uses ``get_list`` and set_list`` methods of its ``StorageStyle`` - strategies to do the actual work. - """ - def __get__(self, mediafile, _): - values = [] - for style in self.styles(mediafile.mgfile): - values.extend(style.get_list(mediafile.mgfile)) - return [_safe_cast(self.out_type, value) for value in values] - - def __set__(self, mediafile, values): - for style in self.styles(mediafile.mgfile): - style.set_list(mediafile.mgfile, values) - - def single_field(self): - """Returns a ``MediaField`` descriptor that gets and sets the - first item. - """ - options = {'out_type': self.out_type} - return MediaField(*self._styles, **options) - - -class DateField(MediaField): - """Descriptor that handles serializing and deserializing dates - - The getter parses value from tags into a ``datetime.date`` instance - and setter serializes such an instance into a string. - - For granular access to year, month, and day, use the ``*_field`` - methods to create corresponding `DateItemField`s. - """ - def __init__(self, *date_styles, **kwargs): - """``date_styles`` is a list of ``StorageStyle``s to store and - retrieve the whole date from. The ``year`` option is an - additional list of fallback styles for the year. The year is - always set on this style, but is only retrieved if the main - storage styles do not return a value. - """ - super(DateField, self).__init__(*date_styles) - year_style = kwargs.get('year', None) - if year_style: - self._year_field = MediaField(*year_style) - - def __get__(self, mediafile, owner=None): - year, month, day = self._get_date_tuple(mediafile) - if not year: - return None - try: - return datetime.date( - year, - month or 1, - day or 1 - ) - except ValueError: # Out of range values. - return None - - def __set__(self, mediafile, date): - if date is None: - self._set_date_tuple(mediafile, None, None, None) - else: - self._set_date_tuple(mediafile, date.year, date.month, date.day) - - def __delete__(self, mediafile): - super(DateField, self).__delete__(mediafile) - if hasattr(self, '_year_field'): - self._year_field.__delete__(mediafile) - - def _get_date_tuple(self, mediafile): - """Get a 3-item sequence representing the date consisting of a - year, month, and day number. Each number is either an integer or - None. - """ - # Get the underlying data and split on hyphens and slashes. - datestring = super(DateField, self).__get__(mediafile, None) - if isinstance(datestring, six.string_types): - datestring = re.sub(r'[Tt ].*$', '', six.text_type(datestring)) - items = re.split('[-/]', six.text_type(datestring)) - else: - items = [] - - # Ensure that we have exactly 3 components, possibly by - # truncating or padding. - items = items[:3] - if len(items) < 3: - items += [None] * (3 - len(items)) - - # Use year field if year is missing. - if not items[0] and hasattr(self, '_year_field'): - items[0] = self._year_field.__get__(mediafile) - - # Convert each component to an integer if possible. - items_ = [] - for item in items: - try: - items_.append(int(item)) - except (TypeError, ValueError): - items_.append(None) - return items_ - - def _set_date_tuple(self, mediafile, year, month=None, day=None): - """Set the value of the field given a year, month, and day - number. Each number can be an integer or None to indicate an - unset component. - """ - if year is None: - self.__delete__(mediafile) - return - - date = [u'{0:04d}'.format(int(year))] - if month: - date.append(u'{0:02d}'.format(int(month))) - if month and day: - date.append(u'{0:02d}'.format(int(day))) - date = map(six.text_type, date) - super(DateField, self).__set__(mediafile, u'-'.join(date)) - - if hasattr(self, '_year_field'): - self._year_field.__set__(mediafile, year) - - def year_field(self): - return DateItemField(self, 0) - - def month_field(self): - return DateItemField(self, 1) - - def day_field(self): - return DateItemField(self, 2) - - -class DateItemField(MediaField): - """Descriptor that gets and sets constituent parts of a `DateField`: - the month, day, or year. - """ - def __init__(self, date_field, item_pos): - self.date_field = date_field - self.item_pos = item_pos - - def __get__(self, mediafile, _): - return self.date_field._get_date_tuple(mediafile)[self.item_pos] - - def __set__(self, mediafile, value): - items = self.date_field._get_date_tuple(mediafile) - items[self.item_pos] = value - self.date_field._set_date_tuple(mediafile, *items) - - def __delete__(self, mediafile): - self.__set__(mediafile, None) - - -class CoverArtField(MediaField): - """A descriptor that provides access to the *raw image data* for the - cover image on a file. This is used for backwards compatibility: the - full `ImageListField` provides richer `Image` objects. - - When there are multiple images we try to pick the most likely to be a front - cover. - """ - def __init__(self): - pass - - def __get__(self, mediafile, _): - candidates = mediafile.images - if candidates: - return self.guess_cover_image(candidates).data - else: - return None - - @staticmethod - def guess_cover_image(candidates): - if len(candidates) == 1: - return candidates[0] - try: - return next(c for c in candidates if c.type == ImageType.front) - except StopIteration: - return candidates[0] - - def __set__(self, mediafile, data): - if data: - mediafile.images = [Image(data=data)] - else: - mediafile.images = [] - - def __delete__(self, mediafile): - delattr(mediafile, 'images') - - -class ImageListField(ListMediaField): - """Descriptor to access the list of images embedded in tags. - - The getter returns a list of `Image` instances obtained from - the tags. The setter accepts a list of `Image` instances to be - written to the tags. - """ - def __init__(self): - # The storage styles used here must implement the - # `ListStorageStyle` interface and get and set lists of - # `Image`s. - super(ImageListField, self).__init__( - MP3ImageStorageStyle(), - MP4ImageStorageStyle(), - ASFImageStorageStyle(), - VorbisImageStorageStyle(), - FlacImageStorageStyle(), - APEv2ImageStorageStyle(), - out_type=Image, - ) - - -# MediaFile is a collection of fields. - -class MediaFile(object): - """Represents a multimedia file on disk and provides access to its - metadata. - """ - def __init__(self, path, id3v23=False): - """Constructs a new `MediaFile` reflecting the file at path. May - throw `UnreadableFileError`. - - By default, MP3 files are saved with ID3v2.4 tags. You can use - the older ID3v2.3 standard by specifying the `id3v23` option. - """ - self.path = path - - self.mgfile = mutagen_call('open', path, mutagen.File, path) - - if self.mgfile is None: - # Mutagen couldn't guess the type - raise FileTypeError(path) - elif (type(self.mgfile).__name__ == 'M4A' or - type(self.mgfile).__name__ == 'MP4'): - info = self.mgfile.info - if info.codec and info.codec.startswith('alac'): - self.type = 'alac' - else: - self.type = 'aac' - elif (type(self.mgfile).__name__ == 'ID3' or - type(self.mgfile).__name__ == 'MP3'): - self.type = 'mp3' - elif type(self.mgfile).__name__ == 'FLAC': - self.type = 'flac' - elif type(self.mgfile).__name__ == 'OggOpus': - self.type = 'opus' - elif type(self.mgfile).__name__ == 'OggVorbis': - self.type = 'ogg' - elif type(self.mgfile).__name__ == 'MonkeysAudio': - self.type = 'ape' - elif type(self.mgfile).__name__ == 'WavPack': - self.type = 'wv' - elif type(self.mgfile).__name__ == 'Musepack': - self.type = 'mpc' - elif type(self.mgfile).__name__ == 'ASF': - self.type = 'asf' - elif type(self.mgfile).__name__ == 'AIFF': - self.type = 'aiff' - elif type(self.mgfile).__name__ == 'DSF': - self.type = 'dsf' - else: - raise FileTypeError(path, type(self.mgfile).__name__) - - # Add a set of tags if it's missing. - if self.mgfile.tags is None: - self.mgfile.add_tags() - - # Set the ID3v2.3 flag only for MP3s. - self.id3v23 = id3v23 and self.type == 'mp3' - - def save(self): - """Write the object's tags back to the file. May - throw `UnreadableFileError`. - """ - # Possibly save the tags to ID3v2.3. - kwargs = {} - if self.id3v23: - id3 = self.mgfile - if hasattr(id3, 'tags'): - # In case this is an MP3 object, not an ID3 object. - id3 = id3.tags - id3.update_to_v23() - kwargs['v2_version'] = 3 - - mutagen_call('save', self.path, self.mgfile.save, **kwargs) - - def delete(self): - """Remove the current metadata tag from the file. May - throw `UnreadableFileError`. - """ - mutagen_call('delete', self.path, self.mgfile.delete) - - # Convenient access to the set of available fields. - - @classmethod - def fields(cls): - """Get the names of all writable properties that reflect - metadata tags (i.e., those that are instances of - :class:`MediaField`). - """ - for property, descriptor in cls.__dict__.items(): - if isinstance(descriptor, MediaField): - if isinstance(property, bytes): - # On Python 2, class field names are bytes. This method - # produces text strings. - yield property.decode('utf8', 'ignore') - else: - yield property - - @classmethod - def _field_sort_name(cls, name): - """Get a sort key for a field name that determines the order - fields should be written in. - - Fields names are kept unchanged, unless they are instances of - :class:`DateItemField`, in which case `year`, `month`, and `day` - are replaced by `date0`, `date1`, and `date2`, respectively, to - make them appear in that order. - """ - if isinstance(cls.__dict__[name], DateItemField): - name = re.sub('year', 'date0', name) - name = re.sub('month', 'date1', name) - name = re.sub('day', 'date2', name) - return name - - @classmethod - def sorted_fields(cls): - """Get the names of all writable metadata fields, sorted in the - order that they should be written. - - This is a lexicographic order, except for instances of - :class:`DateItemField`, which are sorted in year-month-day - order. - """ - for property in sorted(cls.fields(), key=cls._field_sort_name): - yield property - - @classmethod - def readable_fields(cls): - """Get all metadata fields: the writable ones from - :meth:`fields` and also other audio properties. - """ - for property in cls.fields(): - yield property - for property in ('length', 'samplerate', 'bitdepth', 'bitrate', - 'channels', 'format'): - yield property - - @classmethod - def add_field(cls, name, descriptor): - """Add a field to store custom tags. - - :param name: the name of the property the field is accessed - through. It must not already exist on this class. - - :param descriptor: an instance of :class:`MediaField`. - """ - if not isinstance(descriptor, MediaField): - raise ValueError( - u'{0} must be an instance of MediaField'.format(descriptor)) - if name in cls.__dict__: - raise ValueError( - u'property "{0}" already exists on MediaField'.format(name)) - setattr(cls, name, descriptor) - - def update(self, dict): - """Set all field values from a dictionary. - - For any key in `dict` that is also a field to store tags the - method retrieves the corresponding value from `dict` and updates - the `MediaFile`. If a key has the value `None`, the - corresponding property is deleted from the `MediaFile`. - """ - for field in self.sorted_fields(): - if field in dict: - if dict[field] is None: - delattr(self, field) - else: - setattr(self, field, dict[field]) - - # Field definitions. - - title = MediaField( - MP3StorageStyle('TIT2'), - MP4StorageStyle('\xa9nam'), - StorageStyle('TITLE'), - ASFStorageStyle('Title'), - ) - artist = MediaField( - MP3StorageStyle('TPE1'), - MP4StorageStyle('\xa9ART'), - StorageStyle('ARTIST'), - ASFStorageStyle('Author'), - ) - album = MediaField( - MP3StorageStyle('TALB'), - MP4StorageStyle('\xa9alb'), - StorageStyle('ALBUM'), - ASFStorageStyle('WM/AlbumTitle'), - ) - genres = ListMediaField( - MP3ListStorageStyle('TCON'), - MP4ListStorageStyle('\xa9gen'), - ListStorageStyle('GENRE'), - ASFStorageStyle('WM/Genre'), - ) - genre = genres.single_field() - - lyricist = MediaField( - MP3StorageStyle('TEXT'), - MP4StorageStyle('----:com.apple.iTunes:LYRICIST'), - StorageStyle('LYRICIST'), - ASFStorageStyle('WM/Writer'), - ) - composer = MediaField( - MP3StorageStyle('TCOM'), - MP4StorageStyle('\xa9wrt'), - StorageStyle('COMPOSER'), - ASFStorageStyle('WM/Composer'), - ) - composer_sort = MediaField( - MP3StorageStyle('TSOC'), - MP4StorageStyle('soco'), - StorageStyle('COMPOSERSORT'), - ASFStorageStyle('WM/Composersortorder'), - ) - arranger = MediaField( - MP3PeopleStorageStyle('TIPL', involvement='arranger'), - MP4StorageStyle('----:com.apple.iTunes:Arranger'), - StorageStyle('ARRANGER'), - ASFStorageStyle('beets/Arranger'), - ) - - grouping = MediaField( - MP3StorageStyle('TIT1'), - MP4StorageStyle('\xa9grp'), - StorageStyle('GROUPING'), - ASFStorageStyle('WM/ContentGroupDescription'), - ) - track = MediaField( - MP3SlashPackStorageStyle('TRCK', pack_pos=0), - MP4TupleStorageStyle('trkn', index=0), - StorageStyle('TRACK'), - StorageStyle('TRACKNUMBER'), - ASFStorageStyle('WM/TrackNumber'), - out_type=int, - ) - tracktotal = MediaField( - MP3SlashPackStorageStyle('TRCK', pack_pos=1), - MP4TupleStorageStyle('trkn', index=1), - StorageStyle('TRACKTOTAL'), - StorageStyle('TRACKC'), - StorageStyle('TOTALTRACKS'), - ASFStorageStyle('TotalTracks'), - out_type=int, - ) - disc = MediaField( - MP3SlashPackStorageStyle('TPOS', pack_pos=0), - MP4TupleStorageStyle('disk', index=0), - StorageStyle('DISC'), - StorageStyle('DISCNUMBER'), - ASFStorageStyle('WM/PartOfSet'), - out_type=int, - ) - disctotal = MediaField( - MP3SlashPackStorageStyle('TPOS', pack_pos=1), - MP4TupleStorageStyle('disk', index=1), - StorageStyle('DISCTOTAL'), - StorageStyle('DISCC'), - StorageStyle('TOTALDISCS'), - ASFStorageStyle('TotalDiscs'), - out_type=int, - ) - lyrics = MediaField( - MP3DescStorageStyle(key='USLT'), - MP4StorageStyle('\xa9lyr'), - StorageStyle('LYRICS'), - ASFStorageStyle('WM/Lyrics'), - ) - comments = MediaField( - MP3DescStorageStyle(key='COMM'), - MP4StorageStyle('\xa9cmt'), - StorageStyle('DESCRIPTION'), - StorageStyle('COMMENT'), - ASFStorageStyle('WM/Comments'), - ASFStorageStyle('Description') - ) - bpm = MediaField( - MP3StorageStyle('TBPM'), - MP4StorageStyle('tmpo', as_type=int), - StorageStyle('BPM'), - ASFStorageStyle('WM/BeatsPerMinute'), - out_type=int, - ) - comp = MediaField( - MP3StorageStyle('TCMP'), - MP4BoolStorageStyle('cpil'), - StorageStyle('COMPILATION'), - ASFStorageStyle('WM/IsCompilation', as_type=bool), - out_type=bool, - ) - albumartist = MediaField( - MP3StorageStyle('TPE2'), - MP4StorageStyle('aART'), - StorageStyle('ALBUM ARTIST'), - StorageStyle('ALBUMARTIST'), - ASFStorageStyle('WM/AlbumArtist'), - ) - albumtype = MediaField( - MP3DescStorageStyle(u'MusicBrainz Album Type'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Type'), - StorageStyle('MUSICBRAINZ_ALBUMTYPE'), - ASFStorageStyle('MusicBrainz/Album Type'), - ) - label = MediaField( - MP3StorageStyle('TPUB'), - MP4StorageStyle('----:com.apple.iTunes:Label'), - MP4StorageStyle('----:com.apple.iTunes:publisher'), - StorageStyle('LABEL'), - StorageStyle('PUBLISHER'), # Traktor - ASFStorageStyle('WM/Publisher'), - ) - artist_sort = MediaField( - MP3StorageStyle('TSOP'), - MP4StorageStyle('soar'), - StorageStyle('ARTISTSORT'), - ASFStorageStyle('WM/ArtistSortOrder'), - ) - albumartist_sort = MediaField( - MP3DescStorageStyle(u'ALBUMARTISTSORT'), - MP4StorageStyle('soaa'), - StorageStyle('ALBUMARTISTSORT'), - ASFStorageStyle('WM/AlbumArtistSortOrder'), - ) - asin = MediaField( - MP3DescStorageStyle(u'ASIN'), - MP4StorageStyle('----:com.apple.iTunes:ASIN'), - StorageStyle('ASIN'), - ASFStorageStyle('MusicBrainz/ASIN'), - ) - catalognum = MediaField( - MP3DescStorageStyle(u'CATALOGNUMBER'), - MP4StorageStyle('----:com.apple.iTunes:CATALOGNUMBER'), - StorageStyle('CATALOGNUMBER'), - ASFStorageStyle('WM/CatalogNo'), - ) - disctitle = MediaField( - MP3StorageStyle('TSST'), - MP4StorageStyle('----:com.apple.iTunes:DISCSUBTITLE'), - StorageStyle('DISCSUBTITLE'), - ASFStorageStyle('WM/SetSubTitle'), - ) - encoder = MediaField( - MP3StorageStyle('TENC'), - MP4StorageStyle('\xa9too'), - StorageStyle('ENCODEDBY'), - StorageStyle('ENCODER'), - ASFStorageStyle('WM/EncodedBy'), - ) - script = MediaField( - MP3DescStorageStyle(u'Script'), - MP4StorageStyle('----:com.apple.iTunes:SCRIPT'), - StorageStyle('SCRIPT'), - ASFStorageStyle('WM/Script'), - ) - language = MediaField( - MP3StorageStyle('TLAN'), - MP4StorageStyle('----:com.apple.iTunes:LANGUAGE'), - StorageStyle('LANGUAGE'), - ASFStorageStyle('WM/Language'), - ) - country = MediaField( - MP3DescStorageStyle(u'MusicBrainz Album Release Country'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz ' - 'Album Release Country'), - StorageStyle('RELEASECOUNTRY'), - ASFStorageStyle('MusicBrainz/Album Release Country'), - ) - albumstatus = MediaField( - MP3DescStorageStyle(u'MusicBrainz Album Status'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Status'), - StorageStyle('MUSICBRAINZ_ALBUMSTATUS'), - ASFStorageStyle('MusicBrainz/Album Status'), - ) - media = MediaField( - MP3StorageStyle('TMED'), - MP4StorageStyle('----:com.apple.iTunes:MEDIA'), - StorageStyle('MEDIA'), - ASFStorageStyle('WM/Media'), - ) - albumdisambig = MediaField( - # This tag mapping was invented for beets (not used by Picard, etc). - MP3DescStorageStyle(u'MusicBrainz Album Comment'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Comment'), - StorageStyle('MUSICBRAINZ_ALBUMCOMMENT'), - ASFStorageStyle('MusicBrainz/Album Comment'), - ) - - # Release date. - date = DateField( - MP3StorageStyle('TDRC'), - MP4StorageStyle('\xa9day'), - StorageStyle('DATE'), - ASFStorageStyle('WM/Year'), - year=(StorageStyle('YEAR'),)) - - year = date.year_field() - month = date.month_field() - day = date.day_field() - - # *Original* release date. - original_date = DateField( - MP3StorageStyle('TDOR'), - MP4StorageStyle('----:com.apple.iTunes:ORIGINAL YEAR'), - StorageStyle('ORIGINALDATE'), - ASFStorageStyle('WM/OriginalReleaseYear')) - - original_year = original_date.year_field() - original_month = original_date.month_field() - original_day = original_date.day_field() - - # Nonstandard metadata. - artist_credit = MediaField( - MP3DescStorageStyle(u'Artist Credit'), - MP4StorageStyle('----:com.apple.iTunes:Artist Credit'), - StorageStyle('ARTIST_CREDIT'), - ASFStorageStyle('beets/Artist Credit'), - ) - albumartist_credit = MediaField( - MP3DescStorageStyle(u'Album Artist Credit'), - MP4StorageStyle('----:com.apple.iTunes:Album Artist Credit'), - StorageStyle('ALBUMARTIST_CREDIT'), - ASFStorageStyle('beets/Album Artist Credit'), - ) - - # Legacy album art field - art = CoverArtField() - - # Image list - images = ImageListField() - - # MusicBrainz IDs. - mb_trackid = MediaField( - MP3UFIDStorageStyle(owner='http://musicbrainz.org'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Track Id'), - StorageStyle('MUSICBRAINZ_TRACKID'), - ASFStorageStyle('MusicBrainz/Track Id'), - ) - mb_releasetrackid = MediaField( - MP3DescStorageStyle(u'MusicBrainz Release Track Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Release Track Id'), - StorageStyle('MUSICBRAINZ_RELEASETRACKID'), - ASFStorageStyle('MusicBrainz/Release Track Id'), - ) - mb_albumid = MediaField( - MP3DescStorageStyle(u'MusicBrainz Album Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Id'), - StorageStyle('MUSICBRAINZ_ALBUMID'), - ASFStorageStyle('MusicBrainz/Album Id'), - ) - mb_artistid = MediaField( - MP3DescStorageStyle(u'MusicBrainz Artist Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Artist Id'), - StorageStyle('MUSICBRAINZ_ARTISTID'), - ASFStorageStyle('MusicBrainz/Artist Id'), - ) - mb_albumartistid = MediaField( - MP3DescStorageStyle(u'MusicBrainz Album Artist Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Album Artist Id'), - StorageStyle('MUSICBRAINZ_ALBUMARTISTID'), - ASFStorageStyle('MusicBrainz/Album Artist Id'), - ) - mb_releasegroupid = MediaField( - MP3DescStorageStyle(u'MusicBrainz Release Group Id'), - MP4StorageStyle('----:com.apple.iTunes:MusicBrainz Release Group Id'), - StorageStyle('MUSICBRAINZ_RELEASEGROUPID'), - ASFStorageStyle('MusicBrainz/Release Group Id'), - ) - - # Acoustid fields. - acoustid_fingerprint = MediaField( - MP3DescStorageStyle(u'Acoustid Fingerprint'), - MP4StorageStyle('----:com.apple.iTunes:Acoustid Fingerprint'), - StorageStyle('ACOUSTID_FINGERPRINT'), - ASFStorageStyle('Acoustid/Fingerprint'), - ) - acoustid_id = MediaField( - MP3DescStorageStyle(u'Acoustid Id'), - MP4StorageStyle('----:com.apple.iTunes:Acoustid Id'), - StorageStyle('ACOUSTID_ID'), - ASFStorageStyle('Acoustid/Id'), - ) - - # ReplayGain fields. - rg_track_gain = MediaField( - MP3DescStorageStyle( - u'REPLAYGAIN_TRACK_GAIN', - float_places=2, suffix=u' dB' - ), - MP3DescStorageStyle( - u'replaygain_track_gain', - float_places=2, suffix=u' dB' - ), - MP3SoundCheckStorageStyle( - key='COMM', - index=0, desc=u'iTunNORM', - id3_lang='eng' - ), - MP4StorageStyle( - '----:com.apple.iTunes:replaygain_track_gain', - float_places=2, suffix=' dB' - ), - MP4SoundCheckStorageStyle( - '----:com.apple.iTunes:iTunNORM', - index=0 - ), - StorageStyle( - u'REPLAYGAIN_TRACK_GAIN', - float_places=2, suffix=u' dB' - ), - ASFStorageStyle( - u'replaygain_track_gain', - float_places=2, suffix=u' dB' - ), - out_type=float - ) - rg_album_gain = MediaField( - MP3DescStorageStyle( - u'REPLAYGAIN_ALBUM_GAIN', - float_places=2, suffix=u' dB' - ), - MP3DescStorageStyle( - u'replaygain_album_gain', - float_places=2, suffix=u' dB' - ), - MP4StorageStyle( - '----:com.apple.iTunes:replaygain_album_gain', - float_places=2, suffix=' dB' - ), - StorageStyle( - u'REPLAYGAIN_ALBUM_GAIN', - float_places=2, suffix=u' dB' - ), - ASFStorageStyle( - u'replaygain_album_gain', - float_places=2, suffix=u' dB' - ), - out_type=float - ) - rg_track_peak = MediaField( - MP3DescStorageStyle( - u'REPLAYGAIN_TRACK_PEAK', - float_places=6 - ), - MP3DescStorageStyle( - u'replaygain_track_peak', - float_places=6 - ), - MP3SoundCheckStorageStyle( - key=u'COMM', - index=1, desc=u'iTunNORM', - id3_lang='eng' - ), - MP4StorageStyle( - '----:com.apple.iTunes:replaygain_track_peak', - float_places=6 - ), - MP4SoundCheckStorageStyle( - '----:com.apple.iTunes:iTunNORM', - index=1 - ), - StorageStyle(u'REPLAYGAIN_TRACK_PEAK', float_places=6), - ASFStorageStyle(u'replaygain_track_peak', float_places=6), - out_type=float, - ) - rg_album_peak = MediaField( - MP3DescStorageStyle( - u'REPLAYGAIN_ALBUM_PEAK', - float_places=6 - ), - MP3DescStorageStyle( - u'replaygain_album_peak', - float_places=6 - ), - MP4StorageStyle( - '----:com.apple.iTunes:replaygain_album_peak', - float_places=6 - ), - StorageStyle(u'REPLAYGAIN_ALBUM_PEAK', float_places=6), - ASFStorageStyle(u'replaygain_album_peak', float_places=6), - out_type=float, - ) - - # EBU R128 fields. - r128_track_gain = MediaField( - MP3DescStorageStyle( - u'R128_TRACK_GAIN' - ), - MP4StorageStyle( - '----:com.apple.iTunes:R128_TRACK_GAIN' - ), - StorageStyle( - u'R128_TRACK_GAIN' - ), - ASFStorageStyle( - u'R128_TRACK_GAIN' - ), - out_type=int, - ) - r128_album_gain = MediaField( - MP3DescStorageStyle( - u'R128_ALBUM_GAIN' - ), - MP4StorageStyle( - '----:com.apple.iTunes:R128_ALBUM_GAIN' - ), - StorageStyle( - u'R128_ALBUM_GAIN' - ), - ASFStorageStyle( - u'R128_ALBUM_GAIN' - ), - out_type=int, - ) - - initial_key = MediaField( - MP3StorageStyle('TKEY'), - MP4StorageStyle('----:com.apple.iTunes:initialkey'), - StorageStyle('INITIALKEY'), - ASFStorageStyle('INITIALKEY'), - ) - - @property - def length(self): - """The duration of the audio in seconds (a float).""" - return self.mgfile.info.length - - @property - def samplerate(self): - """The audio's sample rate (an int).""" - if hasattr(self.mgfile.info, 'sample_rate'): - return self.mgfile.info.sample_rate - elif self.type == 'opus': - # Opus is always 48kHz internally. - return 48000 - return 0 - - @property - def bitdepth(self): - """The number of bits per sample in the audio encoding (an int). - Only available for certain file formats (zero where - unavailable). - """ - if hasattr(self.mgfile.info, 'bits_per_sample'): - return self.mgfile.info.bits_per_sample - return 0 - - @property - def channels(self): - """The number of channels in the audio (an int).""" - if hasattr(self.mgfile.info, 'channels'): - return self.mgfile.info.channels - return 0 - - @property - def bitrate(self): - """The number of bits per seconds used in the audio coding (an - int). If this is provided explicitly by the compressed file - format, this is a precise reflection of the encoding. Otherwise, - it is estimated from the on-disk file size. In this case, some - imprecision is possible because the file header is incorporated - in the file size. - """ - if hasattr(self.mgfile.info, 'bitrate') and self.mgfile.info.bitrate: - # Many formats provide it explicitly. - return self.mgfile.info.bitrate - else: - # Otherwise, we calculate bitrate from the file size. (This - # is the case for all of the lossless formats.) - if not self.length: - # Avoid division by zero if length is not available. - return 0 - size = os.path.getsize(self.path) - return int(size * 8 / self.length) - - @property - def format(self): - """A string describing the file format/codec.""" - return TYPES[self.type] +del key, value, warnings, mediafile diff --git a/setup.py b/setup.py index 5a705b982..cccffd0b6 100755 --- a/setup.py +++ b/setup.py @@ -91,6 +91,7 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', + 'mediafile>=0.0.1', ] + [ # Avoid a version of munkres incompatible with Python 3. 'munkres~=1.0.0' if sys.version_info < (3, 5, 0) else From 2a7c27352e69a5f6feb12ae751388ac229de195b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 18 May 2019 11:32:11 +1000 Subject: [PATCH 274/339] mediafile: bump MediaFile dependency to 0.1.0 This version is synchronised to the embedded copy of mediafile released with Beets v1.4.8. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cccffd0b6..50c875d12 100755 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', - 'mediafile>=0.0.1', + 'mediafile>=0.1.0', ] + [ # Avoid a version of munkres incompatible with Python 3. 'munkres~=1.0.0' if sys.version_info < (3, 5, 0) else From 8f6e5ede1a00140c654ce4196d2e7b408a68ca14 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 19 May 2019 11:30:58 +1000 Subject: [PATCH 275/339] mediafile: remove tests --- test/test_mediafile.py | 972 ------------------------------------ test/test_mediafile_edge.py | 411 --------------- 2 files changed, 1383 deletions(-) delete mode 100644 test/test_mediafile.py delete mode 100644 test/test_mediafile_edge.py diff --git a/test/test_mediafile.py b/test/test_mediafile.py deleted file mode 100644 index 36a2c53ac..000000000 --- a/test/test_mediafile.py +++ /dev/null @@ -1,972 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Automatically-generated blanket testing for the MediaFile metadata -layer. -""" -from __future__ import division, absolute_import, print_function - -import os -import shutil -import datetime -import time -import unittest -from six import assertCountEqual - -from test import _common -from beets.mediafile import MediaFile, Image, \ - ImageType, CoverArtField, UnreadableFileError - - -class ArtTestMixin(object): - """Test reads and writes of the ``art`` property. - """ - - @property - def png_data(self): - if not self._png_data: - image_file = os.path.join(_common.RSRC, b'image-2x3.png') - with open(image_file, 'rb') as f: - self._png_data = f.read() - return self._png_data - _png_data = None - - @property - def jpg_data(self): - if not self._jpg_data: - image_file = os.path.join(_common.RSRC, b'image-2x3.jpg') - with open(image_file, 'rb') as f: - self._jpg_data = f.read() - return self._jpg_data - _jpg_data = None - - @property - def tiff_data(self): - if not self._jpg_data: - image_file = os.path.join(_common.RSRC, b'image-2x3.tiff') - with open(image_file, 'rb') as f: - self._jpg_data = f.read() - return self._jpg_data - _jpg_data = None - - def test_set_png_art(self): - mediafile = self._mediafile_fixture('empty') - mediafile.art = self.png_data - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.art, self.png_data) - - def test_set_jpg_art(self): - mediafile = self._mediafile_fixture('empty') - mediafile.art = self.jpg_data - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.art, self.jpg_data) - - def test_delete_art(self): - mediafile = self._mediafile_fixture('empty') - mediafile.art = self.jpg_data - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertIsNotNone(mediafile.art) - - del mediafile.art - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertIsNone(mediafile.art) - - -class ImageStructureTestMixin(ArtTestMixin): - """Test reading and writing multiple image tags. - - The tests use the `image` media file fixture. The tags of these files - include two images, on in the PNG format, the other in JPEG format. If - the tag format supports it they also include additional metadata. - """ - - def test_read_image_structures(self): - mediafile = self._mediafile_fixture('image') - - self.assertEqual(len(mediafile.images), 2) - - image = next(i for i in mediafile.images - if i.mime_type == 'image/png') - self.assertEqual(image.data, self.png_data) - self.assertExtendedImageAttributes(image, desc=u'album cover', - type=ImageType.front) - - image = next(i for i in mediafile.images - if i.mime_type == 'image/jpeg') - self.assertEqual(image.data, self.jpg_data) - self.assertExtendedImageAttributes(image, desc=u'the artist', - type=ImageType.artist) - - def test_set_image_structure(self): - mediafile = self._mediafile_fixture('empty') - image = Image(data=self.png_data, desc=u'album cover', - type=ImageType.front) - mediafile.images = [image] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(len(mediafile.images), 1) - - image = mediafile.images[0] - self.assertEqual(image.data, self.png_data) - self.assertEqual(image.mime_type, 'image/png') - self.assertExtendedImageAttributes(image, desc=u'album cover', - type=ImageType.front) - - def test_add_image_structure(self): - mediafile = self._mediafile_fixture('image') - self.assertEqual(len(mediafile.images), 2) - - image = Image(data=self.png_data, desc=u'the composer', - type=ImageType.composer) - mediafile.images += [image] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(len(mediafile.images), 3) - - images = (i for i in mediafile.images if i.desc == u'the composer') - image = next(images, None) - self.assertExtendedImageAttributes( - image, desc=u'the composer', type=ImageType.composer - ) - - def test_delete_image_structures(self): - mediafile = self._mediafile_fixture('image') - self.assertEqual(len(mediafile.images), 2) - - del mediafile.images - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(len(mediafile.images), 0) - - def test_guess_cover(self): - mediafile = self._mediafile_fixture('image') - self.assertEqual(len(mediafile.images), 2) - cover = CoverArtField.guess_cover_image(mediafile.images) - self.assertEqual(cover.desc, u'album cover') - self.assertEqual(mediafile.art, cover.data) - - def assertExtendedImageAttributes(self, image, **kwargs): # noqa - """Ignore extended image attributes in the base tests. - """ - pass - - -class ExtendedImageStructureTestMixin(ImageStructureTestMixin): - """Checks for additional attributes in the image structure. - - Like the base `ImageStructureTestMixin`, per-format test classes - should include this mixin to add image-related tests. - """ - - def assertExtendedImageAttributes(self, image, desc=None, type=None): # noqa - self.assertEqual(image.desc, desc) - self.assertEqual(image.type, type) - - def test_add_tiff_image(self): - mediafile = self._mediafile_fixture('image') - self.assertEqual(len(mediafile.images), 2) - - image = Image(data=self.tiff_data, desc=u'the composer', - type=ImageType.composer) - mediafile.images += [image] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(len(mediafile.images), 3) - - # WMA does not preserve the order, so we have to work around this - image = list(filter(lambda i: i.mime_type == 'image/tiff', - mediafile.images))[0] - self.assertExtendedImageAttributes( - image, desc=u'the composer', type=ImageType.composer) - - -class LazySaveTestMixin(object): - """Mediafile should only write changes when tags have changed - """ - - @unittest.skip(u'not yet implemented') - def test_unmodified(self): - mediafile = self._mediafile_fixture('full') - mtime = self._set_past_mtime(mediafile.path) - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - mediafile.save() - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - @unittest.skip(u'not yet implemented') - def test_same_tag_value(self): - mediafile = self._mediafile_fixture('full') - mtime = self._set_past_mtime(mediafile.path) - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - mediafile.title = mediafile.title - mediafile.save() - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - def test_update_same_tag_value(self): - mediafile = self._mediafile_fixture('full') - mtime = self._set_past_mtime(mediafile.path) - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - mediafile.update({'title': mediafile.title}) - mediafile.save() - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - @unittest.skip(u'not yet implemented') - def test_tag_value_change(self): - mediafile = self._mediafile_fixture('full') - mtime = self._set_past_mtime(mediafile.path) - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - mediafile.title = mediafile.title - mediafile.album = u'another' - mediafile.save() - self.assertNotEqual(os.stat(mediafile.path).st_mtime, mtime) - - def test_update_changed_tag_value(self): - mediafile = self._mediafile_fixture('full') - mtime = self._set_past_mtime(mediafile.path) - self.assertEqual(os.stat(mediafile.path).st_mtime, mtime) - - mediafile.update({'title': mediafile.title, 'album': u'another'}) - mediafile.save() - self.assertNotEqual(os.stat(mediafile.path).st_mtime, mtime) - - def _set_past_mtime(self, path): - mtime = round(time.time() - 10000) - os.utime(path, (mtime, mtime)) - return mtime - - -class GenreListTestMixin(object): - """Tests access to the ``genres`` property as a list. - """ - - def test_read_genre_list(self): - mediafile = self._mediafile_fixture('full') - assertCountEqual(self, mediafile.genres, ['the genre']) - - def test_write_genre_list(self): - mediafile = self._mediafile_fixture('empty') - mediafile.genres = [u'one', u'two'] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - assertCountEqual(self, mediafile.genres, [u'one', u'two']) - - def test_write_genre_list_get_first(self): - mediafile = self._mediafile_fixture('empty') - mediafile.genres = [u'one', u'two'] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.genre, u'one') - - def test_append_genre_list(self): - mediafile = self._mediafile_fixture('full') - self.assertEqual(mediafile.genre, u'the genre') - mediafile.genres += [u'another'] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - assertCountEqual(self, mediafile.genres, [u'the genre', u'another']) - - -class ReadWriteTestBase(ArtTestMixin, GenreListTestMixin, - _common.TempDirMixin): - """Test writing and reading tags. Subclasses must set ``extension`` - and ``audio_properties``. - - The basic tests for all audio formats encompass three files provided - in our `rsrc` folder: `full.*`, `empty.*`, and `unparseable.*`. - Respectively, they should contain a full slate of common fields - listed in `full_initial_tags` below; no fields contents at all; and - an unparseable release date field. - - To add support for a new file format to MediaFile, add these three - files and then create a `ReadWriteTestBase` subclass by copying n' - pasting one of the existing subclasses below. You will want to - update the `format` field in that subclass, and you will probably - need to fiddle with the `bitrate` and other format-specific fields. - - You can also add image tests (using an additional `image.*` fixture - file) by including one of the image-related mixins. - """ - - full_initial_tags = { - 'title': u'full', - 'artist': u'the artist', - 'album': u'the album', - 'genre': u'the genre', - 'composer': u'the composer', - 'grouping': u'the grouping', - 'year': 2001, - 'month': None, - 'day': None, - 'date': datetime.date(2001, 1, 1), - 'track': 2, - 'tracktotal': 3, - 'disc': 4, - 'disctotal': 5, - 'lyrics': u'the lyrics', - 'comments': u'the comments', - 'bpm': 6, - 'comp': True, - 'mb_trackid': '8b882575-08a5-4452-a7a7-cbb8a1531f9e', - 'mb_releasetrackid': 'c29f3a57-b439-46fd-a2e2-93776b1371e0', - 'mb_albumid': '9e873859-8aa4-4790-b985-5a953e8ef628', - 'mb_artistid': '7cf0ea9d-86b9-4dad-ba9e-2355a64899ea', - 'art': None, - 'label': u'the label', - } - - tag_fields = [ - 'title', - 'artist', - 'album', - 'genre', - 'lyricist', - 'composer', - 'composer_sort', - 'arranger', - 'grouping', - 'year', - 'month', - 'day', - 'date', - 'track', - 'tracktotal', - 'disc', - 'disctotal', - 'lyrics', - 'comments', - 'bpm', - 'comp', - 'mb_trackid', - 'mb_releasetrackid', - 'mb_albumid', - 'mb_artistid', - 'art', - 'label', - 'rg_track_peak', - 'rg_track_gain', - 'rg_album_peak', - 'rg_album_gain', - 'r128_track_gain', - 'r128_album_gain', - 'albumartist', - 'mb_albumartistid', - 'artist_sort', - 'albumartist_sort', - 'acoustid_fingerprint', - 'acoustid_id', - 'mb_releasegroupid', - 'asin', - 'catalognum', - 'disctitle', - 'script', - 'language', - 'country', - 'albumstatus', - 'media', - 'albumdisambig', - 'artist_credit', - 'albumartist_credit', - 'original_year', - 'original_month', - 'original_day', - 'original_date', - 'initial_key', - ] - - def setUp(self): - self.create_temp_dir() - - def tearDown(self): - self.remove_temp_dir() - - def test_read_nonexisting(self): - mediafile = self._mediafile_fixture('full') - os.remove(mediafile.path) - self.assertRaises(UnreadableFileError, MediaFile, mediafile.path) - - def test_save_nonexisting(self): - mediafile = self._mediafile_fixture('full') - os.remove(mediafile.path) - try: - mediafile.save() - except UnreadableFileError: - pass - - def test_delete_nonexisting(self): - mediafile = self._mediafile_fixture('full') - os.remove(mediafile.path) - try: - mediafile.delete() - except UnreadableFileError: - pass - - def test_read_audio_properties(self): - mediafile = self._mediafile_fixture('full') - for key, value in self.audio_properties.items(): - if isinstance(value, float): - self.assertAlmostEqual(getattr(mediafile, key), value, - delta=0.1) - else: - self.assertEqual(getattr(mediafile, key), value) - - def test_read_full(self): - mediafile = self._mediafile_fixture('full') - self.assertTags(mediafile, self.full_initial_tags) - - def test_read_empty(self): - mediafile = self._mediafile_fixture('empty') - for field in self.tag_fields: - self.assertIsNone(getattr(mediafile, field)) - - def test_write_empty(self): - mediafile = self._mediafile_fixture('empty') - tags = self._generate_tags() - - for key, value in tags.items(): - setattr(mediafile, key, value) - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertTags(mediafile, tags) - - def test_update_empty(self): - mediafile = self._mediafile_fixture('empty') - tags = self._generate_tags() - - mediafile.update(tags) - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertTags(mediafile, tags) - - def test_overwrite_full(self): - mediafile = self._mediafile_fixture('full') - tags = self._generate_tags() - - for key, value in tags.items(): - setattr(mediafile, key, value) - mediafile.save() - - # Make sure the tags are already set when writing a second time - for key, value in tags.items(): - setattr(mediafile, key, value) - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertTags(mediafile, tags) - - def test_update_full(self): - mediafile = self._mediafile_fixture('full') - tags = self._generate_tags() - - mediafile.update(tags) - mediafile.save() - # Make sure the tags are already set when writing a second time - mediafile.update(tags) - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertTags(mediafile, tags) - - def test_write_date_components(self): - mediafile = self._mediafile_fixture('full') - mediafile.year = 2001 - mediafile.month = 1 - mediafile.day = 2 - mediafile.original_year = 1999 - mediafile.original_month = 12 - mediafile.original_day = 30 - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.year, 2001) - self.assertEqual(mediafile.month, 1) - self.assertEqual(mediafile.day, 2) - self.assertEqual(mediafile.date, datetime.date(2001, 1, 2)) - self.assertEqual(mediafile.original_year, 1999) - self.assertEqual(mediafile.original_month, 12) - self.assertEqual(mediafile.original_day, 30) - self.assertEqual(mediafile.original_date, datetime.date(1999, 12, 30)) - - def test_write_incomplete_date_components(self): - mediafile = self._mediafile_fixture('empty') - mediafile.year = 2001 - mediafile.month = None - mediafile.day = 2 - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.year, 2001) - self.assertIsNone(mediafile.month) - self.assertIsNone(mediafile.day) - self.assertEqual(mediafile.date, datetime.date(2001, 1, 1)) - - def test_write_dates(self): - mediafile = self._mediafile_fixture('full') - mediafile.date = datetime.date(2001, 1, 2) - mediafile.original_date = datetime.date(1999, 12, 30) - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.year, 2001) - self.assertEqual(mediafile.month, 1) - self.assertEqual(mediafile.day, 2) - self.assertEqual(mediafile.date, datetime.date(2001, 1, 2)) - self.assertEqual(mediafile.original_year, 1999) - self.assertEqual(mediafile.original_month, 12) - self.assertEqual(mediafile.original_day, 30) - self.assertEqual(mediafile.original_date, datetime.date(1999, 12, 30)) - - def test_write_packed(self): - mediafile = self._mediafile_fixture('empty') - - mediafile.tracktotal = 2 - mediafile.track = 1 - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.track, 1) - self.assertEqual(mediafile.tracktotal, 2) - - def test_write_counters_without_total(self): - mediafile = self._mediafile_fixture('full') - self.assertEqual(mediafile.track, 2) - self.assertEqual(mediafile.tracktotal, 3) - self.assertEqual(mediafile.disc, 4) - self.assertEqual(mediafile.disctotal, 5) - - mediafile.track = 10 - delattr(mediafile, 'tracktotal') - mediafile.disc = 10 - delattr(mediafile, 'disctotal') - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.track, 10) - self.assertEqual(mediafile.tracktotal, None) - self.assertEqual(mediafile.disc, 10) - self.assertEqual(mediafile.disctotal, None) - - def test_unparseable_date(self): - """The `unparseable.*` fixture should not crash but should return None - for all parts of the release date. - """ - mediafile = self._mediafile_fixture('unparseable') - - self.assertIsNone(mediafile.date) - self.assertIsNone(mediafile.year) - self.assertIsNone(mediafile.month) - self.assertIsNone(mediafile.day) - - def test_delete_tag(self): - mediafile = self._mediafile_fixture('full') - - keys = self.full_initial_tags.keys() - for key in set(keys) - set(['art', 'month', 'day']): - self.assertIsNotNone(getattr(mediafile, key)) - for key in keys: - delattr(mediafile, key) - mediafile.save() - mediafile = MediaFile(mediafile.path) - - for key in keys: - self.assertIsNone(getattr(mediafile, key)) - - def test_delete_packed_total(self): - mediafile = self._mediafile_fixture('full') - - delattr(mediafile, 'tracktotal') - delattr(mediafile, 'disctotal') - - mediafile.save() - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.track, self.full_initial_tags['track']) - self.assertEqual(mediafile.disc, self.full_initial_tags['disc']) - - def test_delete_partial_date(self): - mediafile = self._mediafile_fixture('empty') - - mediafile.date = datetime.date(2001, 12, 3) - mediafile.save() - mediafile = MediaFile(mediafile.path) - self.assertIsNotNone(mediafile.date) - self.assertIsNotNone(mediafile.year) - self.assertIsNotNone(mediafile.month) - self.assertIsNotNone(mediafile.day) - - delattr(mediafile, 'month') - mediafile.save() - mediafile = MediaFile(mediafile.path) - self.assertIsNotNone(mediafile.date) - self.assertIsNotNone(mediafile.year) - self.assertIsNone(mediafile.month) - self.assertIsNone(mediafile.day) - - def test_delete_year(self): - mediafile = self._mediafile_fixture('full') - - self.assertIsNotNone(mediafile.date) - self.assertIsNotNone(mediafile.year) - - delattr(mediafile, 'year') - mediafile.save() - mediafile = MediaFile(mediafile.path) - self.assertIsNone(mediafile.date) - self.assertIsNone(mediafile.year) - - def assertTags(self, mediafile, tags): # noqa - errors = [] - for key, value in tags.items(): - try: - value2 = getattr(mediafile, key) - except AttributeError: - errors.append(u'Tag %s does not exist' % key) - else: - if value2 != value: - errors.append(u'Tag %s: %r != %r' % (key, value2, value)) - if any(errors): - errors = [u'Tags did not match'] + errors - self.fail('\n '.join(errors)) - - def _mediafile_fixture(self, name): - name = name + '.' + self.extension - if not isinstance(name, bytes): - name = name.encode('utf8') - src = os.path.join(_common.RSRC, name) - target = os.path.join(self.temp_dir, name) - shutil.copy(src, target) - return MediaFile(target) - - def _generate_tags(self, base=None): - """Return dictionary of tags, mapping tag names to values. - """ - tags = {} - - for key in self.tag_fields: - if key.startswith('rg_'): - # ReplayGain is float - tags[key] = 1.0 - elif key.startswith('r128_'): - # R128 is int - tags[key] = -1 - else: - tags[key] = 'value\u2010%s' % key - - for key in ['disc', 'disctotal', 'track', 'tracktotal', 'bpm']: - tags[key] = 1 - - tags['art'] = self.jpg_data - tags['comp'] = True - - date = datetime.date(2001, 4, 3) - tags['date'] = date - tags['year'] = date.year - tags['month'] = date.month - tags['day'] = date.day - - original_date = datetime.date(1999, 5, 6) - tags['original_date'] = original_date - tags['original_year'] = original_date.year - tags['original_month'] = original_date.month - tags['original_day'] = original_date.day - - return tags - - -class PartialTestMixin(object): - tags_without_total = { - 'track': 2, - 'tracktotal': 0, - 'disc': 4, - 'disctotal': 0, - } - - def test_read_track_without_total(self): - mediafile = self._mediafile_fixture('partial') - self.assertEqual(mediafile.track, 2) - self.assertIsNone(mediafile.tracktotal) - self.assertEqual(mediafile.disc, 4) - self.assertIsNone(mediafile.disctotal) - - -class MP3Test(ReadWriteTestBase, PartialTestMixin, - ExtendedImageStructureTestMixin, - unittest.TestCase): - extension = 'mp3' - audio_properties = { - 'length': 1.0, - 'bitrate': 80000, - 'format': 'MP3', - 'samplerate': 44100, - 'bitdepth': 0, - 'channels': 1, - } - - def test_unknown_apic_type(self): - mediafile = self._mediafile_fixture('image_unknown_type') - self.assertEqual(mediafile.images[0].type, ImageType.other) - - -class MP4Test(ReadWriteTestBase, PartialTestMixin, - ImageStructureTestMixin, unittest.TestCase): - extension = 'm4a' - audio_properties = { - 'length': 1.0, - 'bitrate': 64000, - 'format': 'AAC', - 'samplerate': 44100, - 'bitdepth': 16, - 'channels': 2, - } - - def test_add_tiff_image_fails(self): - mediafile = self._mediafile_fixture('empty') - with self.assertRaises(ValueError): - mediafile.images = [Image(data=self.tiff_data)] - - def test_guess_cover(self): - # There is no metadata associated with images, we pick one at random - pass - - -class AlacTest(ReadWriteTestBase, unittest.TestCase): - extension = 'alac.m4a' - audio_properties = { - 'length': 1.0, - 'bitrate': 21830, - # 'format': 'ALAC', - 'samplerate': 44100, - 'bitdepth': 16, - 'channels': 1, - } - - -class MusepackTest(ReadWriteTestBase, unittest.TestCase): - extension = 'mpc' - audio_properties = { - 'length': 1.0, - 'bitrate': 24023, - 'format': u'Musepack', - 'samplerate': 44100, - 'bitdepth': 0, - 'channels': 2, - } - - -class WMATest(ReadWriteTestBase, ExtendedImageStructureTestMixin, - unittest.TestCase): - extension = 'wma' - audio_properties = { - 'length': 1.0, - 'bitrate': 128000, - 'format': u'Windows Media', - 'samplerate': 44100, - 'bitdepth': 0, - 'channels': 1, - } - - def test_write_genre_list_get_first(self): - # WMA does not preserve list order - mediafile = self._mediafile_fixture('empty') - mediafile.genres = [u'one', u'two'] - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertIn(mediafile.genre, [u'one', u'two']) - - def test_read_pure_tags(self): - mediafile = self._mediafile_fixture('pure') - self.assertEqual(mediafile.comments, u'the comments') - self.assertEqual(mediafile.title, u'the title') - self.assertEqual(mediafile.artist, u'the artist') - - -class OggTest(ReadWriteTestBase, ExtendedImageStructureTestMixin, - unittest.TestCase): - extension = 'ogg' - audio_properties = { - 'length': 1.0, - 'bitrate': 48000, - 'format': u'OGG', - 'samplerate': 44100, - 'bitdepth': 0, - 'channels': 1, - } - - def test_read_date_from_year_tag(self): - mediafile = self._mediafile_fixture('year') - self.assertEqual(mediafile.year, 2000) - self.assertEqual(mediafile.date, datetime.date(2000, 1, 1)) - - def test_write_date_to_year_tag(self): - mediafile = self._mediafile_fixture('empty') - mediafile.year = 2000 - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertEqual(mediafile.mgfile['YEAR'], [u'2000']) - - def test_legacy_coverart_tag(self): - mediafile = self._mediafile_fixture('coverart') - self.assertTrue('coverart' in mediafile.mgfile) - self.assertEqual(mediafile.art, self.png_data) - - mediafile.art = self.png_data - mediafile.save() - - mediafile = MediaFile(mediafile.path) - self.assertFalse('coverart' in mediafile.mgfile) - - def test_date_tag_with_slashes(self): - mediafile = self._mediafile_fixture('date_with_slashes') - self.assertEqual(mediafile.year, 2005) - self.assertEqual(mediafile.month, 6) - self.assertEqual(mediafile.day, 5) - - -class FlacTest(ReadWriteTestBase, PartialTestMixin, - ExtendedImageStructureTestMixin, - unittest.TestCase): - extension = 'flac' - audio_properties = { - 'length': 1.0, - 'bitrate': 108688, - 'format': u'FLAC', - 'samplerate': 44100, - 'bitdepth': 16, - 'channels': 1, - } - - -class ApeTest(ReadWriteTestBase, ExtendedImageStructureTestMixin, - unittest.TestCase): - extension = 'ape' - audio_properties = { - 'length': 1.0, - 'bitrate': 112608, - 'format': u'APE', - 'samplerate': 44100, - 'bitdepth': 16, - 'channels': 1, - } - - -class WavpackTest(ReadWriteTestBase, unittest.TestCase): - extension = 'wv' - audio_properties = { - 'length': 1.0, - 'bitrate': 109312, - 'format': u'WavPack', - 'samplerate': 44100, - 'bitdepth': 0, - 'channels': 1, - } - - -class OpusTest(ReadWriteTestBase, unittest.TestCase): - extension = 'opus' - audio_properties = { - 'length': 1.0, - 'bitrate': 66792, - 'format': u'Opus', - 'samplerate': 48000, - 'bitdepth': 0, - 'channels': 1, - } - - -class AIFFTest(ReadWriteTestBase, unittest.TestCase): - extension = 'aiff' - audio_properties = { - 'length': 1.0, - 'bitrate': 705600, - 'format': u'AIFF', - 'samplerate': 44100, - 'bitdepth': 0, - 'channels': 1, - } - - -# Check whether we have a Mutagen version with DSF support. We can -# remove this once we require a version that includes the feature. -try: - import mutagen.dsf # noqa -except ImportError: - HAVE_DSF = False -else: - HAVE_DSF = True - - -@unittest.skipIf(not HAVE_DSF, "Mutagen does not have DSF support") -class DSFTest(ReadWriteTestBase, unittest.TestCase): - extension = 'dsf' - audio_properties = { - 'length': 0.01, - 'bitrate': 11289600, - 'format': u'DSD Stream File', - 'samplerate': 5644800, - 'bitdepth': 1, - 'channels': 2, - } - - -class MediaFieldTest(unittest.TestCase): - - def test_properties_from_fields(self): - path = os.path.join(_common.RSRC, b'full.mp3') - mediafile = MediaFile(path) - for field in MediaFile.fields(): - self.assertTrue(hasattr(mediafile, field)) - - def test_properties_from_readable_fields(self): - path = os.path.join(_common.RSRC, b'full.mp3') - mediafile = MediaFile(path) - for field in MediaFile.readable_fields(): - self.assertTrue(hasattr(mediafile, field)) - - def test_known_fields(self): - fields = list(ReadWriteTestBase.tag_fields) - fields.extend(('encoder', 'images', 'genres', 'albumtype')) - assertCountEqual(self, MediaFile.fields(), fields) - - def test_fields_in_readable_fields(self): - readable = MediaFile.readable_fields() - for field in MediaFile.fields(): - self.assertIn(field, readable) - - -def suite(): - return unittest.TestLoader().loadTestsFromName(__name__) - -if __name__ == '__main__': - unittest.main(defaultTest='suite') diff --git a/test/test_mediafile_edge.py b/test/test_mediafile_edge.py deleted file mode 100644 index 8bf9e1916..000000000 --- a/test/test_mediafile_edge.py +++ /dev/null @@ -1,411 +0,0 @@ -# -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2016, Adrian Sampson. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. - -"""Specific, edge-case tests for the MediaFile metadata layer. -""" -from __future__ import division, absolute_import, print_function - -import os -import shutil -import unittest -import mutagen.id3 - -from test import _common - -from beets import mediafile -import six - - -_sc = mediafile._safe_cast - - -class EdgeTest(unittest.TestCase): - def test_emptylist(self): - # Some files have an ID3 frame that has a list with no elements. - # This is very hard to produce, so this is just the first 8192 - # bytes of a file found "in the wild". - emptylist = mediafile.MediaFile( - os.path.join(_common.RSRC, b'emptylist.mp3') - ) - genre = emptylist.genre - self.assertEqual(genre, None) - - def test_release_time_with_space(self): - # Ensures that release times delimited by spaces are ignored. - # Amie Street produces such files. - space_time = mediafile.MediaFile( - os.path.join(_common.RSRC, b'space_time.mp3') - ) - self.assertEqual(space_time.year, 2009) - self.assertEqual(space_time.month, 9) - self.assertEqual(space_time.day, 4) - - def test_release_time_with_t(self): - # Ensures that release times delimited by Ts are ignored. - # The iTunes Store produces such files. - t_time = mediafile.MediaFile( - os.path.join(_common.RSRC, b't_time.m4a') - ) - self.assertEqual(t_time.year, 1987) - self.assertEqual(t_time.month, 3) - self.assertEqual(t_time.day, 31) - - def test_tempo_with_bpm(self): - # Some files have a string like "128 BPM" in the tempo field - # rather than just a number. - f = mediafile.MediaFile(os.path.join(_common.RSRC, b'bpm.mp3')) - self.assertEqual(f.bpm, 128) - - def test_discc_alternate_field(self): - # Different taggers use different vorbis comments to reflect - # the disc and disc count fields: ensure that the alternative - # style works. - f = mediafile.MediaFile(os.path.join(_common.RSRC, b'discc.ogg')) - self.assertEqual(f.disc, 4) - self.assertEqual(f.disctotal, 5) - - def test_old_ape_version_bitrate(self): - media_file = os.path.join(_common.RSRC, b'oldape.ape') - f = mediafile.MediaFile(media_file) - self.assertEqual(f.bitrate, 0) - - def test_only_magic_bytes_jpeg(self): - # Some jpeg files can only be recognized by their magic bytes and as - # such aren't recognized by imghdr. Ensure that this still works thanks - # to our own follow up mimetype detection based on - # https://github.com/file/file/blob/master/magic/Magdir/jpeg#L12 - magic_bytes_file = os.path.join(_common.RSRC, b'only-magic-bytes.jpg') - with open(magic_bytes_file, 'rb') as f: - jpg_data = f.read() - self.assertEqual( - mediafile._imghdr_what_wrapper(jpg_data), 'jpeg') - - def test_soundcheck_non_ascii(self): - # Make sure we don't crash when the iTunes SoundCheck field contains - # non-ASCII binary data. - f = mediafile.MediaFile(os.path.join(_common.RSRC, - b'soundcheck-nonascii.m4a')) - self.assertEqual(f.rg_track_gain, 0.0) - - -class InvalidValueToleranceTest(unittest.TestCase): - - def test_safe_cast_string_to_int(self): - self.assertEqual(_sc(int, u'something'), 0) - - def test_safe_cast_string_to_int_with_no_numbers(self): - self.assertEqual(_sc(int, u'-'), 0) - - def test_safe_cast_int_string_to_int(self): - self.assertEqual(_sc(int, u'20'), 20) - - def test_safe_cast_string_to_bool(self): - self.assertEqual(_sc(bool, u'whatever'), False) - - def test_safe_cast_intstring_to_bool(self): - self.assertEqual(_sc(bool, u'5'), True) - - def test_safe_cast_string_to_float(self): - self.assertAlmostEqual(_sc(float, u'1.234'), 1.234) - - def test_safe_cast_int_to_float(self): - self.assertAlmostEqual(_sc(float, 2), 2.0) - - def test_safe_cast_string_with_cruft_to_float(self): - self.assertAlmostEqual(_sc(float, u'1.234stuff'), 1.234) - - def test_safe_cast_negative_string_to_float(self): - self.assertAlmostEqual(_sc(float, u'-1.234'), -1.234) - - def test_safe_cast_special_chars_to_unicode(self): - us = _sc(six.text_type, 'caf\xc3\xa9') - self.assertTrue(isinstance(us, six.text_type)) - self.assertTrue(us.startswith(u'caf')) - - def test_safe_cast_float_with_no_numbers(self): - v = _sc(float, u'+') - self.assertEqual(v, 0.0) - - def test_safe_cast_float_with_dot_only(self): - v = _sc(float, u'.') - self.assertEqual(v, 0.0) - - def test_safe_cast_float_with_multiple_dots(self): - v = _sc(float, u'1.0.0') - self.assertEqual(v, 1.0) - - -class SafetyTest(unittest.TestCase, _common.TempDirMixin): - def setUp(self): - self.create_temp_dir() - - def tearDown(self): - self.remove_temp_dir() - - def _exccheck(self, fn, exc, data=''): - fn = os.path.join(self.temp_dir, fn) - with open(fn, 'w') as f: - f.write(data) - try: - self.assertRaises(exc, mediafile.MediaFile, fn) - finally: - os.unlink(fn) # delete the temporary file - - def test_corrupt_mp3_raises_unreadablefileerror(self): - # Make sure we catch Mutagen reading errors appropriately. - self._exccheck(b'corrupt.mp3', mediafile.UnreadableFileError) - - def test_corrupt_mp4_raises_unreadablefileerror(self): - self._exccheck(b'corrupt.m4a', mediafile.UnreadableFileError) - - def test_corrupt_flac_raises_unreadablefileerror(self): - self._exccheck(b'corrupt.flac', mediafile.UnreadableFileError) - - def test_corrupt_ogg_raises_unreadablefileerror(self): - self._exccheck(b'corrupt.ogg', mediafile.UnreadableFileError) - - def test_invalid_ogg_header_raises_unreadablefileerror(self): - self._exccheck(b'corrupt.ogg', mediafile.UnreadableFileError, - 'OggS\x01vorbis') - - def test_corrupt_monkeys_raises_unreadablefileerror(self): - self._exccheck(b'corrupt.ape', mediafile.UnreadableFileError) - - def test_invalid_extension_raises_filetypeerror(self): - self._exccheck(b'something.unknown', mediafile.FileTypeError) - - def test_magic_xml_raises_unreadablefileerror(self): - self._exccheck(b'nothing.xml', mediafile.UnreadableFileError, - "ftyp") - - @unittest.skipUnless(_common.HAVE_SYMLINK, u'platform lacks symlink') - def test_broken_symlink(self): - fn = os.path.join(_common.RSRC, b'brokenlink') - os.symlink('does_not_exist', fn) - try: - self.assertRaises(mediafile.UnreadableFileError, - mediafile.MediaFile, fn) - finally: - os.unlink(fn) - - -class SideEffectsTest(unittest.TestCase): - def setUp(self): - self.empty = os.path.join(_common.RSRC, b'empty.mp3') - - def test_opening_tagless_file_leaves_untouched(self): - old_mtime = os.stat(self.empty).st_mtime - mediafile.MediaFile(self.empty) - new_mtime = os.stat(self.empty).st_mtime - self.assertEqual(old_mtime, new_mtime) - - -class MP4EncodingTest(unittest.TestCase, _common.TempDirMixin): - def setUp(self): - self.create_temp_dir() - src = os.path.join(_common.RSRC, b'full.m4a') - self.path = os.path.join(self.temp_dir, b'test.m4a') - shutil.copy(src, self.path) - - self.mf = mediafile.MediaFile(self.path) - - def tearDown(self): - self.remove_temp_dir() - - def test_unicode_label_in_m4a(self): - self.mf.label = u'foo\xe8bar' - self.mf.save() - new_mf = mediafile.MediaFile(self.path) - self.assertEqual(new_mf.label, u'foo\xe8bar') - - -class MP3EncodingTest(unittest.TestCase, _common.TempDirMixin): - def setUp(self): - self.create_temp_dir() - src = os.path.join(_common.RSRC, b'full.mp3') - self.path = os.path.join(self.temp_dir, b'test.mp3') - shutil.copy(src, self.path) - - self.mf = mediafile.MediaFile(self.path) - - def test_comment_with_latin1_encoding(self): - # Set up the test file with a Latin1-encoded COMM frame. The encoding - # indices defined by MP3 are listed here: - # http://id3.org/id3v2.4.0-structure - self.mf.mgfile['COMM::eng'].encoding = 0 - - # Try to store non-Latin1 text. - self.mf.comments = u'\u2028' - self.mf.save() - - -class ZeroLengthMediaFile(mediafile.MediaFile): - @property - def length(self): - return 0.0 - - -class MissingAudioDataTest(unittest.TestCase): - def setUp(self): - super(MissingAudioDataTest, self).setUp() - path = os.path.join(_common.RSRC, b'full.mp3') - self.mf = ZeroLengthMediaFile(path) - - def test_bitrate_with_zero_length(self): - del self.mf.mgfile.info.bitrate # Not available directly. - self.assertEqual(self.mf.bitrate, 0) - - -class TypeTest(unittest.TestCase): - def setUp(self): - super(TypeTest, self).setUp() - path = os.path.join(_common.RSRC, b'full.mp3') - self.mf = mediafile.MediaFile(path) - - def test_year_integer_in_string(self): - self.mf.year = u'2009' - self.assertEqual(self.mf.year, 2009) - - def test_set_replaygain_gain_to_none(self): - self.mf.rg_track_gain = None - self.assertEqual(self.mf.rg_track_gain, 0.0) - - def test_set_replaygain_peak_to_none(self): - self.mf.rg_track_peak = None - self.assertEqual(self.mf.rg_track_peak, 0.0) - - def test_set_year_to_none(self): - self.mf.year = None - self.assertIsNone(self.mf.year) - - def test_set_track_to_none(self): - self.mf.track = None - self.assertEqual(self.mf.track, 0) - - def test_set_date_to_none(self): - self.mf.date = None - self.assertIsNone(self.mf.date) - self.assertIsNone(self.mf.year) - self.assertIsNone(self.mf.month) - self.assertIsNone(self.mf.day) - - -class SoundCheckTest(unittest.TestCase): - def test_round_trip(self): - data = mediafile._sc_encode(1.0, 1.0) - gain, peak = mediafile._sc_decode(data) - self.assertEqual(gain, 1.0) - self.assertEqual(peak, 1.0) - - def test_decode_zero(self): - data = b' 80000000 80000000 00000000 00000000 00000000 00000000 ' \ - b'00000000 00000000 00000000 00000000' - gain, peak = mediafile._sc_decode(data) - self.assertEqual(gain, 0.0) - self.assertEqual(peak, 0.0) - - def test_malformatted(self): - gain, peak = mediafile._sc_decode(b'foo') - self.assertEqual(gain, 0.0) - self.assertEqual(peak, 0.0) - - def test_special_characters(self): - gain, peak = mediafile._sc_decode(u'caf\xe9'.encode('utf-8')) - self.assertEqual(gain, 0.0) - self.assertEqual(peak, 0.0) - - def test_decode_handles_unicode(self): - # Most of the time, we expect to decode the raw bytes. But some formats - # might give us text strings, which we need to handle. - gain, peak = mediafile._sc_decode(u'caf\xe9') - self.assertEqual(gain, 0.0) - self.assertEqual(peak, 0.0) - - -class ID3v23Test(unittest.TestCase, _common.TempDirMixin): - def _make_test(self, ext=b'mp3', id3v23=False): - self.create_temp_dir() - src = os.path.join(_common.RSRC, - b'full.' + ext) - self.path = os.path.join(self.temp_dir, - b'test.' + ext) - shutil.copy(src, self.path) - return mediafile.MediaFile(self.path, id3v23=id3v23) - - def _delete_test(self): - self.remove_temp_dir() - - def test_v24_year_tag(self): - mf = self._make_test(id3v23=False) - try: - mf.year = 2013 - mf.save() - frame = mf.mgfile['TDRC'] - self.assertTrue('2013' in six.text_type(frame)) - self.assertTrue('TYER' not in mf.mgfile) - finally: - self._delete_test() - - def test_v23_year_tag(self): - mf = self._make_test(id3v23=True) - try: - mf.year = 2013 - mf.save() - frame = mf.mgfile['TYER'] - self.assertTrue('2013' in six.text_type(frame)) - self.assertTrue('TDRC' not in mf.mgfile) - finally: - self._delete_test() - - def test_v23_on_non_mp3_is_noop(self): - mf = self._make_test(b'm4a', id3v23=True) - try: - mf.year = 2013 - mf.save() - finally: - self._delete_test() - - def test_image_encoding(self): - """For compatibility with OS X/iTunes. - - See https://github.com/beetbox/beets/issues/899#issuecomment-62437773 - """ - - for v23 in [True, False]: - mf = self._make_test(id3v23=v23) - try: - mf.images = [ - mediafile.Image(b'data', desc=u""), - mediafile.Image(b'data', desc=u"foo"), - mediafile.Image(b'data', desc=u"\u0185"), - ] - mf.save() - apic_frames = mf.mgfile.tags.getall('APIC') - encodings = dict([(f.desc, f.encoding) for f in apic_frames]) - self.assertEqual(encodings, { - u"": mutagen.id3.Encoding.LATIN1, - u"foo": mutagen.id3.Encoding.LATIN1, - u"\u0185": mutagen.id3.Encoding.UTF16, - }) - finally: - self._delete_test() - - -def suite(): - return unittest.TestLoader().loadTestsFromName(__name__) - - -if __name__ == '__main__': - unittest.main(defaultTest='suite') From 546bf3af7ed0c9bdd306353c4add282b814b081d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 19 May 2019 11:32:47 +1000 Subject: [PATCH 276/339] mediafile: import from standalone module --- beets/art.py | 2 +- beets/importer.py | 2 +- beets/library.py | 2 +- beets/plugins.py | 2 +- beetsplug/bpd/__init__.py | 2 +- beetsplug/export.py | 2 +- beetsplug/fetchart.py | 2 +- beetsplug/info.py | 2 +- beetsplug/scrub.py | 2 +- beetsplug/zero.py | 2 +- test/helper.py | 2 +- test/test_convert.py | 2 +- test/test_embedart.py | 2 +- test/test_filefilter.py | 2 +- test/test_importer.py | 2 +- test/test_info.py | 2 +- test/test_library.py | 5 ++--- test/test_plugin_mediafield.py | 2 +- test/test_plugins.py | 2 +- test/test_replaygain.py | 2 +- test/test_ui.py | 2 +- test/test_zero.py | 2 +- 22 files changed, 23 insertions(+), 24 deletions(-) diff --git a/beets/art.py b/beets/art.py index b65838126..e7a087a05 100644 --- a/beets/art.py +++ b/beets/art.py @@ -26,7 +26,7 @@ import os from beets.util import displayable_path, syspath, bytestring_path from beets.util.artresizer import ArtResizer -from beets import mediafile +import mediafile def mediafile_image(image_path, maxwidth=None): diff --git a/beets/importer.py b/beets/importer.py index 889f1297e..d2943b511 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -40,7 +40,7 @@ from beets import config from beets.util import pipeline, sorted_walk, ancestry, MoveOperation from beets.util import syspath, normpath, displayable_path from enum import Enum -from beets import mediafile +import mediafile action = Enum('action', ['SKIP', 'ASIS', 'TRACKS', 'APPLY', 'ALBUMS', 'RETAG']) diff --git a/beets/library.py b/beets/library.py index 9a9d95256..c421500fa 100644 --- a/beets/library.py +++ b/beets/library.py @@ -26,7 +26,7 @@ import six import string from beets import logging -from beets.mediafile import MediaFile, UnreadableFileError +from mediafile import MediaFile, UnreadableFileError from beets import plugins from beets import util from beets.util import bytestring_path, syspath, normpath, samefile, \ diff --git a/beets/plugins.py b/beets/plugins.py index 5ca9ae3bb..edc6d8fe8 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -26,7 +26,7 @@ from functools import wraps import beets from beets import logging -from beets import mediafile +import mediafile import six PLUGIN_NAMESPACE = 'beetsplug' diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 5c54c9eab..a4a987a55 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -36,7 +36,7 @@ from beets import vfs from beets.util import bluelet from beets.library import Item from beets import dbcore -from beets.mediafile import MediaFile +from mediafile import MediaFile import six PROTOCOL_VERSION = '0.14.0' diff --git a/beetsplug/export.py b/beetsplug/export.py index 641b9fefc..d783f5b93 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -24,7 +24,7 @@ import codecs from datetime import datetime, date from beets.plugins import BeetsPlugin from beets import ui -from beets import mediafile +import mediafile from beetsplug.info import make_key_filter, library_data, tag_data diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index bfda94670..2583e4c16 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -29,7 +29,7 @@ from beets import importer from beets import ui from beets import util from beets import config -from beets.mediafile import image_mime_type +from mediafile import image_mime_type from beets.util.artresizer import ArtResizer from beets.util import confit, sorted_walk from beets.util import syspath, bytestring_path, py3_path diff --git a/beetsplug/info.py b/beetsplug/info.py index 0f7ccb875..b8a0c9375 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -23,7 +23,7 @@ import re from beets.plugins import BeetsPlugin from beets import ui -from beets import mediafile +import mediafile from beets.library import Item from beets.util import displayable_path, normpath, syspath diff --git a/beetsplug/scrub.py b/beetsplug/scrub.py index be6e7fd1f..a905899da 100644 --- a/beetsplug/scrub.py +++ b/beetsplug/scrub.py @@ -23,7 +23,7 @@ from beets.plugins import BeetsPlugin from beets import ui from beets import util from beets import config -from beets import mediafile +import mediafile import mutagen _MUTAGEN_FORMATS = { diff --git a/beetsplug/zero.py b/beetsplug/zero.py index 022c2c721..ebe1b486e 100644 --- a/beetsplug/zero.py +++ b/beetsplug/zero.py @@ -21,7 +21,7 @@ import six import re from beets.plugins import BeetsPlugin -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets.importer import action from beets.ui import Subcommand, decargs, input_yn from beets.util import confit diff --git a/test/helper.py b/test/helper.py index 392d01a55..c6afb063e 100644 --- a/test/helper.py +++ b/test/helper.py @@ -50,7 +50,7 @@ import beets.plugins from beets.library import Library, Item, Album from beets import importer from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.mediafile import MediaFile, Image +from mediafile import MediaFile, Image from beets import util from beets.util import MoveOperation diff --git a/test/test_convert.py b/test/test_convert.py index aa0cd0a34..33bdb3b24 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -24,7 +24,7 @@ from test import _common from test import helper from test.helper import control_stdin, capture_log -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets import util diff --git a/test/test_embedart.py b/test/test_embedart.py index 049e4694d..c465a5a96 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -24,7 +24,7 @@ import unittest from test import _common from test.helper import TestHelper -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets import config, logging, ui from beets.util import syspath, displayable_path from beets.util.artresizer import ArtResizer diff --git a/test/test_filefilter.py b/test/test_filefilter.py index 57310fd2a..0b0da0d3f 100644 --- a/test/test_filefilter.py +++ b/test/test_filefilter.py @@ -26,7 +26,7 @@ from test import _common from test.helper import capture_log from test.test_importer import ImportHelper from beets import config -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets.util import displayable_path, bytestring_path from beetsplug.filefilter import FileFilterPlugin diff --git a/test/test_importer.py b/test/test_importer.py index 8d6ba425a..980a769f6 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -35,7 +35,7 @@ from beets.util import displayable_path, bytestring_path, py3_path from test.helper import TestImportSession, TestHelper, has_program, capture_log from beets import importer from beets.importer import albums_in_dir -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets import autotag from beets.autotag import AlbumInfo, TrackInfo, AlbumMatch from beets import config diff --git a/test/test_info.py b/test/test_info.py index f5375e2f6..814205527 100644 --- a/test/test_info.py +++ b/test/test_info.py @@ -18,7 +18,7 @@ from __future__ import division, absolute_import, print_function import unittest from test.helper import TestHelper -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets.util import displayable_path diff --git a/test/test_library.py b/test/test_library.py index e8cdb0530..4e3be878c 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -30,12 +30,11 @@ import unittest from test import _common from test._common import item import beets.library -import beets.mediafile import beets.dbcore.query from beets import util from beets import plugins from beets import config -from beets.mediafile import MediaFile +from mediafile import MediaFile, UnreadableFileError from beets.util import syspath, bytestring_path from test.helper import TestHelper import six @@ -1169,7 +1168,7 @@ class ItemReadTest(unittest.TestCase): with self.assertRaises(beets.library.ReadError) as cm: item.read(unreadable) self.assertIsInstance(cm.exception.reason, - beets.mediafile.UnreadableFileError) + UnreadableFileError) def test_nonexistent_raise_read_error(self): item = beets.library.Item() diff --git a/test/test_plugin_mediafield.py b/test/test_plugin_mediafield.py index 983f6e2c8..a08db4542 100644 --- a/test/test_plugin_mediafield.py +++ b/test/test_plugin_mediafield.py @@ -24,7 +24,7 @@ import unittest from test import _common from beets.library import Item -from beets import mediafile +import mediafile from beets.plugins import BeetsPlugin from beets.util import bytestring_path diff --git a/test/test_plugins.py b/test/test_plugins.py index b14158699..884aa7875 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -26,7 +26,7 @@ from beets.importer import SingletonImportTask, SentinelImportTask, \ from beets import plugins, config, ui from beets.library import Item from beets.dbcore import types -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets.util import displayable_path, bytestring_path, syspath from test.test_importer import ImportHelper, AutotagStub diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 81da7139e..9f14374cc 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -23,7 +23,7 @@ from mock import patch from test.helper import TestHelper, capture_log, has_program from beets import config -from beets.mediafile import MediaFile +from mediafile import MediaFile from beetsplug.replaygain import (FatalGstreamerPluginReplayGainError, GStreamerBackend) diff --git a/test/test_ui.py b/test/test_ui.py index 8267c9be8..9aae9584b 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -35,7 +35,7 @@ from beets import ui from beets.ui import commands from beets import autotag from beets.autotag.match import distance -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets import config from beets import plugins from beets.util.confit import ConfigError diff --git a/test/test_zero.py b/test/test_zero.py index 025eaa540..1b7cc92d7 100644 --- a/test/test_zero.py +++ b/test/test_zero.py @@ -9,7 +9,7 @@ from test.helper import TestHelper, control_stdin from beets.library import Item from beetsplug.zero import ZeroPlugin -from beets.mediafile import MediaFile +from mediafile import MediaFile from beets.util import syspath From 1289efeeee543e65733e8aabc14e480b41945fba Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 19 May 2019 12:07:06 +1000 Subject: [PATCH 277/339] mediafile: update docs for MediaFile split --- docs/dev/index.rst | 6 +++++- docs/dev/media_file.rst | 21 --------------------- docs/dev/plugins.rst | 9 ++++----- 3 files changed, 9 insertions(+), 27 deletions(-) delete mode 100644 docs/dev/media_file.rst diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 82651a781..45640254c 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -4,8 +4,12 @@ For Developers This section contains information for developers. Read on if you're interested in hacking beets itself or creating plugins for it. +See also the documentation for `MediaFile`_, the library used by beets to read +and write metadata tags in media files. + +.. _MediaFile: http://mediafile.readthedocs.io/ + .. toctree:: plugins api - media_file diff --git a/docs/dev/media_file.rst b/docs/dev/media_file.rst deleted file mode 100644 index c703377d8..000000000 --- a/docs/dev/media_file.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _mediafile: - -MediaFile ---------- - -.. currentmodule:: beets.mediafile - -.. autoclass:: MediaFile - - .. automethod:: __init__ - .. automethod:: fields - .. automethod:: readable_fields - .. automethod:: save - .. automethod:: update - -.. autoclass:: MediaField - - .. automethod:: __init__ - -.. autoclass:: StorageStyle - :members: diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 745c8340a..7cec77d62 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -370,17 +370,16 @@ template fields by adding a function accepting an ``Album`` argument to the Extend MediaFile ^^^^^^^^^^^^^^^^ -:ref:`MediaFile` is the file tag abstraction layer that beets uses to make +`MediaFile`_ is the file tag abstraction layer that beets uses to make cross-format metadata manipulation simple. Plugins can add fields to MediaFile to extend the kinds of metadata that they can easily manage. The ``MediaFile`` class uses ``MediaField`` descriptors to provide -access to file tags. Have a look at the ``beets.mediafile`` source code -to learn how to use this descriptor class. If you have created a -descriptor you can add it through your plugins ``add_media_field()`` -method. +access to file tags. If you have created a descriptor you can add it through +your plugins ``add_media_field()`` method. .. automethod:: beets.plugins.BeetsPlugin.add_media_field +.. _MediaFile: http://mediafile.readthedocs.io/ Here's an example plugin that provides a meaningless new field "foo":: From 48568c0e70ffc1699b90ecae9024fd76c4327228 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 19 May 2019 12:07:31 +1000 Subject: [PATCH 278/339] Changelog for #3237 --- docs/changelog.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 21b0ae0bc..6f0c2e835 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,22 @@ New features: .. _NO_COLOR: https://no-color.org +For plugin developers: + +* `MediaFile`_ has been split into a standalone project. Where you used to do + ``from beets import mediafile``, now just do ``import mediafile``. Beets + re-exports MediaFile at the old location for backwards-compatibility, but a + deprecation warning is raised if you do this since we might drop this wrapper + in a future release. + +For packagers: + +* Beets' library for manipulating media file metadata has now been split to a + standalone project called `MediaFile`_, released as :pypi:`mediafile`. Beets + now depends on this new package. + +.. _MediaFile: https://github.com/beetbox/mediafile + 1.4.8 (May 16, 2019) -------------------- From 1de894ab836f3ff849f2146e759455d06df96f06 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 19 May 2019 12:24:20 +1000 Subject: [PATCH 279/339] Remove direct dependency on Mutagen --- docs/changelog.rst | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f0c2e835..2e3debf45 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,7 +35,9 @@ For packagers: * Beets' library for manipulating media file metadata has now been split to a standalone project called `MediaFile`_, released as :pypi:`mediafile`. Beets - now depends on this new package. + now depends on this new package. Beets now depends on Mutagen transitively + through MediaFile rather than directly, except in the case of one of beets' + plugins (scrub). .. _MediaFile: https://github.com/beetbox/mediafile diff --git a/setup.py b/setup.py index 50c875d12..4a8b85f2d 100755 --- a/setup.py +++ b/setup.py @@ -87,7 +87,6 @@ setup( install_requires=[ 'six>=1.9', - 'mutagen>=1.33', 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', @@ -143,6 +142,7 @@ setup( (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], + 'scrub': ['mutagen>=1.33'], 'bpd': ['PyGObject'], 'replaygain': ['PyGObject'], }, From ece57265674c65663f48855e7391d6ef4c1e171d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 30 May 2019 12:47:18 +1000 Subject: [PATCH 280/339] Make pathlib dependency in tests conditional We already do this in the optional dependency section so this shouldn't cause any problems. It's only used by the `thumbnails` plugin. --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5a705b982..79278f8be 100755 --- a/setup.py +++ b/setup.py @@ -115,10 +115,12 @@ setup( 'rarfile', 'responses', 'pyxdg', - 'pathlib', 'python-mpd2', 'discogs-client' - ], + ] + ( + # Tests for the thumbnails plugin need pathlib on Python 2 too. + ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] + ), # Plugin (optional) dependencies: extras_require={ From 66e63598122f15c223745a8caca06a8fb625dac0 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 30 May 2019 12:59:16 +1000 Subject: [PATCH 281/339] Make test_random work with setup.py test Without this change `python setup.py test` fails to run, since this test was missing a `suite`. --- test/test_random.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_random.py b/test/test_random.py index 4c31acdd9..4f243efd7 100644 --- a/test/test_random.py +++ b/test/test_random.py @@ -80,3 +80,10 @@ class RandomTest(unittest.TestCase, TestHelper): self.assertAlmostEqual(0, median1, delta=1) self.assertAlmostEqual(len(self.items) // 2, median2, delta=1) self.assertGreater(stdev2, stdev1) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 7dfaec664ae51e61f86ba232f26c4d0508a9549d Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 30 May 2019 16:02:07 +1000 Subject: [PATCH 282/339] Update distro package links --- docs/guides/main.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 1c0eb80e0..563b7ef82 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -26,37 +26,37 @@ Beets works on `Python 2.7`_ and Python 3.4 or later. as described below by running: ``apt-get install python-dev python-pip`` -* On **Arch Linux**, `beets is in [community]`_, so just run ``pacman -S - beets``. (There's also a bleeding-edge `dev package`_ in the AUR, which will +* On **Arch Linux**, `beets is in [community] `_, so just run ``pacman -S + beets``. (There's also a bleeding-edge `dev package `_ in the AUR, which will probably set your computer on fire.) * For **Gentoo Linux**, beets is in Portage as ``media-sound/beets``. Just run ``emerge beets`` to install. There are several USE flags available for optional plugin dependencies. -* On **FreeBSD**, there's a `beets port`_ at ``audio/beets``. +* On **FreeBSD**, there's a `beets port `_ at ``audio/beets``. -* On **OpenBSD**, beets can be installed with ``pkg_add beets``. +* On **OpenBSD**, there's a `beets port `_ can be installed with ``pkg_add beets``. * For **Slackware**, there's a `SlackBuild`_ available. -* On **Fedora** 22 or later, there is a `DNF package`_ (or three):: +* On **Fedora** 22 or later, there is a `DNF package`_:: $ sudo dnf install beets beets-plugins beets-doc * On **Solus**, run ``eopkg install beets``. -* On **NixOS**, run ``nix-env -i beets``. +* On **NixOS**, there's a `package `_ you can install with ``nix-env -i beets``. -.. _copr: https://copr.fedoraproject.org/coprs/afreof/beets/ -.. _dnf package: https://apps.fedoraproject.org/packages/beets -.. _SlackBuild: http://slackbuilds.org/repository/14.1/multimedia/beets/ -.. _beets port: http://portsmon.freebsd.org/portoverview.py?category=audio&portname=beets -.. _beets from AUR: https://aur.archlinux.org/packages/beets-git/ -.. _dev package: https://aur.archlinux.org/packages/beets-git/ -.. _Debian details: http://packages.qa.debian.org/b/beets.html +.. _DNF package: https://apps.fedoraproject.org/packages/beets +.. _SlackBuild: http://slackbuilds.org/repository/14.2/multimedia/beets/ +.. _FreeBSD: http://portsmon.freebsd.org/portoverview.py?category=audio&portname=beets +.. _AUR: https://aur.archlinux.org/packages/beets-git/ +.. _Debian details: https://tracker.debian.org/pkg/beets .. _Ubuntu details: https://launchpad.net/ubuntu/+source/beets -.. _beets is in [community]: https://www.archlinux.org/packages/community/any/beets/ +.. _OpenBSD: http://openports.se/audio/beets +.. _Arch community: https://www.archlinux.org/packages/community/any/beets/ +.. _NixOS: https://github.com/NixOS/nixpkgs/tree/master/pkgs/tools/audio/beets If you have `pip`_, just say ``pip install beets`` (or ``pip install --user beets`` if you run into permissions problems). From 4ebb118f5f3763e2162ee5318898558ea126f725 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 30 May 2019 10:47:17 +0200 Subject: [PATCH 283/339] functemplate: Adapt ast syntax to PEP570 changes on python3.8 --- beets/util/functemplate.py | 46 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 5d9900f0b..79dcb9ad4 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -118,30 +118,36 @@ def compile_func(arg_names, statements, name='_the_func', debug=False): bytecode of the compiled function. """ if six.PY2: - func_def = ast.FunctionDef( - name=name.encode('utf-8'), - args=ast.arguments( - args=[ast.Name(n, ast.Param()) for n in arg_names], - vararg=None, - kwarg=None, - defaults=[ex_literal(None) for _ in arg_names], - ), - body=statements, - decorator_list=[], + name = name.encode('utf-8') + args = ast.arguments( + args=[ast.Name(n, ast.Param()) for n in arg_names], + vararg=None, + kwarg=None, + defaults=[ex_literal(None) for _ in arg_names], + ) + elif sys.version_info >= (3, 8): + args = ast.arguments( + args=[ast.arg(arg=n, annotation=None) for n in arg_names], + posonlyargs=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[ex_literal(None) for _ in arg_names], ) else: - func_def = ast.FunctionDef( - name=name, - args=ast.arguments( - args=[ast.arg(arg=n, annotation=None) for n in arg_names], - kwonlyargs=[], - kw_defaults=[], - defaults=[ex_literal(None) for _ in arg_names], - ), - body=statements, - decorator_list=[], + args = ast.arguments( + args=[ast.arg(arg=n, annotation=None) for n in arg_names], + kwonlyargs=[], + kw_defaults=[], + defaults=[ex_literal(None) for _ in arg_names], ) + func_def = ast.FunctionDef( + name=name, + args=args, + body=statements, + decorator_list=[], + ) + # The ast.Module signature changed in 3.8 to accept a list of types to # ignore. if sys.version_info >= (3, 8): From a0f326e7b51ff2160da033a63eda3266abddca13 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 09:11:25 -0400 Subject: [PATCH 284/339] Changelog for #3275 --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 21b0ae0bc..80107c69d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,12 @@ New features: * Support for the `NO_COLOR`_ environment variable. :bug:`3273` +For packagers: + +* ``pathlib`` is now an optional test dependency on Python 3.4+, removing the + need for `a Debian patch `_. + :bug:`3275` + .. _NO_COLOR: https://no-color.org From 99778d9ece151736ea2447a70e3fe7efd146b6c8 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 12:02:47 -0400 Subject: [PATCH 285/339] bpd tests: Randomize port number This is a hacky but effective way to work around a problem when running the tests in parallel where two different test executions want to use the same port. --- test/test_player.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test_player.py b/test/test_player.py index 102df1d7d..6b3e57d53 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -29,6 +29,7 @@ import time import yaml import tempfile from contextlib import contextmanager +import random from beets.util import confit, py3_path from beetsplug import bpd @@ -261,12 +262,17 @@ class BPDTestHelper(unittest.TestCase, TestHelper): self.unload_plugins() @contextmanager - def run_bpd(self, host='localhost', port=9876, password=None, + def run_bpd(self, host='localhost', port=None, password=None, do_hello=True, second_client=False): """ Runs BPD in another process, configured with the same library database as we created in the setUp method. Exposes a client that is connected to the server, and kills the server at the end. """ + # Choose a port (randomly) to avoid conflicts between parallel + # tests. + if not port: + port = 9876 + random.randint(0, 10000) + # Create a config file: config = { 'pluginpath': [py3_path(self.temp_dir)], From 5f88ec52a54319e0037a41aaa1fb328f8a820473 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Thu, 30 May 2019 22:34:40 +0200 Subject: [PATCH 286/339] fetch only one work --- beets/autotag/mb.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index a7bb6566d..b8f91f440 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -210,16 +210,13 @@ def track_info(recording, index=None, medium=None, medium_index=None, lyricist = [] composer = [] composer_sort = [] - work = [] - mb_workid = [] - work_disambig = [] for work_relation in recording.get('work-relation-list', ()): if work_relation['type'] != 'performance': continue - work.append(work_relation['work']['title']) - mb_workid.append(work_relation['work']['id']) + info.work = work_relation['work']['title'] + info.mb_workid = work_relation['work']['id'] if 'disambiguation' in work_relation['work']: - work_disambig.append(work_relation['work']['disambiguation']) + info.work_disambig = work_relation['work']['disambiguation'] for artist_relation in work_relation['work'].get( 'artist-relation-list', ()): @@ -245,10 +242,6 @@ def track_info(recording, index=None, medium=None, medium_index=None, arranger.append(artist_relation['artist']['name']) if arranger: info.arranger = u', '.join(arranger) - if work: - info.work = u', '.join(work) - info.mb_workid = u', '.join(mb_workid) - info.work_disambig = u', '.join(work_disambig) info.decode() return info From ade1df52a2ae06d8cd0e704327ac0a6ae2d16746 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 18:14:40 -0400 Subject: [PATCH 287/339] Use "feature detection" for 3.8 compat (#3278) --- beets/util/functemplate.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 79dcb9ad4..af22b7908 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -125,21 +125,16 @@ def compile_func(arg_names, statements, name='_the_func', debug=False): kwarg=None, defaults=[ex_literal(None) for _ in arg_names], ) - elif sys.version_info >= (3, 8): - args = ast.arguments( - args=[ast.arg(arg=n, annotation=None) for n in arg_names], - posonlyargs=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[ex_literal(None) for _ in arg_names], - ) else: - args = ast.arguments( - args=[ast.arg(arg=n, annotation=None) for n in arg_names], - kwonlyargs=[], - kw_defaults=[], - defaults=[ex_literal(None) for _ in arg_names], - ) + args_fields = { + 'args': [ast.arg(arg=n, annotation=None) for n in arg_names], + 'kwonlyargs': [], + 'kw_defaults': [], + 'defaults': [ex_literal(None) for _ in arg_names], + } + if 'posonlyargs' in ast.arguments._fields: # Added in Python 3.8. + args_fields['posonlyargs'] = [] + args = ast.arguments(**args_fields) func_def = ast.FunctionDef( name=name, From 0ad084a7064600b73638d6a39d53740569256af6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 19:27:39 -0400 Subject: [PATCH 288/339] Slight changelog reorg --- docs/changelog.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 80107c69d..723339d95 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,16 @@ Changelog 1.4.9 (in development) ---------------------- -Fixes: +This small update is part of our attempt to release new versions more often! +There are a few important fixes, and we're clearing the deck for a change to +beets' dependencies in the next version. + +The new feature is: + +* You can use the `NO_COLOR`_ environment variable to disable terminal colors. + :bug:`3273` + +There are some fixes in this release: * Fix a regression in the last release that made the image resizer fail to detect older versions of ImageMagick. @@ -16,12 +25,7 @@ Fixes: the ``gmusicapi`` module. :bug:`3270` -New features: - -* Support for the `NO_COLOR`_ environment variable. - :bug:`3273` - -For packagers: +Here's a note for packagers: * ``pathlib`` is now an optional test dependency on Python 3.4+, removing the need for `a Debian patch `_. From 2c6efc3127115e115f4d75d11252bda0b723c4d4 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 19:32:14 -0400 Subject: [PATCH 289/339] Changelog for #3278 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 723339d95..ce1c09844 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ There are some fixes in this release: * :doc:`/plugins/gmusic`: Fix a crash when using version 12.0.0 or later of the ``gmusicapi`` module. :bug:`3270` +* Fix an incompatibility with Python 3.8's AST changes. + :bug:`3278` Here's a note for packagers: From 2b8a2eb96bcaf418cdfcc0cfe762a18e31cff138 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 19:32:42 -0400 Subject: [PATCH 290/339] Release date for 1.4.9 --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ce1c09844..2ada09e52 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,8 +1,8 @@ Changelog ========= -1.4.9 (in development) ----------------------- +1.4.9 (May 30, 2019) +-------------------- This small update is part of our attempt to release new versions more often! There are a few important fixes, and we're clearing the deck for a change to From 1835422934b50c304388404c269ea3bc2f3dab3a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 20:08:03 -0400 Subject: [PATCH 291/339] Version bump: 1.5.0 --- beets/__init__.py | 2 +- docs/changelog.rst | 6 ++++++ docs/conf.py | 4 ++-- setup.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index 92afa2c87..a43148ae3 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -19,7 +19,7 @@ import os from beets.util import confit -__version__ = u'1.4.9' +__version__ = u'1.5.0' __author__ = u'Adrian Sampson ' diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ada09e52..b5905b938 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +1.5.0 (in development) +---------------------- + +Changelog goes here! + + 1.4.9 (May 30, 2019) -------------------- diff --git a/docs/conf.py b/docs/conf.py index 0459c85a0..bb3e3d00f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,8 +15,8 @@ master_doc = 'index' project = u'beets' copyright = u'2016, Adrian Sampson' -version = '1.4' -release = '1.4.9' +version = '1.5' +release = '1.5.0' pygments_style = 'sphinx' diff --git a/setup.py b/setup.py index 79278f8be..9968dbc59 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if 'sdist' in sys.argv: setup( name='beets', - version='1.4.9', + version='1.5.0', description='music tagger and library organizer', author='Adrian Sampson', author_email='adrian@radbox.org', From d778a5236ba163844f20bf5f499a1773ddd7bb91 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 20:24:11 -0400 Subject: [PATCH 292/339] Move changelog for #3237 to the right release --- docs/changelog.rst | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2a8e21614..eccf23728 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,23 @@ Changelog 1.5.0 (in development) ---------------------- -Changelog goes here! +For plugin developers: + +* `MediaFile`_ has been split into a standalone project. Where you used to do + ``from beets import mediafile``, now just do ``import mediafile``. Beets + re-exports MediaFile at the old location for backwards-compatibility, but a + deprecation warning is raised if you do this since we might drop this wrapper + in a future release. + +For packagers: + +* Beets' library for manipulating media file metadata has now been split to a + standalone project called `MediaFile`_, released as :pypi:`mediafile`. Beets + now depends on this new package. Beets now depends on Mutagen transitively + through MediaFile rather than directly, except in the case of one of beets' + plugins (scrub). + +.. _MediaFile: https://github.com/beetbox/mediafile 1.4.9 (May 30, 2019) @@ -41,24 +57,6 @@ Here's a note for packagers: .. _NO_COLOR: https://no-color.org -For plugin developers: - -* `MediaFile`_ has been split into a standalone project. Where you used to do - ``from beets import mediafile``, now just do ``import mediafile``. Beets - re-exports MediaFile at the old location for backwards-compatibility, but a - deprecation warning is raised if you do this since we might drop this wrapper - in a future release. - -For packagers: - -* Beets' library for manipulating media file metadata has now been split to a - standalone project called `MediaFile`_, released as :pypi:`mediafile`. Beets - now depends on this new package. Beets now depends on Mutagen transitively - through MediaFile rather than directly, except in the case of one of beets' - plugins (scrub). - -.. _MediaFile: https://github.com/beetbox/mediafile - 1.4.8 (May 16, 2019) -------------------- From 1da34f26ff1017879e34dcb18efd6715847a753f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 30 May 2019 20:58:30 -0400 Subject: [PATCH 293/339] Changelog for #3272 --- docs/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index eccf23728..a7384ec62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog 1.5.0 (in development) ---------------------- +New features: + +* We now fetch information about `works`_ from MusicBrainz. + MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` + (the MBID), and ``work_disambig`` (the disambiguation string). + Thanks to :user:`dosoe`. + :bug:`2580` :bug:`3272` + For plugin developers: * `MediaFile`_ has been split into a standalone project. Where you used to do @@ -21,6 +29,7 @@ For packagers: plugins (scrub). .. _MediaFile: https://github.com/beetbox/mediafile +.. _works: https://musicbrainz.org/doc/Work 1.4.9 (May 30, 2019) From bc9525356632c640048d0d55d5e67b9ce25c3f15 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Thu, 30 May 2019 14:45:31 +1000 Subject: [PATCH 294/339] Setup path correctly in testall.py By comparing `sys.path` as setup by nose vs. testall.py it seems that we weren't adding the top-level beets directory to the path. The script was also previously changing the working directory before running the tests. --- test/testall.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/testall.py b/test/testall.py index 88eb70117..418b4a3ca 100755 --- a/test/testall.py +++ b/test/testall.py @@ -22,16 +22,15 @@ import re import sys import unittest -pkgpath = os.path.dirname(__file__) or '.' -sys.path.append(pkgpath) -os.chdir(pkgpath) +pkgpath = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) or '..' +sys.path.insert(0, pkgpath) def suite(): s = unittest.TestSuite() # Get the suite() of every module in this directory beginning with # "test_". - for fname in os.listdir(pkgpath): + for fname in os.listdir(os.path.join(pkgpath, 'test')): match = re.match(r'(test_\S+)\.py$', fname) if match: modname = match.group(1) From e2d7780f972553d4757b41279ba9ca368f22872b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 19:03:35 +1000 Subject: [PATCH 295/339] confit: replace with a wrapper around confuse This re-exports all of the confuse module under `beets.util.confit` and patches in the minimum subset of commits to confit that have not yet been ported to confuse such that the test suite passes. A warning is issued when importing from the deprecated namespace. --- beets/util/confit.py | 1516 ++---------------------------------------- setup.py | 1 + tox.ini | 4 + 3 files changed, 56 insertions(+), 1465 deletions(-) diff --git a/beets/util/confit.py b/beets/util/confit.py index a5e522552..39fa5d83d 100644 --- a/beets/util/confit.py +++ b/beets/util/confit.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# This file is part of Confuse. -# Copyright 2016, Adrian Sampson. +# This file is part of beets. +# Copyright 2016-2019, Adrian Sampson. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -13,1295 +13,71 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Worry-free YAML configuration files. -""" from __future__ import division, absolute_import, print_function -import platform -import os -import pkgutil -import sys -import yaml -import re -import six -from collections import OrderedDict -if six.PY2: - from collections import Mapping, Sequence -else: - from collections.abc import Mapping, Sequence +import confuse -UNIX_DIR_VAR = 'XDG_CONFIG_HOME' -UNIX_DIR_FALLBACK = '~/.config' -WINDOWS_DIR_VAR = 'APPDATA' -WINDOWS_DIR_FALLBACK = '~\\AppData\\Roaming' -MAC_DIR = '~/Library/Application Support' +import warnings +warnings.warn("beets.util.confit is deprecated; use confuse instead") -CONFIG_FILENAME = 'config.yaml' -DEFAULT_FILENAME = 'config_default.yaml' -ROOT_NAME = 'root' +# Import everything from the confuse module into this module. +for key, value in confuse.__dict__.items(): + if key not in ['__name__']: + globals()[key] = value -YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token" -REDACTED_TOMBSTONE = 'REDACTED' +# Beets commit 1a8b20f3541992d4e5c575bfa2b166be5f5868df +def _as_str_seq(self, split=True): + return self.get(StrSeq(split=split)) +ConfigView.as_str_seq = _as_str_seq +del _as_str_seq -# Utilities. +# Beets commit 60bffbadbdee3652f65ff495a0436797a5296f71 +def _as_pairs(self, default_value=None): + return self.get(Pairs(default_value=default_value)) +ConfigView.as_pairs = _as_pairs +del _as_pairs -PY3 = sys.version_info[0] == 3 -STRING = str if PY3 else unicode # noqa: F821 -BASESTRING = str if PY3 else basestring # noqa: F821 -NUMERIC_TYPES = (int, float) if PY3 else (int, float, long) # noqa: F821 - -def iter_first(sequence): - """Get the first element from an iterable or raise a ValueError if - the iterator generates no values. - """ - it = iter(sequence) - try: - return next(it) - except StopIteration: - raise ValueError() - - -# Exceptions. - -class ConfigError(Exception): - """Base class for exceptions raised when querying a configuration. - """ - - -class NotFoundError(ConfigError): - """A requested value could not be found in the configuration trees. - """ - - -class ConfigValueError(ConfigError): - """The value in the configuration is illegal.""" - - -class ConfigTypeError(ConfigValueError): - """The value in the configuration did not match the expected type. - """ - - -class ConfigTemplateError(ConfigError): - """Base class for exceptions raised because of an invalid template. - """ - - -class ConfigReadError(ConfigError): - """A configuration file could not be read.""" - def __init__(self, filename, reason=None): - self.filename = filename - self.reason = reason - - message = u'file {0} could not be read'.format(filename) - if isinstance(reason, yaml.scanner.ScannerError) and \ - reason.problem == YAML_TAB_PROBLEM: - # Special-case error message for tab indentation in YAML markup. - message += u': found tab character at line {0}, column {1}'.format( - reason.problem_mark.line + 1, - reason.problem_mark.column + 1, - ) - elif reason: - # Generic error message uses exception's message. - message += u': {0}'.format(reason) - - super(ConfigReadError, self).__init__(message) - - -# Views and sources. - -class ConfigSource(dict): - """A dictionary augmented with metadata about the source of the - configuration. - """ - def __init__(self, value, filename=None, default=False): - super(ConfigSource, self).__init__(value) - if filename is not None and not isinstance(filename, BASESTRING): - raise TypeError(u'filename must be a string or None') - self.filename = filename - self.default = default - - def __repr__(self): - return 'ConfigSource({0!r}, {1!r}, {2!r})'.format( - super(ConfigSource, self), - self.filename, - self.default, - ) - - @classmethod - def of(cls, value): - """Given either a dictionary or a `ConfigSource` object, return - a `ConfigSource` object. This lets a function accept either type - of object as an argument. - """ - if isinstance(value, ConfigSource): - return value - elif isinstance(value, dict): - return ConfigSource(value) - else: - raise TypeError(u'source value must be a dict') - - -class ConfigView(object): - """A configuration "view" is a query into a program's configuration - data. A view represents a hypothetical location in the configuration - tree; to extract the data from the location, a client typically - calls the ``view.get()`` method. The client can access children in - the tree (subviews) by subscripting the parent view (i.e., - ``view[key]``). - """ - - name = None - """The name of the view, depicting the path taken through the - configuration in Python-like syntax (e.g., ``foo['bar'][42]``). - """ - - def resolve(self): - """The core (internal) data retrieval method. Generates (value, - source) pairs for each source that contains a value for this - view. May raise ConfigTypeError if a type error occurs while - traversing a source. - """ - raise NotImplementedError - - def first(self): - """Return a (value, source) pair for the first object found for - this view. This amounts to the first element returned by - `resolve`. If no values are available, a NotFoundError is - raised. - """ - pairs = self.resolve() - try: - return iter_first(pairs) - except ValueError: - raise NotFoundError(u"{0} not found".format(self.name)) - - def exists(self): - """Determine whether the view has a setting in any source. - """ - try: - self.first() - except NotFoundError: - return False - return True - - def add(self, value): - """Set the *default* value for this configuration view. The - specified value is added as the lowest-priority configuration - data source. - """ - raise NotImplementedError - - def set(self, value): - """*Override* the value for this configuration view. The - specified value is added as the highest-priority configuration - data source. - """ - raise NotImplementedError - - def root(self): - """The RootView object from which this view is descended. - """ - raise NotImplementedError - - def __repr__(self): - return '<{}: {}>'.format(self.__class__.__name__, self.name) - - def __iter__(self): - """Iterate over the keys of a dictionary view or the *subviews* - of a list view. - """ - # Try getting the keys, if this is a dictionary view. - try: - keys = self.keys() - for key in keys: - yield key - - except ConfigTypeError: - # Otherwise, try iterating over a list. - collection = self.get() - if not isinstance(collection, (list, tuple)): - raise ConfigTypeError( - u'{0} must be a dictionary or a list, not {1}'.format( - self.name, type(collection).__name__ - ) - ) - - # Yield all the indices in the list. - for index in range(len(collection)): - yield self[index] - - def __getitem__(self, key): - """Get a subview of this view.""" - return Subview(self, key) - - def __setitem__(self, key, value): - """Create an overlay source to assign a given key under this - view. - """ - self.set({key: value}) - - def __contains__(self, key): - return self[key].exists() - - def set_args(self, namespace): - """Overlay parsed command-line arguments, generated by a library - like argparse or optparse, onto this view's value. ``namespace`` - can be a ``dict`` or namespace object. - """ - args = {} - if isinstance(namespace, dict): - items = namespace.items() - else: - items = namespace.__dict__.items() - for key, value in items: - if value is not None: # Avoid unset options. - args[key] = value - self.set(args) - - # Magical conversions. These special methods make it possible to use - # View objects somewhat transparently in certain circumstances. For - # example, rather than using ``view.get(bool)``, it's possible to - # just say ``bool(view)`` or use ``view`` in a conditional. - - def __str__(self): - """Get the value for this view as a bytestring. - """ - if PY3: - return self.__unicode__() - else: - return bytes(self.get()) - - def __unicode__(self): - """Get the value for this view as a Unicode string. - """ - return STRING(self.get()) - - def __nonzero__(self): - """Gets the value for this view as a boolean. (Python 2 only.) - """ - return self.__bool__() - - def __bool__(self): - """Gets the value for this view as a boolean. (Python 3 only.) - """ - return bool(self.get()) - - # Dictionary emulation methods. - - def keys(self): - """Returns a list containing all the keys available as subviews - of the current views. This enumerates all the keys in *all* - dictionaries matching the current view, in contrast to - ``view.get(dict).keys()``, which gets all the keys for the - *first* dict matching the view. If the object for this view in - any source is not a dict, then a ConfigTypeError is raised. The - keys are ordered according to how they appear in each source. - """ - keys = [] - - for dic, _ in self.resolve(): - try: - cur_keys = dic.keys() - except AttributeError: - raise ConfigTypeError( - u'{0} must be a dict, not {1}'.format( - self.name, type(dic).__name__ - ) - ) - - for key in cur_keys: - if key not in keys: - keys.append(key) - - return keys - - def items(self): - """Iterates over (key, subview) pairs contained in dictionaries - from *all* sources at this view. If the object for this view in - any source is not a dict, then a ConfigTypeError is raised. - """ - for key in self.keys(): - yield key, self[key] - - def values(self): - """Iterates over all the subviews contained in dictionaries from - *all* sources at this view. If the object for this view in any - source is not a dict, then a ConfigTypeError is raised. - """ - for key in self.keys(): - yield self[key] - - # List/sequence emulation. - - def all_contents(self): - """Iterates over all subviews from collections at this view from - *all* sources. If the object for this view in any source is not - iterable, then a ConfigTypeError is raised. This method is - intended to be used when the view indicates a list; this method - will concatenate the contents of the list from all sources. - """ - for collection, _ in self.resolve(): - try: - it = iter(collection) - except TypeError: - raise ConfigTypeError( - u'{0} must be an iterable, not {1}'.format( - self.name, type(collection).__name__ - ) - ) - for value in it: - yield value - - # Validation and conversion. - - def flatten(self, redact=False): - """Create a hierarchy of OrderedDicts containing the data from - this view, recursively reifying all views to get their - represented values. - - If `redact` is set, then sensitive values are replaced with - the string "REDACTED". - """ - od = OrderedDict() - for key, view in self.items(): - if redact and view.redact: - od[key] = REDACTED_TOMBSTONE - else: - try: - od[key] = view.flatten(redact=redact) - except ConfigTypeError: - od[key] = view.get() - return od - - def get(self, template=None): - """Retrieve the value for this view according to the template. - - The `template` against which the values are checked can be - anything convertible to a `Template` using `as_template`. This - means you can pass in a default integer or string value, for - example, or a type to just check that something matches the type - you expect. - - May raise a `ConfigValueError` (or its subclass, - `ConfigTypeError`) or a `NotFoundError` when the configuration - doesn't satisfy the template. - """ - return as_template(template).value(self, template) - - # Shortcuts for common templates. - - def as_filename(self): - """Get the value as a path. Equivalent to `get(Filename())`. - """ - return self.get(Filename()) - - def as_choice(self, choices): - """Get the value from a list of choices. Equivalent to - `get(Choice(choices))`. - """ - return self.get(Choice(choices)) - - def as_number(self): - """Get the value as any number type: int or float. Equivalent to - `get(Number())`. - """ - return self.get(Number()) - - def as_str_seq(self, split=True): - """Get the value as a sequence of strings. Equivalent to - `get(StrSeq())`. - """ - return self.get(StrSeq(split=split)) - - def as_pairs(self, default_value=None): - """Get the value as a sequence of pairs of two strings. Equivalent to - `get(Pairs())`. - """ - return self.get(Pairs(default_value=default_value)) - - def as_str(self): - """Get the value as a (Unicode) string. Equivalent to - `get(unicode)` on Python 2 and `get(str)` on Python 3. - """ - return self.get(String()) - - # Redaction. - - @property - def redact(self): - """Whether the view contains sensitive information and should be - redacted from output. - """ - return () in self.get_redactions() - - @redact.setter - def redact(self, flag): - self.set_redaction((), flag) - - def set_redaction(self, path, flag): - """Add or remove a redaction for a key path, which should be an - iterable of keys. - """ - raise NotImplementedError() - - def get_redactions(self): - """Get the set of currently-redacted sub-key-paths at this view. - """ - raise NotImplementedError() - - -class RootView(ConfigView): - """The base of a view hierarchy. This view keeps track of the - sources that may be accessed by subviews. - """ - def __init__(self, sources): - """Create a configuration hierarchy for a list of sources. At - least one source must be provided. The first source in the list - has the highest priority. - """ - self.sources = list(sources) - self.name = ROOT_NAME - self.redactions = set() - - def add(self, obj): - self.sources.append(ConfigSource.of(obj)) - - def set(self, value): - self.sources.insert(0, ConfigSource.of(value)) - - def resolve(self): - return ((dict(s), s) for s in self.sources) - - def clear(self): - """Remove all sources (and redactions) from this - configuration. - """ - del self.sources[:] - self.redactions.clear() - - def root(self): - return self - - def set_redaction(self, path, flag): - if flag: - self.redactions.add(path) - elif path in self.redactions: - self.redactions.remove(path) - - def get_redactions(self): - return self.redactions - - -class Subview(ConfigView): - """A subview accessed via a subscript of a parent view.""" - def __init__(self, parent, key): - """Make a subview of a parent view for a given subscript key. - """ - self.parent = parent - self.key = key - - # Choose a human-readable name for this view. - if isinstance(self.parent, RootView): - self.name = '' - else: - self.name = self.parent.name - if not isinstance(self.key, int): - self.name += '.' - if isinstance(self.key, int): - self.name += u'#{0}'.format(self.key) - elif isinstance(self.key, bytes): - self.name += self.key.decode('utf-8') - elif isinstance(self.key, STRING): - self.name += self.key - else: - self.name += repr(self.key) - - def resolve(self): - for collection, source in self.parent.resolve(): - try: - value = collection[self.key] - except IndexError: - # List index out of bounds. - continue - except KeyError: - # Dict key does not exist. - continue - except TypeError: - # Not subscriptable. - raise ConfigTypeError( - u"{0} must be a collection, not {1}".format( - self.parent.name, type(collection).__name__ - ) - ) - yield value, source - - def set(self, value): - self.parent.set({self.key: value}) - - def add(self, value): - self.parent.add({self.key: value}) - - def root(self): - return self.parent.root() - - def set_redaction(self, path, flag): - self.parent.set_redaction((self.key,) + path, flag) - - def get_redactions(self): - return (kp[1:] for kp in self.parent.get_redactions() - if kp and kp[0] == self.key) - - -# Config file paths, including platform-specific paths and in-package -# defaults. - -# Based on get_root_path from Flask by Armin Ronacher. -def _package_path(name): - """Returns the path to the package containing the named module or - None if the path could not be identified (e.g., if - ``name == "__main__"``). - """ - loader = pkgutil.get_loader(name) - if loader is None or name == '__main__': - return None - - if hasattr(loader, 'get_filename'): - filepath = loader.get_filename(name) +# Beets commit 60bffbadbdee3652f65ff495a0436797a5296f71 +def _convert_value(self, x, view): + if isinstance(x, STRING): + return x + elif isinstance(x, bytes): + return x.decode('utf-8', 'ignore') else: - # Fall back to importing the specified module. - __import__(name) - filepath = sys.modules[name].__file__ - - return os.path.dirname(os.path.abspath(filepath)) + self.fail(u'must be a list of strings', view, True) +StrSeq._convert_value = _convert_value +del _convert_value -def config_dirs(): - """Return a platform-specific list of candidates for user - configuration directories on the system. - - The candidates are in order of priority, from highest to lowest. The - last element is the "fallback" location to be used when no - higher-priority config file exists. - """ - paths = [] - - if platform.system() == 'Darwin': - paths.append(MAC_DIR) - paths.append(UNIX_DIR_FALLBACK) - if UNIX_DIR_VAR in os.environ: - paths.append(os.environ[UNIX_DIR_VAR]) - - elif platform.system() == 'Windows': - paths.append(WINDOWS_DIR_FALLBACK) - if WINDOWS_DIR_VAR in os.environ: - paths.append(os.environ[WINDOWS_DIR_VAR]) +# Beets commit 60bffbadbdee3652f65ff495a0436797a5296f71 +def _convert(self, value, view): + if isinstance(value, bytes): + value = value.decode('utf-8', 'ignore') + if isinstance(value, STRING): + if self.split: + value = value.split() + else: + value = [value] else: - # Assume Unix. - paths.append(UNIX_DIR_FALLBACK) - if UNIX_DIR_VAR in os.environ: - paths.append(os.environ[UNIX_DIR_VAR]) + try: + value = list(value) + except TypeError: + self.fail(u'must be a whitespace-separated string or a list', + view, True) - # Expand and deduplicate paths. - out = [] - for path in paths: - path = os.path.abspath(os.path.expanduser(path)) - if path not in out: - out.append(path) - return out - - -# YAML loading. - -class Loader(yaml.SafeLoader): - """A customized YAML loader. This loader deviates from the official - YAML spec in a few convenient ways: - - - All strings as are Unicode objects. - - All maps are OrderedDicts. - - Strings can begin with % without quotation. - """ - # All strings should be Unicode objects, regardless of contents. - def _construct_unicode(self, node): - return self.construct_scalar(node) - - # Use ordered dictionaries for every YAML map. - # From https://gist.github.com/844388 - def construct_yaml_map(self, node): - data = OrderedDict() - yield data - value = self.construct_mapping(node) - data.update(value) - - def construct_mapping(self, node, deep=False): - if isinstance(node, yaml.MappingNode): - self.flatten_mapping(node) - else: - raise yaml.constructor.ConstructorError( - None, None, - u'expected a mapping node, but found %s' % node.id, - node.start_mark - ) - - mapping = OrderedDict() - for key_node, value_node in node.value: - key = self.construct_object(key_node, deep=deep) - try: - hash(key) - except TypeError as exc: - raise yaml.constructor.ConstructorError( - u'while constructing a mapping', - node.start_mark, 'found unacceptable key (%s)' % exc, - key_node.start_mark - ) - value = self.construct_object(value_node, deep=deep) - mapping[key] = value - return mapping - - # Allow bare strings to begin with %. Directives are still detected. - def check_plain(self): - plain = super(Loader, self).check_plain() - return plain or self.peek() == '%' - - -Loader.add_constructor('tag:yaml.org,2002:str', Loader._construct_unicode) -Loader.add_constructor('tag:yaml.org,2002:map', Loader.construct_yaml_map) -Loader.add_constructor('tag:yaml.org,2002:omap', Loader.construct_yaml_map) - - -def load_yaml(filename): - """Read a YAML document from a file. If the file cannot be read or - parsed, a ConfigReadError is raised. - """ - try: - with open(filename, 'rb') as f: - return yaml.load(f, Loader=Loader) - except (IOError, yaml.error.YAMLError) as exc: - raise ConfigReadError(filename, exc) - - -# YAML dumping. - -class Dumper(yaml.SafeDumper): - """A PyYAML Dumper that represents OrderedDicts as ordinary mappings - (in order, of course). - """ - # From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py - def represent_mapping(self, tag, mapping, flow_style=None): - value = [] - node = yaml.MappingNode(tag, value, flow_style=flow_style) - if self.alias_key is not None: - self.represented_objects[self.alias_key] = node - best_style = False - if hasattr(mapping, 'items'): - mapping = list(mapping.items()) - for item_key, item_value in mapping: - node_key = self.represent_data(item_key) - node_value = self.represent_data(item_value) - if not (isinstance(node_key, yaml.ScalarNode) and - not node_key.style): - best_style = False - if not (isinstance(node_value, yaml.ScalarNode) and - not node_value.style): - best_style = False - value.append((node_key, node_value)) - if flow_style is None: - if self.default_flow_style is not None: - node.flow_style = self.default_flow_style - else: - node.flow_style = best_style - return node - - def represent_list(self, data): - """If a list has less than 4 items, represent it in inline style - (i.e. comma separated, within square brackets). - """ - node = super(Dumper, self).represent_list(data) - length = len(data) - if self.default_flow_style is None and length < 4: - node.flow_style = True - elif self.default_flow_style is None: - node.flow_style = False - return node - - def represent_bool(self, data): - """Represent bool as 'yes' or 'no' instead of 'true' or 'false'. - """ - if data: - value = u'yes' - else: - value = u'no' - return self.represent_scalar('tag:yaml.org,2002:bool', value) - - def represent_none(self, data): - """Represent a None value with nothing instead of 'none'. - """ - return self.represent_scalar('tag:yaml.org,2002:null', '') - - -Dumper.add_representer(OrderedDict, Dumper.represent_dict) -Dumper.add_representer(bool, Dumper.represent_bool) -Dumper.add_representer(type(None), Dumper.represent_none) -Dumper.add_representer(list, Dumper.represent_list) - - -def restore_yaml_comments(data, default_data): - """Scan default_data for comments (we include empty lines in our - definition of comments) and place them before the same keys in data. - Only works with comments that are on one or more own lines, i.e. - not next to a yaml mapping. - """ - comment_map = dict() - default_lines = iter(default_data.splitlines()) - for line in default_lines: - if not line: - comment = "\n" - elif line.startswith("#"): - comment = "{0}\n".format(line) - else: - continue - while True: - line = next(default_lines) - if line and not line.startswith("#"): - break - comment += "{0}\n".format(line) - key = line.split(':')[0].strip() - comment_map[key] = comment - out_lines = iter(data.splitlines()) - out_data = "" - for line in out_lines: - key = line.split(':')[0].strip() - if key in comment_map: - out_data += comment_map[key] - out_data += "{0}\n".format(line) - return out_data - - -# Main interface. - -class Configuration(RootView): - def __init__(self, appname, modname=None, read=True): - """Create a configuration object by reading the - automatically-discovered config files for the application for a - given name. If `modname` is specified, it should be the import - name of a module whose package will be searched for a default - config file. (Otherwise, no defaults are used.) Pass `False` for - `read` to disable automatic reading of all discovered - configuration files. Use this when creating a configuration - object at module load time and then call the `read` method - later. - """ - super(Configuration, self).__init__([]) - self.appname = appname - self.modname = modname - - # Resolve default source location. We do this ahead of time to - # avoid unexpected problems if the working directory changes. - self._package_path = _package_path(appname) - - self._env_var = '{0}DIR'.format(self.appname.upper()) - - if read: - self.read() - - def user_config_path(self): - """Points to the location of the user configuration. - - The file may not exist. - """ - return os.path.join(self.config_dir(), CONFIG_FILENAME) - - def _add_user_source(self): - """Add the configuration options from the YAML file in the - user's configuration directory (given by `config_dir`) if it - exists. - """ - filename = self.user_config_path() - if os.path.isfile(filename): - self.add(ConfigSource(load_yaml(filename) or {}, filename)) - - def _add_default_source(self): - """Add the package's default configuration settings. This looks - for a YAML file located inside the package for the module - `modname` if it was given. - """ - if self.modname: - if self._package_path: - filename = os.path.join(self._package_path, DEFAULT_FILENAME) - if os.path.isfile(filename): - self.add(ConfigSource(load_yaml(filename), filename, True)) - - def read(self, user=True, defaults=True): - """Find and read the files for this configuration and set them - as the sources for this configuration. To disable either - discovered user configuration files or the in-package defaults, - set `user` or `defaults` to `False`. - """ - if user: - self._add_user_source() - if defaults: - self._add_default_source() - - def config_dir(self): - """Get the path to the user configuration directory. The - directory is guaranteed to exist as a postcondition (one may be - created if none exist). - - If the application's ``...DIR`` environment variable is set, it - is used as the configuration directory. Otherwise, - platform-specific standard configuration locations are searched - for a ``config.yaml`` file. If no configuration file is found, a - fallback path is used. - """ - # If environment variable is set, use it. - if self._env_var in os.environ: - appdir = os.environ[self._env_var] - appdir = os.path.abspath(os.path.expanduser(appdir)) - if os.path.isfile(appdir): - raise ConfigError(u'{0} must be a directory'.format( - self._env_var - )) - - else: - # Search platform-specific locations. If no config file is - # found, fall back to the final directory in the list. - for confdir in config_dirs(): - appdir = os.path.join(confdir, self.appname) - if os.path.isfile(os.path.join(appdir, CONFIG_FILENAME)): - break - - # Ensure that the directory exists. - if not os.path.isdir(appdir): - os.makedirs(appdir) - return appdir - - def set_file(self, filename): - """Parses the file as YAML and inserts it into the configuration - sources with highest priority. - """ - filename = os.path.abspath(filename) - self.set(ConfigSource(load_yaml(filename), filename)) - - def dump(self, full=True, redact=False): - """Dump the Configuration object to a YAML file. - - The order of the keys is determined from the default - configuration file. All keys not in the default configuration - will be appended to the end of the file. - - :param filename: The file to dump the configuration to, or None - if the YAML string should be returned instead - :type filename: unicode - :param full: Dump settings that don't differ from the defaults - as well - :param redact: Remove sensitive information (views with the `redact` - flag set) from the output - """ - if full: - out_dict = self.flatten(redact=redact) - else: - # Exclude defaults when flattening. - sources = [s for s in self.sources if not s.default] - temp_root = RootView(sources) - temp_root.redactions = self.redactions - out_dict = temp_root.flatten(redact=redact) - - yaml_out = yaml.dump(out_dict, Dumper=Dumper, - default_flow_style=None, indent=4, - width=1000) - - # Restore comments to the YAML text. - default_source = None - for source in self.sources: - if source.default: - default_source = source - break - if default_source and default_source.filename: - with open(default_source.filename, 'rb') as fp: - default_data = fp.read() - yaml_out = restore_yaml_comments(yaml_out, - default_data.decode('utf8')) - - return yaml_out - - -class LazyConfig(Configuration): - """A Configuration at reads files on demand when it is first - accessed. This is appropriate for using as a global config object at - the module level. - """ - def __init__(self, appname, modname=None): - super(LazyConfig, self).__init__(appname, modname, False) - self._materialized = False # Have we read the files yet? - self._lazy_prefix = [] # Pre-materialization calls to set(). - self._lazy_suffix = [] # Calls to add(). - - def read(self, user=True, defaults=True): - self._materialized = True - super(LazyConfig, self).read(user, defaults) - - def resolve(self): - if not self._materialized: - # Read files and unspool buffers. - self.read() - self.sources += self._lazy_suffix - self.sources[:0] = self._lazy_prefix - return super(LazyConfig, self).resolve() - - def add(self, value): - super(LazyConfig, self).add(value) - if not self._materialized: - # Buffer additions to end. - self._lazy_suffix += self.sources - del self.sources[:] - - def set(self, value): - super(LazyConfig, self).set(value) - if not self._materialized: - # Buffer additions to beginning. - self._lazy_prefix[:0] = self.sources - del self.sources[:] - - def clear(self): - """Remove all sources from this configuration.""" - super(LazyConfig, self).clear() - self._lazy_suffix = [] - self._lazy_prefix = [] - - -# "Validated" configuration views: experimental! - - -REQUIRED = object() -"""A sentinel indicating that there is no default value and an exception -should be raised when the value is missing. -""" - - -class Template(object): - """A value template for configuration fields. - - The template works like a type and instructs Confuse about how to - interpret a deserialized YAML value. This includes type conversions, - providing a default value, and validating for errors. For example, a - filepath type might expand tildes and check that the file exists. - """ - def __init__(self, default=REQUIRED): - """Create a template with a given default value. - - If `default` is the sentinel `REQUIRED` (as it is by default), - then an error will be raised when a value is missing. Otherwise, - missing values will instead return `default`. - """ - self.default = default - - def __call__(self, view): - """Invoking a template on a view gets the view's value according - to the template. - """ - return self.value(view, self) - - def value(self, view, template=None): - """Get the value for a `ConfigView`. - - May raise a `NotFoundError` if the value is missing (and the - template requires it) or a `ConfigValueError` for invalid values. - """ - if view.exists(): - value, _ = view.first() - return self.convert(value, view) - elif self.default is REQUIRED: - # Missing required value. This is an error. - raise NotFoundError(u"{0} not found".format(view.name)) - else: - # Missing value, but not required. - return self.default - - def convert(self, value, view): - """Convert the YAML-deserialized value to a value of the desired - type. - - Subclasses should override this to provide useful conversions. - May raise a `ConfigValueError` when the configuration is wrong. - """ - # Default implementation does no conversion. - return value - - def fail(self, message, view, type_error=False): - """Raise an exception indicating that a value cannot be - accepted. - - `type_error` indicates whether the error is due to a type - mismatch rather than a malformed value. In this case, a more - specific exception is raised. - """ - exc_class = ConfigTypeError if type_error else ConfigValueError - raise exc_class( - u'{0}: {1}'.format(view.name, message) - ) - - def __repr__(self): - return '{0}({1})'.format( - type(self).__name__, - '' if self.default is REQUIRED else repr(self.default), - ) - - -class Integer(Template): - """An integer configuration value template. - """ - def convert(self, value, view): - """Check that the value is an integer. Floats are rounded. - """ - if isinstance(value, int): - return value - elif isinstance(value, float): - return int(value) - else: - self.fail(u'must be a number', view, True) - - -class Number(Template): - """A numeric type: either an integer or a floating-point number. - """ - def convert(self, value, view): - """Check that the value is an int or a float. - """ - if isinstance(value, NUMERIC_TYPES): - return value - else: - self.fail( - u'must be numeric, not {0}'.format(type(value).__name__), - view, - True - ) - - -class MappingTemplate(Template): - """A template that uses a dictionary to specify other types for the - values for a set of keys and produce a validated `AttrDict`. - """ - def __init__(self, mapping): - """Create a template according to a dict (mapping). The - mapping's values should themselves either be Types or - convertible to Types. - """ - subtemplates = {} - for key, typ in mapping.items(): - subtemplates[key] = as_template(typ) - self.subtemplates = subtemplates - - def value(self, view, template=None): - """Get a dict with the same keys as the template and values - validated according to the value types. - """ - out = AttrDict() - for key, typ in self.subtemplates.items(): - out[key] = typ.value(view[key], self) - return out - - def __repr__(self): - return 'MappingTemplate({0})'.format(repr(self.subtemplates)) - - -class String(Template): - """A string configuration value template. - """ - def __init__(self, default=REQUIRED, pattern=None): - """Create a template with the added optional `pattern` argument, - a regular expression string that the value should match. - """ - super(String, self).__init__(default) - self.pattern = pattern - if pattern: - self.regex = re.compile(pattern) - - def __repr__(self): - args = [] - - if self.default is not REQUIRED: - args.append(repr(self.default)) - - if self.pattern is not None: - args.append('pattern=' + repr(self.pattern)) - - return 'String({0})'.format(', '.join(args)) - - def convert(self, value, view): - """Check that the value is a string and matches the pattern. - """ - if isinstance(value, BASESTRING): - if self.pattern and not self.regex.match(value): - self.fail( - u"must match the pattern {0}".format(self.pattern), - view - ) - return value - else: - self.fail(u'must be a string', view, True) - - -class Choice(Template): - """A template that permits values from a sequence of choices. - """ - def __init__(self, choices): - """Create a template that validates any of the values from the - iterable `choices`. - - If `choices` is a map, then the corresponding value is emitted. - Otherwise, the value itself is emitted. - """ - self.choices = choices - - def convert(self, value, view): - """Ensure that the value is among the choices (and remap if the - choices are a mapping). - """ - if value not in self.choices: - self.fail( - u'must be one of {0}, not {1}'.format( - repr(list(self.choices)), repr(value) - ), - view - ) - - if isinstance(self.choices, Mapping): - return self.choices[value] - else: - return value - - def __repr__(self): - return 'Choice({0!r})'.format(self.choices) - - -class OneOf(Template): - """A template that permits values complying to one of the given templates. - """ - def __init__(self, allowed, default=REQUIRED): - super(OneOf, self).__init__(default) - self.allowed = list(allowed) - - def __repr__(self): - args = [] - - if self.allowed is not None: - args.append('allowed=' + repr(self.allowed)) - - if self.default is not REQUIRED: - args.append(repr(self.default)) - - return 'OneOf({0})'.format(', '.join(args)) - - def value(self, view, template): - self.template = template - return super(OneOf, self).value(view, template) - - def convert(self, value, view): - """Ensure that the value follows at least one template. - """ - is_mapping = isinstance(self.template, MappingTemplate) - - for candidate in self.allowed: - try: - if is_mapping: - if isinstance(candidate, Filename) and \ - candidate.relative_to: - next_template = candidate.template_with_relatives( - view, - self.template - ) - - next_template.subtemplates[view.key] = as_template( - candidate - ) - else: - next_template = MappingTemplate({view.key: candidate}) - - return view.parent.get(next_template)[view.key] - else: - return view.get(candidate) - except ConfigTemplateError: - raise - except ConfigError: - pass - except ValueError as exc: - raise ConfigTemplateError(exc) - - self.fail( - u'must be one of {0}, not {1}'.format( - repr(self.allowed), repr(value) - ), - view - ) - - -class StrSeq(Template): - """A template for values that are lists of strings. - - Validates both actual YAML string lists and single strings. Strings - can optionally be split on whitespace. - """ - def __init__(self, split=True): - """Create a new template. - - `split` indicates whether, when the underlying value is a single - string, it should be split on whitespace. Otherwise, the - resulting value is a list containing a single string. - """ - super(StrSeq, self).__init__() - self.split = split - - def _convert_value(self, x, view): - if isinstance(x, STRING): - return x - elif isinstance(x, bytes): - return x.decode('utf-8', 'ignore') - else: - self.fail(u'must be a list of strings', view, True) - - def convert(self, value, view): - if isinstance(value, bytes): - value = value.decode('utf-8', 'ignore') - - if isinstance(value, STRING): - if self.split: - value = value.split() - else: - value = [value] - else: - try: - value = list(value) - except TypeError: - self.fail(u'must be a whitespace-separated string or a list', - view, True) - - return [self._convert_value(v, view) for v in value] + return [self._convert_value(v, view) for v in value] +StrSeq.convert = _convert +del _convert +# Beets commits 60bffbadbdee3652f65ff495a0436797a5296f71 +# and 318f0c4d16710712ed49d10c621fbf52705161dd class Pairs(StrSeq): - """A template for ordered key-value pairs. - - This can either be given with the same syntax as for `StrSeq` (i.e. without - values), or as a list of strings and/or single-element mappings such as:: - - - key: value - - [key, value] - - key - - The result is a list of two-element tuples. If no value is provided, the - `default_value` will be returned as the second element. - """ - def __init__(self, default_value=None): - """Create a new template. - - `default` is the dictionary value returned for items that are not - a mapping, but a single string. - """ super(Pairs, self).__init__(split=True) self.default_value = default_value @@ -1310,17 +86,15 @@ class Pairs(StrSeq): return (super(Pairs, self)._convert_value(x, view), self.default_value) except ConfigTypeError: - if isinstance(x, Mapping): + if isinstance(x, collections.Mapping): if len(x) != 1: self.fail(u'must be a single-element mapping', view, True) k, v = iter_first(x.items()) - elif isinstance(x, Sequence): + elif isinstance(x, collections.Sequence): if len(x) != 2: self.fail(u'must be a two-element list', view, True) k, v = x else: - # Is this even possible? -> Likely, if some !directive cause - # YAML to parse this to some custom type. self.fail(u'must be a single string, mapping, or a list' u'' + str(x), view, True) @@ -1328,193 +102,5 @@ class Pairs(StrSeq): super(Pairs, self)._convert_value(v, view)) -class Filename(Template): - """A template that validates strings as filenames. - - Filenames are returned as absolute, tilde-free paths. - - Relative paths are relative to the template's `cwd` argument - when it is specified, then the configuration directory (see - the `config_dir` method) if they come from a file. Otherwise, - they are relative to the current working directory. This helps - attain the expected behavior when using command-line options. - """ - def __init__(self, default=REQUIRED, cwd=None, relative_to=None, - in_app_dir=False): - """`relative_to` is the name of a sibling value that is - being validated at the same time. - - `in_app_dir` indicates whether the path should be resolved - inside the application's config directory (even when the setting - does not come from a file). - """ - super(Filename, self).__init__(default) - self.cwd = cwd - self.relative_to = relative_to - self.in_app_dir = in_app_dir - - def __repr__(self): - args = [] - - if self.default is not REQUIRED: - args.append(repr(self.default)) - - if self.cwd is not None: - args.append('cwd=' + repr(self.cwd)) - - if self.relative_to is not None: - args.append('relative_to=' + repr(self.relative_to)) - - if self.in_app_dir: - args.append('in_app_dir=True') - - return 'Filename({0})'.format(', '.join(args)) - - def resolve_relative_to(self, view, template): - if not isinstance(template, (Mapping, MappingTemplate)): - # disallow config.get(Filename(relative_to='foo')) - raise ConfigTemplateError( - u'relative_to may only be used when getting multiple values.' - ) - - elif self.relative_to == view.key: - raise ConfigTemplateError( - u'{0} is relative to itself'.format(view.name) - ) - - elif self.relative_to not in view.parent.keys(): - # self.relative_to is not in the config - self.fail( - ( - u'needs sibling value "{0}" to expand relative path' - ).format(self.relative_to), - view - ) - - old_template = {} - old_template.update(template.subtemplates) - - # save time by skipping MappingTemplate's init loop - next_template = MappingTemplate({}) - next_relative = self.relative_to - - # gather all the needed templates and nothing else - while next_relative is not None: - try: - # pop to avoid infinite loop because of recursive - # relative paths - rel_to_template = old_template.pop(next_relative) - except KeyError: - if next_relative in template.subtemplates: - # we encountered this config key previously - raise ConfigTemplateError(( - u'{0} and {1} are recursively relative' - ).format(view.name, self.relative_to)) - else: - raise ConfigTemplateError(( - u'missing template for {0}, needed to expand {1}\'s' + - u'relative path' - ).format(self.relative_to, view.name)) - - next_template.subtemplates[next_relative] = rel_to_template - next_relative = rel_to_template.relative_to - - return view.parent.get(next_template)[self.relative_to] - - def value(self, view, template=None): - path, source = view.first() - if not isinstance(path, BASESTRING): - self.fail( - u'must be a filename, not {0}'.format(type(path).__name__), - view, - True - ) - path = os.path.expanduser(STRING(path)) - - if not os.path.isabs(path): - if self.cwd is not None: - # relative to the template's argument - path = os.path.join(self.cwd, path) - - elif self.relative_to is not None: - path = os.path.join( - self.resolve_relative_to(view, template), - path, - ) - - elif source.filename or self.in_app_dir: - # From defaults: relative to the app's directory. - path = os.path.join(view.root().config_dir(), path) - - return os.path.abspath(path) - - -class TypeTemplate(Template): - """A simple template that checks that a value is an instance of a - desired Python type. - """ - def __init__(self, typ, default=REQUIRED): - """Create a template that checks that the value is an instance - of `typ`. - """ - super(TypeTemplate, self).__init__(default) - self.typ = typ - - def convert(self, value, view): - if not isinstance(value, self.typ): - self.fail( - u'must be a {0}, not {1}'.format( - self.typ.__name__, - type(value).__name__, - ), - view, - True - ) - return value - - -class AttrDict(dict): - """A `dict` subclass that can be accessed via attributes (dot - notation) for convenience. - """ - def __getattr__(self, key): - if key in self: - return self[key] - else: - raise AttributeError(key) - - -def as_template(value): - """Convert a simple "shorthand" Python value to a `Template`. - """ - if isinstance(value, Template): - # If it's already a Template, pass it through. - return value - elif isinstance(value, Mapping): - # Dictionaries work as templates. - return MappingTemplate(value) - elif value is int: - return Integer() - elif isinstance(value, int): - return Integer(value) - elif isinstance(value, type) and issubclass(value, BASESTRING): - return String() - elif isinstance(value, BASESTRING): - return String(value) - elif isinstance(value, set): - # convert to list to avoid hash related problems - return Choice(list(value)) - elif isinstance(value, list): - return OneOf(value) - elif value is float: - return Number() - elif value is None: - return Template() - elif value is dict: - return TypeTemplate(Mapping) - elif value is list: - return TypeTemplate(Sequence) - elif isinstance(value, type): - return TypeTemplate(value) - else: - raise ValueError(u'cannot convert to template: {0!r}'.format(value)) +# Cleanup namespace. +del key, value, warnings, confuse diff --git a/setup.py b/setup.py index 0d9fc8840..d21fa2f57 100755 --- a/setup.py +++ b/setup.py @@ -91,6 +91,7 @@ setup( 'musicbrainzngs>=0.4', 'pyyaml', 'mediafile>=0.1.0', + 'confuse>=0.5.0', ] + [ # Avoid a version of munkres incompatible with Python 3. 'munkres~=1.0.0' if sys.version_info < (3, 5, 0) else diff --git a/tox.ini b/tox.ini index 8736f0f3c..fabec7261 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,10 @@ deps = coverage discogs-client +[flake8] +per-file-ignores = + beets/util/confit.py:F821 + [_flake8] deps = flake8 From f1c9b116353c9867231041d8a55b799ad4220485 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 09:34:03 +1000 Subject: [PATCH 296/339] confit: depend on new confuse and remove backports --- beets/util/confit.py | 76 -------------------------------------------- setup.py | 2 +- tox.ini | 4 --- 3 files changed, 1 insertion(+), 81 deletions(-) diff --git a/beets/util/confit.py b/beets/util/confit.py index 39fa5d83d..450e37210 100644 --- a/beets/util/confit.py +++ b/beets/util/confit.py @@ -26,81 +26,5 @@ for key, value in confuse.__dict__.items(): globals()[key] = value -# Beets commit 1a8b20f3541992d4e5c575bfa2b166be5f5868df -def _as_str_seq(self, split=True): - return self.get(StrSeq(split=split)) -ConfigView.as_str_seq = _as_str_seq -del _as_str_seq - - -# Beets commit 60bffbadbdee3652f65ff495a0436797a5296f71 -def _as_pairs(self, default_value=None): - return self.get(Pairs(default_value=default_value)) -ConfigView.as_pairs = _as_pairs -del _as_pairs - - -# Beets commit 60bffbadbdee3652f65ff495a0436797a5296f71 -def _convert_value(self, x, view): - if isinstance(x, STRING): - return x - elif isinstance(x, bytes): - return x.decode('utf-8', 'ignore') - else: - self.fail(u'must be a list of strings', view, True) -StrSeq._convert_value = _convert_value -del _convert_value - - -# Beets commit 60bffbadbdee3652f65ff495a0436797a5296f71 -def _convert(self, value, view): - if isinstance(value, bytes): - value = value.decode('utf-8', 'ignore') - - if isinstance(value, STRING): - if self.split: - value = value.split() - else: - value = [value] - else: - try: - value = list(value) - except TypeError: - self.fail(u'must be a whitespace-separated string or a list', - view, True) - - return [self._convert_value(v, view) for v in value] -StrSeq.convert = _convert -del _convert - - -# Beets commits 60bffbadbdee3652f65ff495a0436797a5296f71 -# and 318f0c4d16710712ed49d10c621fbf52705161dd -class Pairs(StrSeq): - def __init__(self, default_value=None): - super(Pairs, self).__init__(split=True) - self.default_value = default_value - - def _convert_value(self, x, view): - try: - return (super(Pairs, self)._convert_value(x, view), - self.default_value) - except ConfigTypeError: - if isinstance(x, collections.Mapping): - if len(x) != 1: - self.fail(u'must be a single-element mapping', view, True) - k, v = iter_first(x.items()) - elif isinstance(x, collections.Sequence): - if len(x) != 2: - self.fail(u'must be a two-element list', view, True) - k, v = x - else: - self.fail(u'must be a single string, mapping, or a list' - u'' + str(x), - view, True) - return (super(Pairs, self)._convert_value(k, view), - super(Pairs, self)._convert_value(v, view)) - - # Cleanup namespace. del key, value, warnings, confuse diff --git a/setup.py b/setup.py index d21fa2f57..0cbba71de 100755 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ setup( 'musicbrainzngs>=0.4', 'pyyaml', 'mediafile>=0.1.0', - 'confuse>=0.5.0', + 'confuse>=1.0.0', ] + [ # Avoid a version of munkres incompatible with Python 3. 'munkres~=1.0.0' if sys.version_info < (3, 5, 0) else diff --git a/tox.ini b/tox.ini index fabec7261..8736f0f3c 100644 --- a/tox.ini +++ b/tox.ini @@ -25,10 +25,6 @@ deps = coverage discogs-client -[flake8] -per-file-ignores = - beets/util/confit.py:F821 - [_flake8] deps = flake8 From a82002e6c1928d5fd37a574c024d29ad421b5f96 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 09:51:56 +1000 Subject: [PATCH 297/339] confit: replace with confuse in core beets --- beets/__init__.py | 8 ++++---- beets/plugins.py | 2 +- beets/ui/__init__.py | 7 ++++--- beets/ui/commands.py | 2 +- docs/dev/plugins.rst | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index a43148ae3..20075073c 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -17,14 +17,14 @@ from __future__ import division, absolute_import, print_function import os -from beets.util import confit +import confuse __version__ = u'1.5.0' __author__ = u'Adrian Sampson ' -class IncludeLazyConfig(confit.LazyConfig): - """A version of Confit's LazyConfig that also merges in data from +class IncludeLazyConfig(confuse.LazyConfig): + """A version of Confuse's LazyConfig that also merges in data from YAML files specified in an `include` setting. """ def read(self, user=True, defaults=True): @@ -35,7 +35,7 @@ class IncludeLazyConfig(confit.LazyConfig): filename = view.as_filename() if os.path.isfile(filename): self.set_file(filename) - except confit.NotFoundError: + except confuse.NotFoundError: pass diff --git a/beets/plugins.py b/beets/plugins.py index edc6d8fe8..7c98225ca 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -526,7 +526,7 @@ def sanitize_choices(choices, choices_all): def sanitize_pairs(pairs, pairs_all): """Clean up a single-element mapping configuration attribute as returned - by `confit`'s `Pairs` template: keep only two-element tuples present in + by Confuse's `Pairs` template: keep only two-element tuples present in pairs_all, remove duplicate elements, expand ('str', '*') and ('*', '*') wildcards while keeping the original order. Note that ('*', '*') and ('*', 'whatever') have the same effect. diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 5b8f9680a..a88ed9aef 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -38,10 +38,11 @@ from beets import plugins from beets import util from beets.util.functemplate import template from beets import config -from beets.util import confit, as_string +from beets.util import as_string from beets.autotag import mb from beets.dbcore import query as db_query from beets.dbcore import db +import confuse import six # On Windows platforms, use colorama to support "ANSI" terminal colors. @@ -621,7 +622,7 @@ def get_path_formats(subview=None): def get_replacements(): - """Confit validation function that reads regex/string pairs. + """Confuse validation function that reads regex/string pairs. """ replacements = [] for pattern, repl in config['replace'].get(dict).items(): @@ -1277,7 +1278,7 @@ def main(args=None): log.debug('{}', traceback.format_exc()) log.error('{}', exc) sys.exit(1) - except confit.ConfigError as exc: + except confuse.ConfigError as exc: log.error(u'configuration error: {0}', exc) sys.exit(1) except db_query.InvalidQueryError as exc: diff --git a/beets/ui/commands.py b/beets/ui/commands.py index c89dbb6db..3f540a1fa 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -39,7 +39,7 @@ from beets.util import syspath, normpath, ancestry, displayable_path, \ from beets import library from beets import config from beets import logging -from beets.util.confit import _package_path +from confuse import _package_path import six from . import _store_dict diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 7cec77d62..7ff397bc6 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -298,10 +298,10 @@ this in their ``config.yaml``:: foo: bar To access this value, say ``self.config['foo'].get()`` at any point in your -plugin's code. The `self.config` object is a *view* as defined by the `Confit`_ +plugin's code. The `self.config` object is a *view* as defined by the `Confuse`_ library. -.. _Confit: http://confit.readthedocs.org/ +.. _Confuse: http://confuse.readthedocs.org/ If you want to access configuration values *outside* of your plugin's section, import the `config` object from the `beets` module. That is, just put ``from From 6382c364e21e45f209e80f6d21ec38914efc1462 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 10:03:41 +1000 Subject: [PATCH 298/339] confit: replace with confuse in plugins --- beetsplug/badfiles.py | 5 +++-- beetsplug/beatport.py | 4 ++-- beetsplug/chroma.py | 4 ++-- beetsplug/convert.py | 2 +- beetsplug/discogs.py | 4 ++-- beetsplug/fetchart.py | 11 ++++++----- beetsplug/metasync/__init__.py | 2 +- beetsplug/metasync/itunes.py | 2 +- beetsplug/spotify.py | 4 ++-- beetsplug/types.py | 2 +- beetsplug/zero.py | 4 ++-- 11 files changed, 23 insertions(+), 21 deletions(-) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index fdfbf204a..36b45de3a 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -25,9 +25,10 @@ import os import errno import sys import six +import confuse from beets.plugins import BeetsPlugin from beets.ui import Subcommand -from beets.util import displayable_path, confit, par_map +from beets.util import displayable_path, par_map from beets import ui @@ -90,7 +91,7 @@ class BadFiles(BeetsPlugin): ext = ext.lower() try: command = self.config['commands'].get(dict).get(ext) - except confit.NotFoundError: + except confuse.NotFoundError: command = None if command: return self.check_custom(command) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 0c25912b2..da59bef87 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -30,7 +30,7 @@ import beets import beets.ui from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance from beets.plugins import BeetsPlugin -from beets.util import confit +import confuse AUTH_ERRORS = (TokenRequestDenied, TokenMissing, VerifierMissing) @@ -318,7 +318,7 @@ class BeatportPlugin(BeetsPlugin): def _tokenfile(self): """Get the path to the JSON file for storing the OAuth token. """ - return self.config['tokenfile'].get(confit.Filename(in_app_dir=True)) + return self.config['tokenfile'].get(confuse.Filename(in_app_dir=True)) def album_distance(self, items, album_info, mapping): """Returns the beatport source weight and the maximum source weight diff --git a/beetsplug/chroma.py b/beetsplug/chroma.py index 42abe09b5..c4230b069 100644 --- a/beetsplug/chroma.py +++ b/beetsplug/chroma.py @@ -22,8 +22,8 @@ from beets import plugins from beets import ui from beets import util from beets import config -from beets.util import confit from beets.autotag import hooks +import confuse import acoustid from collections import defaultdict from functools import partial @@ -221,7 +221,7 @@ class AcoustidPlugin(plugins.BeetsPlugin): def submit_cmd_func(lib, opts, args): try: apikey = config['acoustid']['apikey'].as_str() - except confit.NotFoundError: + except confuse.NotFoundError: raise ui.UserError(u'no Acoustid user API key provided') submit_items(self._log, apikey, lib.items(ui.decargs(args))) submit_cmd.func = submit_cmd_func diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 303563a7a..6ed139da0 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -28,7 +28,7 @@ import platform from beets import ui, util, plugins, config from beets.plugins import BeetsPlugin -from beets.util.confit import ConfigTypeError +from confuse import ConfigTypeError from beets import art from beets.util.artresizer import ArtResizer from beets.library import parse_query_string diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 5a2bf57e0..68b4b5a95 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -22,7 +22,7 @@ import beets.ui from beets import config from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance from beets.plugins import BeetsPlugin -from beets.util import confit +import confuse from discogs_client import Release, Master, Client from discogs_client.exceptions import DiscogsAPIError from requests.exceptions import ConnectionError @@ -122,7 +122,7 @@ class DiscogsPlugin(BeetsPlugin): def _tokenfile(self): """Get the path to the JSON file for storing the OAuth token. """ - return self.config['tokenfile'].get(confit.Filename(in_app_dir=True)) + return self.config['tokenfile'].get(confuse.Filename(in_app_dir=True)) def authenticate(self, c_key, c_secret): # Get the link for the OAuth page. diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 2583e4c16..af1aaa567 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -31,8 +31,9 @@ from beets import util from beets import config from mediafile import image_mime_type from beets.util.artresizer import ArtResizer -from beets.util import confit, sorted_walk +from beets.util import sorted_walk from beets.util import syspath, bytestring_path, py3_path +import confuse import six CONTENT_TYPES = { @@ -776,9 +777,9 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): # allow both pixel and percentage-based margin specifications self.enforce_ratio = self.config['enforce_ratio'].get( - confit.OneOf([bool, - confit.String(pattern=self.PAT_PX), - confit.String(pattern=self.PAT_PERCENT)])) + confuse.OneOf([bool, + confuse.String(pattern=self.PAT_PX), + confuse.String(pattern=self.PAT_PERCENT)])) self.margin_px = None self.margin_percent = None if type(self.enforce_ratio) is six.text_type: @@ -788,7 +789,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.margin_px = int(self.enforce_ratio[:-2]) else: # shouldn't happen - raise confit.ConfigValueError() + raise confuse.ConfigValueError() self.enforce_ratio = True cover_names = self.config['cover_names'].as_str_seq() diff --git a/beetsplug/metasync/__init__.py b/beetsplug/metasync/__init__.py index 02f0b0f9b..943dbac1f 100644 --- a/beetsplug/metasync/__init__.py +++ b/beetsplug/metasync/__init__.py @@ -21,7 +21,7 @@ from __future__ import division, absolute_import, print_function from abc import abstractmethod, ABCMeta from importlib import import_module -from beets.util.confit import ConfigValueError +from confuse import ConfigValueError from beets import ui from beets.plugins import BeetsPlugin import six diff --git a/beetsplug/metasync/itunes.py b/beetsplug/metasync/itunes.py index d594fa591..067ca8d91 100644 --- a/beetsplug/metasync/itunes.py +++ b/beetsplug/metasync/itunes.py @@ -31,7 +31,7 @@ from time import mktime from beets import util from beets.dbcore import types from beets.library import DateType -from beets.util.confit import ConfigValueError +from confuse import ConfigValueError from beetsplug.metasync import MetaSource diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 75f2c8523..f6df91bb3 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -14,7 +14,7 @@ import requests from beets import ui from beets.plugins import BeetsPlugin -from beets.util import confit +import confuse from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance @@ -49,7 +49,7 @@ class SpotifyPlugin(BeetsPlugin): self.config['client_secret'].redact = True self.tokenfile = self.config['tokenfile'].get( - confit.Filename(in_app_dir=True) + confuse.Filename(in_app_dir=True) ) # Path to the JSON file for storing the OAuth access token. self.setup() diff --git a/beetsplug/types.py b/beetsplug/types.py index 0c078881c..4a39f05b1 100644 --- a/beetsplug/types.py +++ b/beetsplug/types.py @@ -17,7 +17,7 @@ from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets.dbcore import types -from beets.util.confit import ConfigValueError +from confuse import ConfigValueError from beets import library diff --git a/beetsplug/zero.py b/beetsplug/zero.py index ebe1b486e..0cca199c5 100644 --- a/beetsplug/zero.py +++ b/beetsplug/zero.py @@ -24,7 +24,7 @@ from beets.plugins import BeetsPlugin from mediafile import MediaFile from beets.importer import action from beets.ui import Subcommand, decargs, input_yn -from beets.util import confit +import confuse __author__ = 'baobab@heresiarch.info' @@ -98,7 +98,7 @@ class ZeroPlugin(BeetsPlugin): for pattern in self.config[field].as_str_seq(): prog = re.compile(pattern, re.IGNORECASE) self.fields_to_progs.setdefault(field, []).append(prog) - except confit.NotFoundError: + except confuse.NotFoundError: # Matches everything self.fields_to_progs[field] = [] From d2f13bf65c01494cc09f6269bd4594fb59593516 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 10:13:13 +1000 Subject: [PATCH 299/339] confit: replace with confuse in tests --- test/_common.py | 3 +-- test/test_art.py | 4 ++-- test/test_lyrics.py | 6 ++++-- test/test_player.py | 5 +++-- test/test_types_plugin.py | 2 +- test/test_ui.py | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/_common.py b/test/_common.py index 7a3ece4d8..5412ab650 100644 --- a/test/_common.py +++ b/test/_common.py @@ -175,8 +175,7 @@ class TestCase(unittest.TestCase, Assertions): beets.config['directory'] = \ util.py3_path(os.path.join(self.temp_dir, b'libdir')) - # Set $HOME, which is used by confit's `config_dir()` to create - # directories. + # Set $HOME, which is used by Confuse to create directories. self._old_home = os.environ.get('HOME') os.environ['HOME'] = util.py3_path(self.temp_dir) diff --git a/test/test_art.py b/test/test_art.py index 857f5d3c6..556222f48 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -34,7 +34,7 @@ from beets import importer from beets import logging from beets import util from beets.util.artresizer import ArtResizer, WEBPROXY -from beets.util import confit +import confuse logger = logging.getLogger('beets.test_art') @@ -753,7 +753,7 @@ class EnforceRatioConfigTest(_common.TestCase): if should_raise: for v in values: config['fetchart']['enforce_ratio'] = v - with self.assertRaises(confit.ConfigValueError): + with self.assertRaises(confuse.ConfigValueError): fetchart.FetchArtPlugin() else: for v in values: diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 4d48ed9ed..f7ea538e2 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -28,7 +28,8 @@ from test import _common from beets import logging from beets.library import Item -from beets.util import bytestring_path, confit +from beets.util import bytestring_path +import confuse from beetsplug import lyrics @@ -222,7 +223,8 @@ def is_lyrics_content_ok(title, text): return keywords <= words LYRICS_ROOT_DIR = os.path.join(_common.RSRC, b'lyrics') -LYRICS_TEXTS = confit.load_yaml(os.path.join(_common.RSRC, b'lyricstext.yaml')) +yaml_path = os.path.join(_common.RSRC, b'lyricstext.yaml') +LYRICS_TEXTS = confuse.load_yaml(yaml_path) class LyricsGoogleBaseTest(unittest.TestCase): diff --git a/test/test_player.py b/test/test_player.py index 6b3e57d53..66f1f5d38 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -31,8 +31,9 @@ import tempfile from contextlib import contextmanager import random -from beets.util import confit, py3_path +from beets.util import py3_path from beetsplug import bpd +import confuse # Mock GstPlayer so that the forked process doesn't attempt to import gi: @@ -285,7 +286,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): mode='wb', dir=py3_path(self.temp_dir), suffix='.yaml', delete=False) config_file.write( - yaml.dump(config, Dumper=confit.Dumper, encoding='utf-8')) + yaml.dump(config, Dumper=confuse.Dumper, encoding='utf-8')) config_file.close() # Fork and launch BPD in the new process: diff --git a/test/test_types_plugin.py b/test/test_types_plugin.py index 541bd30df..77d6c8bcf 100644 --- a/test/test_types_plugin.py +++ b/test/test_types_plugin.py @@ -21,7 +21,7 @@ import unittest from test.helper import TestHelper -from beets.util.confit import ConfigValueError +from confuse import ConfigValueError class TypesPluginTest(unittest.TestCase, TestHelper): diff --git a/test/test_ui.py b/test/test_ui.py index 9aae9584b..da3160db7 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -38,7 +38,7 @@ from beets.autotag.match import distance from mediafile import MediaFile from beets import config from beets import plugins -from beets.util.confit import ConfigError +from confuse import ConfigError from beets import util from beets.util import syspath, MoveOperation From 57d5c77b33cf01c7e8fa0d138d731ab3b46b0560 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 10:24:15 +1000 Subject: [PATCH 300/339] Changelog for #3224 --- docs/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a7384ec62..371acb8c8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,12 @@ For plugin developers: re-exports MediaFile at the old location for backwards-compatibility, but a deprecation warning is raised if you do this since we might drop this wrapper in a future release. +* We've replaced beets' configuration library confit with a standalone + version called `Confuse`_. Where you used to do + ``from beets.util import confit``, now just do ``import confuse``. The code + is almost identical apart from the name change. Again, we'll re-export at the + old location (with a deprecation warning) for backwards compatibility, but + might stop doing this in a future release. For packagers: @@ -27,8 +33,13 @@ For packagers: now depends on this new package. Beets now depends on Mutagen transitively through MediaFile rather than directly, except in the case of one of beets' plugins (scrub). +* Beets' library for configuration has been split into a standalone project + called `Confuse`_, released as :pypi:`confuse`. Beets now depends on this + package. Confuse has existed separately for some time and is used by + unrelated projects, but until now we've been bundling a copy within beets. .. _MediaFile: https://github.com/beetbox/mediafile +.. _Confuse: https://github.com/beetbox/confuse .. _works: https://musicbrainz.org/doc/Work From a86b74943a4aba9b514fb929210ff92a868c261b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 10:39:51 +1000 Subject: [PATCH 301/339] Update Python 3 versions for Travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46018cd4c..8376be522 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,8 @@ matrix: # dist: xenial # - python: pypy # - env: {TOX_ENV: pypy-test} - - python: 3.4 - env: {TOX_ENV: py34-flake8} + - python: 3.6 + env: {TOX_ENV: py36-flake8} - python: 2.7.13 env: {TOX_ENV: docs} # Non-Python dependencies. From a04666eea26d7f903111676e02513483e3046211 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 1 Jun 2019 10:54:45 +1000 Subject: [PATCH 302/339] Test on Python 3.7 on AppVeyor --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index cb747fe1f..9a3102ba8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,6 +16,8 @@ environment: TOX_ENV: py35-test - PYTHON: C:\Python36 TOX_ENV: py36-test + - PYTHON: C:\Python37 + TOX_ENV: py37-test # Install Tox for running tests. install: From cd66c5d7523e536272bbcd762aa84a2ca91b3588 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 1 Jun 2019 12:03:03 -0400 Subject: [PATCH 303/339] Link to beet-summarize (fix #3177) --- docs/plugins/index.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index e75e2f810..f95a6285d 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -258,7 +258,10 @@ Here are a few of the plugins written by the beets community: * `beets-barcode`_ lets you scan or enter barcodes for physical media to search for their metadata. -* `beets-ydl`_ download audio from youtube-dl sources and import into beets +* `beets-ydl`_ downloads audio from youtube-dl sources and import into beets. + +* `beet-summarize`_ can compute lots of counts and statistics about your music + library. .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check @@ -280,3 +283,4 @@ Here are a few of the plugins written by the beets community: .. _beets-usertag: https://github.com/igordertigor/beets-usertag .. _beets-popularity: https://github.com/abba23/beets-popularity .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl +.. _beet-summarize: https://github.com/steven-murray/beet-summarize From 81b1faa053867be53f9b3f72fbc148d11ca67b07 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 1 Jun 2019 12:28:47 -0400 Subject: [PATCH 304/339] inline: Fix a ridiculously subtle flexattr bug As detailed here: https://github.com/beetbox/beets/issues/2406#issuecomment-274423601 In a *function-style* definition, we didn't properly *un-define* the values for a given item after each function invocation. So when a field wasn't defined, it would get the value for the previously-formatted object instead. It now properly throws a NameError. --- beetsplug/inline.py | 4 ++++ docs/changelog.rst | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/beetsplug/inline.py b/beetsplug/inline.py index fd0e9fc30..bf6ff92da 100644 --- a/beetsplug/inline.py +++ b/beetsplug/inline.py @@ -117,9 +117,13 @@ class InlinePlugin(BeetsPlugin): # For function bodies, invoke the function with values as global # variables. def _func_func(obj): + old_globals = dict(func.__globals__) func.__globals__.update(_dict_for(obj)) try: return func() except Exception as exc: raise InlineError(python_code, exc) + finally: + func.__globals__.clear() + func.__globals__.update(old_globals) return _func_func diff --git a/docs/changelog.rst b/docs/changelog.rst index 371acb8c8..7a74f556b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,15 @@ New features: Thanks to :user:`dosoe`. :bug:`2580` :bug:`3272` +Fixes: + +* :doc:`/plugins/inline`: In function-style field definitions that refer to + flexible attributes, values could stick around from one function invocation + to the next. This meant that, when displaying a list of objects, later + objects could seem to reuse values from earlier objects when they were + missing a value for a given field. These values are now properly undefined. + :bug:`2406` + For plugin developers: * `MediaFile`_ has been split into a standalone project. Where you used to do From 001183ec545b40c07cca47de775ce46d031b8859 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 16:01:10 +1000 Subject: [PATCH 305/339] docs: add cron import recipe --- docs/guides/advanced.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index ba7ca12b6..3bee05261 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -151,3 +151,31 @@ differently. Put something like this in your configuration file:: Used together, flexible attributes and path format conditions let you sort your music by any criteria you can imagine. + + +Automatically add new music to your library +------------------------------------------- + +As a command-line tool, beets is perfect for automated operation via a cron job +or the like. To use it this way, you might want to use these options in your +`config file`_:: + + [beets] + import_incremental: yes + import_quiet: yes + import_log: /path/to/log.txt + +The ``import_incremental`` option will skip importing any directories that have +been imported in the past. +``import_quiet`` avoids asking you any questions (since this will be run +automatically, no input is possible). +You might also want to use the ``import_quiet_fallback`` options to configure +what should happen when no near-perfect match is found -- this option depends +on your level of paranoia. +Finally, ``import_log`` will make beets record its decisions so you can come +back later and see what you need to handle manually. + +The last step is to set up cron or some other automation system to run +``beet import /path/to/incoming/music``. + +.. _config file: http://beets.readthedocs.org/page/reference/config.html From 8bea1cc7f55c5069eb709cd1d9676ca7b7eac50b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 16:07:07 +1000 Subject: [PATCH 306/339] docs: add report recipe --- docs/guides/advanced.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index 3bee05261..845e0f33b 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -179,3 +179,31 @@ The last step is to set up cron or some other automation system to run ``beet import /path/to/incoming/music``. .. _config file: http://beets.readthedocs.org/page/reference/config.html + + +Useful reports +-------------- + +Since beets has a quite powerful query tool, this list contains some useful and +powerful queries to run on your library. + +* See a list of all albums which have files which are 128 bit rate:: + + beet list bitrate:128000 + +* See a list of all albums with the tracks listed in order of bit rate:: + + beet ls -f '$bitrate $artist - $title' | sort -n + +* See a list of all albums with the tracks listed in order of sample rate:: + + beet ls -f '$samplerate $artist - $title' | sort -n + +* See a list of albums and their formats:: + + beet ls -f '$albumartist $album $format' | sort | uniq + + Note that ``beet ls --album -f '... $format'`` doesn't do what you want, + because there is no notion of an album format. + If an album's tracks exist in multiple formats, the album will appear in the + list once for each format. From 10d41a1cbbe0c27cd6c18465f7704ce4b5448bff Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 16:28:40 +1000 Subject: [PATCH 307/339] tests: avoid non-test classes named Test* When using pytest's test collector, any class with a name starting with Test is collected. If it notices that the class has an `__init__` member then it skips it with a warning since it's probably a false positive. This isn't a big deal, but we can avoid warnings like this: test/test_ui_importer.py:33 beets/test/test_ui_importer.py:33: PytestCollectionWarning: cannot collect test class 'TestTerminalImportSession' because it has a __init__ constructor class TestTerminalImportSession(TerminalImportSession): simply by renaming TestX to XFixture. --- test/helper.py | 14 ++-- test/test_dbcore.py | 174 +++++++++++++++++++-------------------- test/test_importer.py | 5 +- test/test_pipeline.py | 16 ++-- test/test_ui_importer.py | 10 +-- 5 files changed, 110 insertions(+), 109 deletions(-) diff --git a/test/helper.py b/test/helper.py index c6afb063e..0b6eba718 100644 --- a/test/helper.py +++ b/test/helper.py @@ -24,7 +24,7 @@ information or mock the environment. - The `generate_album_info` and `generate_track_info` functions return fixtures to be used when mocking the autotagger. -- The `TestImportSession` allows one to run importer code while +- The `ImportSessionFixture` allows one to run importer code while controlling the interactions through code. - The `TestHelper` class encapsulates various fixtures that can be set up. @@ -251,7 +251,7 @@ class TestHelper(object): """Create files to import and return corresponding session. Copies the specified number of files to a subdirectory of - `self.temp_dir` and creates a `TestImportSession` for this path. + `self.temp_dir` and creates a `ImportSessionFixture` for this path. """ import_dir = os.path.join(self.temp_dir, b'import') if not os.path.isdir(import_dir): @@ -294,8 +294,8 @@ class TestHelper(object): config['import']['autotag'] = False config['import']['resume'] = False - return TestImportSession(self.lib, loghandler=None, query=None, - paths=[import_dir]) + return ImportSessionFixture(self.lib, loghandler=None, query=None, + paths=[import_dir]) # Library fixtures methods @@ -501,11 +501,11 @@ class TestHelper(object): return path -class TestImportSession(importer.ImportSession): +class ImportSessionFixture(importer.ImportSession): """ImportSession that can be controlled programaticaly. >>> lib = Library(':memory:') - >>> importer = TestImportSession(lib, paths=['/path/to/import']) + >>> importer = ImportSessionFixture(lib, paths=['/path/to/import']) >>> importer.add_choice(importer.action.SKIP) >>> importer.add_choice(importer.action.ASIS) >>> importer.default_choice = importer.action.APPLY @@ -517,7 +517,7 @@ class TestImportSession(importer.ImportSession): """ def __init__(self, *args, **kwargs): - super(TestImportSession, self).__init__(*args, **kwargs) + super(ImportSessionFixture, self).__init__(*args, **kwargs) self._choices = [] self._resolutions = [] diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 34994e3b3..9bf78de67 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -32,11 +32,11 @@ import six # Fixture: concrete database and model classes. For migration tests, we # have multiple models with different numbers of fields. -class TestSort(dbcore.query.FieldSort): +class SortFixture(dbcore.query.FieldSort): pass -class TestQuery(dbcore.query.Query): +class QueryFixture(dbcore.query.Query): def __init__(self, pattern): self.pattern = pattern @@ -47,7 +47,7 @@ class TestQuery(dbcore.query.Query): return True -class TestModel1(dbcore.Model): +class ModelFixture1(dbcore.Model): _table = 'test' _flex_table = 'testflex' _fields = { @@ -58,10 +58,10 @@ class TestModel1(dbcore.Model): 'some_float_field': dbcore.types.FLOAT, } _sorts = { - 'some_sort': TestSort, + 'some_sort': SortFixture, } _queries = { - 'some_query': TestQuery, + 'some_query': QueryFixture, } @classmethod @@ -72,12 +72,12 @@ class TestModel1(dbcore.Model): return {} -class TestDatabase1(dbcore.Database): - _models = (TestModel1,) +class DatabaseFixture1(dbcore.Database): + _models = (ModelFixture1,) pass -class TestModel2(TestModel1): +class ModelFixture2(ModelFixture1): _fields = { 'id': dbcore.types.PRIMARY_ID, 'field_one': dbcore.types.INTEGER, @@ -85,12 +85,12 @@ class TestModel2(TestModel1): } -class TestDatabase2(dbcore.Database): - _models = (TestModel2,) +class DatabaseFixture2(dbcore.Database): + _models = (ModelFixture2,) pass -class TestModel3(TestModel1): +class ModelFixture3(ModelFixture1): _fields = { 'id': dbcore.types.PRIMARY_ID, 'field_one': dbcore.types.INTEGER, @@ -99,12 +99,12 @@ class TestModel3(TestModel1): } -class TestDatabase3(dbcore.Database): - _models = (TestModel3,) +class DatabaseFixture3(dbcore.Database): + _models = (ModelFixture3,) pass -class TestModel4(TestModel1): +class ModelFixture4(ModelFixture1): _fields = { 'id': dbcore.types.PRIMARY_ID, 'field_one': dbcore.types.INTEGER, @@ -114,12 +114,12 @@ class TestModel4(TestModel1): } -class TestDatabase4(dbcore.Database): - _models = (TestModel4,) +class DatabaseFixture4(dbcore.Database): + _models = (ModelFixture4,) pass -class AnotherTestModel(TestModel1): +class AnotherModelFixture(ModelFixture1): _table = 'another' _flex_table = 'anotherflex' _fields = { @@ -128,7 +128,7 @@ class AnotherTestModel(TestModel1): } -class TestModel5(TestModel1): +class ModelFixture5(ModelFixture1): _fields = { 'some_string_field': dbcore.types.STRING, 'some_float_field': dbcore.types.FLOAT, @@ -136,17 +136,17 @@ class TestModel5(TestModel1): } -class TestDatabase5(dbcore.Database): - _models = (TestModel5,) +class DatabaseFixture5(dbcore.Database): + _models = (ModelFixture5,) pass -class TestDatabaseTwoModels(dbcore.Database): - _models = (TestModel2, AnotherTestModel) +class DatabaseFixtureTwoModels(dbcore.Database): + _models = (ModelFixture2, AnotherModelFixture) pass -class TestModelWithGetters(dbcore.Model): +class ModelFixtureWithGetters(dbcore.Model): @classmethod def _getters(cls): @@ -167,7 +167,7 @@ class MigrationTest(unittest.TestCase): handle, cls.orig_libfile = mkstemp('orig_db') os.close(handle) # Set up a database with the two-field schema. - old_lib = TestDatabase2(cls.orig_libfile) + old_lib = DatabaseFixture2(cls.orig_libfile) # Add an item to the old library. old_lib._connection().execute( @@ -189,35 +189,35 @@ class MigrationTest(unittest.TestCase): os.remove(self.libfile) def test_open_with_same_fields_leaves_untouched(self): - new_lib = TestDatabase2(self.libfile) + new_lib = DatabaseFixture2(self.libfile) c = new_lib._connection().cursor() c.execute("select * from test") row = c.fetchone() - self.assertEqual(len(row.keys()), len(TestModel2._fields)) + self.assertEqual(len(row.keys()), len(ModelFixture2._fields)) def test_open_with_new_field_adds_column(self): - new_lib = TestDatabase3(self.libfile) + new_lib = DatabaseFixture3(self.libfile) c = new_lib._connection().cursor() c.execute("select * from test") row = c.fetchone() - self.assertEqual(len(row.keys()), len(TestModel3._fields)) + self.assertEqual(len(row.keys()), len(ModelFixture3._fields)) def test_open_with_fewer_fields_leaves_untouched(self): - new_lib = TestDatabase1(self.libfile) + new_lib = DatabaseFixture1(self.libfile) c = new_lib._connection().cursor() c.execute("select * from test") row = c.fetchone() - self.assertEqual(len(row.keys()), len(TestModel2._fields)) + self.assertEqual(len(row.keys()), len(ModelFixture2._fields)) def test_open_with_multiple_new_fields(self): - new_lib = TestDatabase4(self.libfile) + new_lib = DatabaseFixture4(self.libfile) c = new_lib._connection().cursor() c.execute("select * from test") row = c.fetchone() - self.assertEqual(len(row.keys()), len(TestModel4._fields)) + self.assertEqual(len(row.keys()), len(ModelFixture4._fields)) def test_extra_model_adds_table(self): - new_lib = TestDatabaseTwoModels(self.libfile) + new_lib = DatabaseFixtureTwoModels(self.libfile) try: new_lib._connection().execute("select * from another") except sqlite3.OperationalError: @@ -226,19 +226,19 @@ class MigrationTest(unittest.TestCase): class ModelTest(unittest.TestCase): def setUp(self): - self.db = TestDatabase1(':memory:') + self.db = DatabaseFixture1(':memory:') def tearDown(self): self.db._connection().close() def test_add_model(self): - model = TestModel1() + model = ModelFixture1() model.add(self.db) rows = self.db._connection().execute('select * from test').fetchall() self.assertEqual(len(rows), 1) def test_store_fixed_field(self): - model = TestModel1() + model = ModelFixture1() model.add(self.db) model.field_one = 123 model.store() @@ -246,54 +246,54 @@ class ModelTest(unittest.TestCase): self.assertEqual(row['field_one'], 123) def test_retrieve_by_id(self): - model = TestModel1() + model = ModelFixture1() model.add(self.db) - other_model = self.db._get(TestModel1, model.id) + other_model = self.db._get(ModelFixture1, model.id) self.assertEqual(model.id, other_model.id) def test_store_and_retrieve_flexattr(self): - model = TestModel1() + model = ModelFixture1() model.add(self.db) model.foo = 'bar' model.store() - other_model = self.db._get(TestModel1, model.id) + other_model = self.db._get(ModelFixture1, model.id) self.assertEqual(other_model.foo, 'bar') def test_delete_flexattr(self): - model = TestModel1() + model = ModelFixture1() model['foo'] = 'bar' self.assertTrue('foo' in model) del model['foo'] self.assertFalse('foo' in model) def test_delete_flexattr_via_dot(self): - model = TestModel1() + model = ModelFixture1() model['foo'] = 'bar' self.assertTrue('foo' in model) del model.foo self.assertFalse('foo' in model) def test_delete_flexattr_persists(self): - model = TestModel1() + model = ModelFixture1() model.add(self.db) model.foo = 'bar' model.store() - model = self.db._get(TestModel1, model.id) + model = self.db._get(ModelFixture1, model.id) del model['foo'] model.store() - model = self.db._get(TestModel1, model.id) + model = self.db._get(ModelFixture1, model.id) self.assertFalse('foo' in model) def test_delete_non_existent_attribute(self): - model = TestModel1() + model = ModelFixture1() with self.assertRaises(KeyError): del model['foo'] def test_delete_fixed_attribute(self): - model = TestModel5() + model = ModelFixture5() model.some_string_field = 'foo' model.some_float_field = 1.23 model.some_boolean_field = True @@ -306,26 +306,26 @@ class ModelTest(unittest.TestCase): self.assertEqual(model[field], type_.null) def test_null_value_normalization_by_type(self): - model = TestModel1() + model = ModelFixture1() model.field_one = None self.assertEqual(model.field_one, 0) def test_null_value_stays_none_for_untyped_field(self): - model = TestModel1() + model = ModelFixture1() model.foo = None self.assertEqual(model.foo, None) def test_normalization_for_typed_flex_fields(self): - model = TestModel1() + model = ModelFixture1() model.some_float_field = None self.assertEqual(model.some_float_field, 0.0) def test_load_deleted_flex_field(self): - model1 = TestModel1() + model1 = ModelFixture1() model1['flex_field'] = True model1.add(self.db) - model2 = self.db._get(TestModel1, model1.id) + model2 = self.db._get(ModelFixture1, model1.id) self.assertIn('flex_field', model2) del model1['flex_field'] @@ -338,22 +338,22 @@ class ModelTest(unittest.TestCase): with assertRaisesRegex(self, ValueError, 'no database'): dbcore.Model()._check_db() with assertRaisesRegex(self, ValueError, 'no id'): - TestModel1(self.db)._check_db() + ModelFixture1(self.db)._check_db() dbcore.Model(self.db)._check_db(need_id=False) def test_missing_field(self): with self.assertRaises(AttributeError): - TestModel1(self.db).nonExistingKey + ModelFixture1(self.db).nonExistingKey def test_computed_field(self): - model = TestModelWithGetters() + model = ModelFixtureWithGetters() self.assertEqual(model.aComputedField, 'thing') with assertRaisesRegex(self, KeyError, u'computed field .+ deleted'): del model.aComputedField def test_items(self): - model = TestModel1(self.db) + model = ModelFixture1(self.db) model.id = 5 self.assertEqual({('id', 5), ('field_one', 0)}, set(model.items())) @@ -371,31 +371,31 @@ class ModelTest(unittest.TestCase): class FormatTest(unittest.TestCase): def test_format_fixed_field(self): - model = TestModel1() + model = ModelFixture1() model.field_one = u'caf\xe9' value = model.formatted().get('field_one') self.assertEqual(value, u'caf\xe9') def test_format_flex_field(self): - model = TestModel1() + model = ModelFixture1() model.other_field = u'caf\xe9' value = model.formatted().get('other_field') self.assertEqual(value, u'caf\xe9') def test_format_flex_field_bytes(self): - model = TestModel1() + model = ModelFixture1() model.other_field = u'caf\xe9'.encode('utf-8') value = model.formatted().get('other_field') self.assertTrue(isinstance(value, six.text_type)) self.assertEqual(value, u'caf\xe9') def test_format_unset_field(self): - model = TestModel1() + model = ModelFixture1() value = model.formatted().get('other_field') self.assertEqual(value, u'') def test_format_typed_flex_field(self): - model = TestModel1() + model = ModelFixture1() model.some_float_field = 3.14159265358979 value = model.formatted().get('some_float_field') self.assertEqual(value, u'3.1') @@ -403,40 +403,40 @@ class FormatTest(unittest.TestCase): class FormattedMappingTest(unittest.TestCase): def test_keys_equal_model_keys(self): - model = TestModel1() + model = ModelFixture1() formatted = model.formatted() self.assertEqual(set(model.keys(True)), set(formatted.keys())) def test_get_unset_field(self): - model = TestModel1() + model = ModelFixture1() formatted = model.formatted() with self.assertRaises(KeyError): formatted['other_field'] def test_get_method_with_default(self): - model = TestModel1() + model = ModelFixture1() formatted = model.formatted() self.assertEqual(formatted.get('other_field'), u'') def test_get_method_with_specified_default(self): - model = TestModel1() + model = ModelFixture1() formatted = model.formatted() self.assertEqual(formatted.get('other_field', 'default'), 'default') class ParseTest(unittest.TestCase): def test_parse_fixed_field(self): - value = TestModel1._parse('field_one', u'2') + value = ModelFixture1._parse('field_one', u'2') self.assertIsInstance(value, int) self.assertEqual(value, 2) def test_parse_flex_field(self): - value = TestModel1._parse('some_float_field', u'2') + value = ModelFixture1._parse('some_float_field', u'2') self.assertIsInstance(value, float) self.assertEqual(value, 2.0) def test_parse_untyped_field(self): - value = TestModel1._parse('field_nine', u'2') + value = ModelFixture1._parse('field_nine', u'2') self.assertEqual(value, u'2') @@ -503,7 +503,7 @@ class QueryFromStringsTest(unittest.TestCase): def qfs(self, strings): return dbcore.queryparse.query_from_strings( dbcore.query.AndQuery, - TestModel1, + ModelFixture1, {':': dbcore.query.RegexpQuery}, strings, ) @@ -535,13 +535,13 @@ class QueryFromStringsTest(unittest.TestCase): def test_parse_named_query(self): q = self.qfs(['some_query:foo']) - self.assertIsInstance(q.subqueries[0], TestQuery) + self.assertIsInstance(q.subqueries[0], QueryFixture) class SortFromStringsTest(unittest.TestCase): def sfs(self, strings): return dbcore.queryparse.sort_from_strings( - TestModel1, + ModelFixture1, strings, ) @@ -571,13 +571,13 @@ class SortFromStringsTest(unittest.TestCase): def test_special_sort(self): s = self.sfs(['some_sort+']) - self.assertIsInstance(s, TestSort) + self.assertIsInstance(s, SortFixture) class ParseSortedQueryTest(unittest.TestCase): def psq(self, parts): return dbcore.parse_sorted_query( - TestModel1, + ModelFixture1, parts.split(), ) @@ -626,11 +626,11 @@ class ParseSortedQueryTest(unittest.TestCase): class ResultsIteratorTest(unittest.TestCase): def setUp(self): - self.db = TestDatabase1(':memory:') - model = TestModel1() + self.db = DatabaseFixture1(':memory:') + model = ModelFixture1() model['foo'] = 'baz' model.add(self.db) - model = TestModel1() + model = ModelFixture1() model['foo'] = 'bar' model.add(self.db) @@ -638,16 +638,16 @@ class ResultsIteratorTest(unittest.TestCase): self.db._connection().close() def test_iterate_once(self): - objs = self.db._fetch(TestModel1) + objs = self.db._fetch(ModelFixture1) self.assertEqual(len(list(objs)), 2) def test_iterate_twice(self): - objs = self.db._fetch(TestModel1) + objs = self.db._fetch(ModelFixture1) list(objs) self.assertEqual(len(list(objs)), 2) def test_concurrent_iterators(self): - results = self.db._fetch(TestModel1) + results = self.db._fetch(ModelFixture1) it1 = iter(results) it2 = iter(results) next(it1) @@ -656,44 +656,44 @@ class ResultsIteratorTest(unittest.TestCase): def test_slow_query(self): q = dbcore.query.SubstringQuery('foo', 'ba', False) - objs = self.db._fetch(TestModel1, q) + objs = self.db._fetch(ModelFixture1, q) self.assertEqual(len(list(objs)), 2) def test_slow_query_negative(self): q = dbcore.query.SubstringQuery('foo', 'qux', False) - objs = self.db._fetch(TestModel1, q) + objs = self.db._fetch(ModelFixture1, q) self.assertEqual(len(list(objs)), 0) def test_iterate_slow_sort(self): s = dbcore.query.SlowFieldSort('foo') - res = self.db._fetch(TestModel1, sort=s) + res = self.db._fetch(ModelFixture1, sort=s) objs = list(res) self.assertEqual(objs[0].foo, 'bar') self.assertEqual(objs[1].foo, 'baz') def test_unsorted_subscript(self): - objs = self.db._fetch(TestModel1) + objs = self.db._fetch(ModelFixture1) self.assertEqual(objs[0].foo, 'baz') self.assertEqual(objs[1].foo, 'bar') def test_slow_sort_subscript(self): s = dbcore.query.SlowFieldSort('foo') - objs = self.db._fetch(TestModel1, sort=s) + objs = self.db._fetch(ModelFixture1, sort=s) self.assertEqual(objs[0].foo, 'bar') self.assertEqual(objs[1].foo, 'baz') def test_length(self): - objs = self.db._fetch(TestModel1) + objs = self.db._fetch(ModelFixture1) self.assertEqual(len(objs), 2) def test_out_of_range(self): - objs = self.db._fetch(TestModel1) + objs = self.db._fetch(ModelFixture1) with self.assertRaises(IndexError): objs[100] def test_no_results(self): self.assertIsNone(self.db._fetch( - TestModel1, dbcore.query.FalseQuery()).get()) + ModelFixture1, dbcore.query.FalseQuery()).get()) def suite(): diff --git a/test/test_importer.py b/test/test_importer.py index 980a769f6..8f637a077 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -32,7 +32,8 @@ import unittest from test import _common from beets.util import displayable_path, bytestring_path, py3_path -from test.helper import TestImportSession, TestHelper, has_program, capture_log +from test.helper import TestHelper, has_program, capture_log +from test.helper import ImportSessionFixture from beets import importer from beets.importer import albums_in_dir from mediafile import MediaFile @@ -223,7 +224,7 @@ class ImportHelper(TestHelper): config['import']['link'] = link config['import']['hardlink'] = hardlink - self.importer = TestImportSession( + self.importer = ImportSessionFixture( self.lib, loghandler=None, query=None, paths=[import_dir or self.import_dir] ) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 228deb50e..82f155521 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -43,7 +43,7 @@ def _consume(l): # A worker that raises an exception. -class TestException(Exception): +class ExceptionFixture(Exception): pass @@ -52,7 +52,7 @@ def _exc_work(num=3): while True: i = yield i if i == num: - raise TestException() + raise ExceptionFixture() i *= 2 @@ -126,10 +126,10 @@ class ExceptionTest(unittest.TestCase): _consume(self.l))) def test_run_sequential(self): - self.assertRaises(TestException, self.pl.run_sequential) + self.assertRaises(ExceptionFixture, self.pl.run_sequential) def test_run_parallel(self): - self.assertRaises(TestException, self.pl.run_parallel) + self.assertRaises(ExceptionFixture, self.pl.run_parallel) def test_pull(self): pl = pipeline.Pipeline((_produce(), _exc_work())) @@ -137,9 +137,9 @@ class ExceptionTest(unittest.TestCase): for i in range(3): next(pull) if six.PY2: - self.assertRaises(TestException, pull.next) + self.assertRaises(ExceptionFixture, pull.next) else: - self.assertRaises(TestException, pull.__next__) + self.assertRaises(ExceptionFixture, pull.__next__) class ParallelExceptionTest(unittest.TestCase): @@ -150,7 +150,7 @@ class ParallelExceptionTest(unittest.TestCase): )) def test_run_parallel(self): - self.assertRaises(TestException, self.pl.run_parallel) + self.assertRaises(ExceptionFixture, self.pl.run_parallel) class ConstrainedThreadedPipelineTest(unittest.TestCase): @@ -166,7 +166,7 @@ class ConstrainedThreadedPipelineTest(unittest.TestCase): # Raise an exception in a constrained pipeline. l = [] pl = pipeline.Pipeline((_produce(1000), _exc_work(), _consume(l))) - self.assertRaises(TestException, pl.run_parallel, 1) + self.assertRaises(ExceptionFixture, pl.run_parallel, 1) def test_constrained_parallel(self): l = [] diff --git a/test/test_ui_importer.py b/test/test_ui_importer.py index 48a66dc5f..229dac5d2 100644 --- a/test/test_ui_importer.py +++ b/test/test_ui_importer.py @@ -30,11 +30,11 @@ from beets import config import six -class TestTerminalImportSession(TerminalImportSession): +class TerminalImportSessionFixture(TerminalImportSession): def __init__(self, *args, **kwargs): self.io = kwargs.pop('io') - super(TestTerminalImportSession, self).__init__(*args, **kwargs) + super(TerminalImportSessionFixture, self).__init__(*args, **kwargs) self._choices = [] default_choice = importer.action.APPLY @@ -47,11 +47,11 @@ class TestTerminalImportSession(TerminalImportSession): def choose_match(self, task): self._add_choice_input() - return super(TestTerminalImportSession, self).choose_match(task) + return super(TerminalImportSessionFixture, self).choose_match(task) def choose_item(self, task): self._add_choice_input() - return super(TestTerminalImportSession, self).choose_item(task) + return super(TerminalImportSessionFixture, self).choose_item(task) def _add_choice_input(self): try: @@ -96,7 +96,7 @@ class TerminalImportSessionSetup(object): if not hasattr(self, 'io'): self.io = DummyIO() self.io.install() - self.importer = TestTerminalImportSession( + self.importer = TerminalImportSessionFixture( self.lib, loghandler=None, query=None, io=self.io, paths=[import_dir or self.import_dir], ) From b0d476ae78b2610d46207b01fd8ed87d25c1f501 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 21:30:57 +1000 Subject: [PATCH 308/339] Retry dependency installation commands on AppVeyor There's an `appveyor-retry` command that should help to make some of the dependency installation more reliable and the AppVeyor tests less flaky. --- appveyor.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9a3102ba8..00a3eb189 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,11 +21,11 @@ environment: # Install Tox for running tests. install: - - cinst imagemagick -y + - appveyor-retry cinst imagemagick -y # TODO: remove --allow-empty-checksums when unrar offers a proper checksum - - cinst unrar -y --allow-empty-checksums - - '%PYTHON%/Scripts/pip.exe install "tox<=3.8.1"' - - "%PYTHON%/Scripts/tox.exe -e %TOX_ENV% --notest" + - appveyor-retry cinst unrar -y --allow-empty-checksums + - 'appveyor-retry %PYTHON%/Scripts/pip.exe install "tox<=3.8.1"' + - "appveyor-retry %PYTHON%/Scripts/tox.exe -e %TOX_ENV% --notest" test_script: - "%PYTHON%/Scripts/tox.exe -e %TOX_ENV%" From aa9e39327d38fa9ec4f573f8eb1610d5c3fa62ef Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 22:25:12 +1000 Subject: [PATCH 309/339] docs: document 'quiet' config item --- docs/reference/config.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 4d3d6fff1..a96e3dfb3 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -522,6 +522,17 @@ Either ``yes`` or ``no`` (default), controlling whether existing metadata is discarded when a match is applied. This corresponds to the ``--from_scratch`` flag to ``beet import``. +.. _quiet: + +quiet +~~~~~ + +Either ``yes`` or ``no`` (default), controlling whether to ask for a manual +decision from the user when the importer is unsure how to proceed. This +corresponds to the ``--quiet`` flag to ``beet import``. + +.. _quiet_fallback: + quiet_fallback ~~~~~~~~~~~~~~ From 176fac781f09d37d8f11eb243122833f0f2c67ad Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 22:24:49 +1000 Subject: [PATCH 310/339] docs: update cookbook for modern beets --- docs/guides/advanced.rst | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index 845e0f33b..38cc31d0c 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -158,28 +158,28 @@ Automatically add new music to your library As a command-line tool, beets is perfect for automated operation via a cron job or the like. To use it this way, you might want to use these options in your -`config file`_:: +:doc:`config file `: - [beets] - import_incremental: yes - import_quiet: yes - import_log: /path/to/log.txt +.. code-block:: yaml -The ``import_incremental`` option will skip importing any directories that have + import: + incremental: yes + quiet: yes + log: /path/to/log.txt + +The :ref:`incremental` option will skip importing any directories that have been imported in the past. -``import_quiet`` avoids asking you any questions (since this will be run +:ref:`quiet` avoids asking you any questions (since this will be run automatically, no input is possible). -You might also want to use the ``import_quiet_fallback`` options to configure +You might also want to use the :ref:`quiet_fallback` options to configure what should happen when no near-perfect match is found -- this option depends on your level of paranoia. -Finally, ``import_log`` will make beets record its decisions so you can come +Finally, :ref:`import_log` will make beets record its decisions so you can come back later and see what you need to handle manually. The last step is to set up cron or some other automation system to run ``beet import /path/to/incoming/music``. -.. _config file: http://beets.readthedocs.org/page/reference/config.html - Useful reports -------------- @@ -193,17 +193,13 @@ powerful queries to run on your library. * See a list of all albums with the tracks listed in order of bit rate:: - beet ls -f '$bitrate $artist - $title' | sort -n - -* See a list of all albums with the tracks listed in order of sample rate:: - - beet ls -f '$samplerate $artist - $title' | sort -n + beet ls -f '$bitrate $artist - $title' bitrate+ * See a list of albums and their formats:: beet ls -f '$albumartist $album $format' | sort | uniq Note that ``beet ls --album -f '... $format'`` doesn't do what you want, - because there is no notion of an album format. + because ``format`` is an item-level field, not an album-level one. If an album's tracks exist in multiple formats, the album will appear in the list once for each format. From 5b7b5061e7716378139fb7c6210c5e88cf05c36a Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 23:16:30 +1000 Subject: [PATCH 311/339] Avoid using an internal confuse function --- beets/ui/commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 3f540a1fa..ef4fd144a 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -39,7 +39,6 @@ from beets.util import syspath, normpath, ancestry, displayable_path, \ from beets import library from beets import config from beets import logging -from confuse import _package_path import six from . import _store_dict @@ -1726,7 +1725,7 @@ def completion_script(commands): ``commands`` is alist of ``ui.Subcommand`` instances to generate completion data for. """ - base_script = os.path.join(_package_path('beets.ui'), 'completion_base.sh') + base_script = os.path.join(os.path.dirname(__file__), 'completion_base.sh') with open(base_script, 'r') as base_script: yield util.text_string(base_script.read()) From 80286ea89871ef0313230055c6c80e945b757cac Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 14:18:31 +1000 Subject: [PATCH 312/339] bluelet: catch ECONNRESET When ncmpcpp quits after an error it causes a "connection reset by peer" exception, also known as ECONNRESET (104) in errno terms. In Python 2 this is mapped to a `socket.error` and in Python 3 this is `ConnectionResetError` which is thankfully a subclass of the `socket.error` exception class. --- beets/util/bluelet.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/util/bluelet.py b/beets/util/bluelet.py index 0da17559b..dcc80e041 100644 --- a/beets/util/bluelet.py +++ b/beets/util/bluelet.py @@ -346,6 +346,10 @@ def run(root_coro): exc.args[0] == errno.EPIPE: # Broken pipe. Remote host disconnected. pass + elif isinstance(exc.args, tuple) and \ + exc.args[0] == errno.ECONNRESET: + # Connection was reset by peer. + pass else: traceback.print_exc() # Abort the coroutine. From 64ed54330bad3d9dc5978f793e2a03ffb0338b1a Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 12 Apr 2019 12:23:07 +1000 Subject: [PATCH 313/339] bpd: mention control socket address in log --- beetsplug/bpd/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index a4a987a55..1087f8037 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1094,6 +1094,8 @@ class Server(BaseServer): self.cmd_update(None) log.info(u'Server ready and listening on {}:{}'.format( host, port)) + log.debug(u'Listening for control signals on {}:{}'.format( + host, ctrl_port)) def run(self): self.player.run() From 59c506990a386fc1ec61e30b6f7e0d41aaf3f92b Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 14:16:43 +1000 Subject: [PATCH 314/339] bpd: fix bug in playlistid The playlistid command is supposed to list the whole playlist if no argument is provided, but we were accidentally trying to look up an impossible negative id in that case causing an error to always be returned. --- beetsplug/bpd/__init__.py | 4 +++- test/test_player.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 1087f8037..5bf35e1d5 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -593,7 +593,9 @@ class BaseServer(object): yield self._item_info(track) def cmd_playlistid(self, conn, track_id=-1): - return self.cmd_playlistinfo(conn, self._id_to_index(track_id)) + if track_id != -1: + track_id = self._id_to_index(track_id) + return self.cmd_playlistinfo(conn, track_id) def cmd_plchanges(self, conn, version): """Sends playlist changes since the given version. diff --git a/test/test_player.py b/test/test_player.py index 66f1f5d38..bce1a0625 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -684,7 +684,7 @@ class BPDControlTest(BPDTestHelper): class BPDQueueTest(BPDTestHelper): test_implements_queue = implements({ 'addid', 'clear', 'delete', 'deleteid', 'move', - 'moveid', 'playlist', 'playlistfind', 'playlistid', + 'moveid', 'playlist', 'playlistfind', 'playlistsearch', 'plchanges', 'plchangesposid', 'prio', 'prioid', 'rangeid', 'shuffle', 'swap', 'swapid', 'addtagid', 'cleartagid', @@ -703,6 +703,16 @@ class BPDQueueTest(BPDTestHelper): ('playlistinfo', '200')) self._assert_failed(responses, bpd.ERROR_ARG, pos=2) + def test_cmd_playlistid(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('playlistid', '2'), + ('playlistid',)) + self._assert_ok(*responses) + self.assertEqual('Track Two Title', responses[0].data['Title']) + self.assertEqual(['1', '2'], responses[1].data['Track']) + class BPDPlaylistsTest(BPDTestHelper): test_implements_playlists = implements({'playlistadd'}) From 5c37a58ad6cf4c8f631184df86419077d376b6f4 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 14:18:03 +1000 Subject: [PATCH 315/339] bpd: add more tests --- test/test_player.py | 54 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index bce1a0625..b4ed03618 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -392,7 +392,7 @@ class BPDTest(BPDTestHelper): class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ - 'clearerror', 'currentsong', 'stats', + 'clearerror', 'currentsong', }) def test_cmd_status(self): @@ -414,6 +414,14 @@ class BPDQueryTest(BPDTestHelper): } self.assertEqual(fields_playing, set(responses[2].data.keys())) + def test_cmd_stats(self): + with self.run_bpd() as client: + response = client.send_command('stats') + self._assert_ok(response) + details = {'artists', 'albums', 'songs', 'uptime', 'db_playtime', + 'db_update', 'playtime'} + self.assertEqual(details, set(response.data.keys())) + def test_cmd_idle(self): def _toggle(c): for _ in range(3): @@ -630,9 +638,8 @@ class BPDPlaybackTest(BPDTestHelper): class BPDControlTest(BPDTestHelper): test_implements_control = implements({ - 'pause', 'playid', 'seek', - 'seekid', 'seekcur', 'stop', - }, expectedFailure=True) + 'seek', 'seekid', 'seekcur', + }, expectedFailure=True) def test_cmd_play(self): with self.run_bpd() as client: @@ -648,6 +655,45 @@ class BPDControlTest(BPDTestHelper): self.assertEqual('play', responses[2].data['state']) self.assertEqual('2', responses[4].data['Id']) + def test_cmd_playid(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1, self.item2) + responses = client.send_commands( + ('playid', '2'), + ('currentsong',), + ('clear',)) + self._bpd_add(client, self.item2, self.item1) + responses.extend(client.send_commands( + ('playid', '2'), + ('currentsong',))) + self._assert_ok(*responses) + self.assertEqual('2', responses[1].data['Id']) + self.assertEqual('2', responses[4].data['Id']) + + def test_cmd_pause(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1) + responses = client.send_commands( + ('play',), + ('pause',), + ('status',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('pause', responses[2].data['state']) + self.assertEqual('1', responses[3].data['Id']) + + def test_cmd_stop(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1) + responses = client.send_commands( + ('play',), + ('stop',), + ('status',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('stop', responses[2].data['state']) + self.assertNotIn('Id', responses[3].data) + def test_cmd_next(self): with self.run_bpd() as client: self._bpd_add(client, self.item1, self.item2) From 1a5263b68ff193941ec8d7af941fa8a94ad7e5a4 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 15 Apr 2019 14:19:09 +1000 Subject: [PATCH 316/339] bpd: support volume command for real --- beetsplug/bpd/__init__.py | 3 ++- test/test_player.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 5bf35e1d5..05bde6d33 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -454,7 +454,8 @@ class BaseServer(object): def cmd_volume(self, conn, vol_delta): """Deprecated command to change the volume by a relative amount.""" - raise BPDError(ERROR_SYSTEM, u'No mixer') + vol_delta = cast_arg(int, vol_delta) + return self.cmd_setvol(conn, self.volume + vol_delta) def cmd_crossfade(self, conn, crossfade): """Set the number of seconds of crossfading.""" diff --git a/test/test_player.py b/test/test_player.py index b4ed03618..048f2cedd 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -623,8 +623,13 @@ class BPDPlaybackTest(BPDTestHelper): def test_cmd_volume(self): with self.run_bpd() as client: - response = client.send_command('volume', '10') - self._assert_failed(response, bpd.ERROR_SYSTEM) + responses = client.send_commands( + ('setvol', '10'), + ('volume', '5'), + ('volume', '-2'), + ('status',)) + self._assert_ok(*responses) + self.assertEqual('13', responses[3].data['volume']) def test_cmd_replay_gain(self): with self.run_bpd() as client: From e708d28f850893f1aa529d23a2172d30f6dee3be Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 19 Apr 2019 12:59:46 +1000 Subject: [PATCH 317/339] bpd: allow fractional seconds in seek The documented type is float, not integer, and clients like mpDris2 send fractional seconds, causing them to crash if these values ar enot accepted. --- beetsplug/bpd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 05bde6d33..7c40cb6b6 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1548,7 +1548,7 @@ class Server(BaseServer): def cmd_seek(self, conn, index, pos): """Seeks to the specified position in the specified song.""" index = cast_arg(int, index) - pos = cast_arg(int, pos) + pos = cast_arg(float, pos) super(Server, self).cmd_seek(conn, index, pos) self.player.seek(pos) From 27c462d287a7c4788a2f3aab6fe9410ea19098c2 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Mon, 15 Apr 2019 15:08:47 +1000 Subject: [PATCH 318/339] bpd: make noidle a no-op outside idle mode The real MPD ignores `noidle` when the client is not idle. It doesn't even send a successful response, just ignores the command. Although I don't understand why a client would fail to keep track of its own state, it seems that this is necessary to get ncmpcpp working. --- beetsplug/bpd/__init__.py | 3 +++ test/test_player.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 7c40cb6b6..97d89d914 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -847,6 +847,9 @@ class MPDConnection(Connection): yield self.send(err.response()) break continue + if line == u'noidle': + # When not in idle, this command sends no response. + continue if clist is not None: # Command list already opened. diff --git a/test/test_player.py b/test/test_player.py index 048f2cedd..f22c19261 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -457,6 +457,14 @@ class BPDQueryTest(BPDTestHelper): response = client.send_command('noidle') self._assert_ok(response) + def test_cmd_noidle_when_not_idle(self): + with self.run_bpd() as client: + # Manually send a command without reading a response. + request = client.serialise_command('noidle') + client.sock.sendall(request) + response = client.send_command('notacommand') + self._assert_failed(response, bpd.ERROR_UNKNOWN) + class BPDPlaybackTest(BPDTestHelper): test_implements_playback = implements({ From 5187100294a9b36581d358c755ff39051c956fc6 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 16:23:42 +1000 Subject: [PATCH 319/339] bpd: accept all idle events --- beetsplug/bpd/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 97d89d914..487999cc7 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -77,8 +77,8 @@ SAFE_COMMANDS = ( SUBSYSTEMS = [ u'update', u'player', u'mixer', u'options', u'playlist', u'database', # Related to unsupported commands: - # u'stored_playlist', u'output', u'subscription', u'sticker', u'message', - # u'partition', + u'stored_playlist', u'output', u'subscription', u'sticker', u'message', + u'partition', ] ITEM_KEYS_WRITABLE = set(MediaFile.fields()).intersection(Item._fields.keys()) From cc2c35221d28b9be80eb4b1be5c40fc27e0cfd5c Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 25 Apr 2019 11:41:00 +1000 Subject: [PATCH 320/339] bpd: avoid sending playlist events on navigation --- beetsplug/bpd/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 487999cc7..dca3310d8 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -627,7 +627,6 @@ class BaseServer(object): """Advance to the next song in the playlist.""" old_index = self.current_index self.current_index = self._succ_idx() - self._send_event('playlist') if self.consume: # TODO how does consume interact with single+repeat? self.playlist.pop(old_index) @@ -648,7 +647,6 @@ class BaseServer(object): """Step back to the last song.""" old_index = self.current_index self.current_index = self._prev_idx() - self._send_event('playlist') if self.consume: self.playlist.pop(old_index) if self.current_index < 0: From fdd809fd36def07d01b770e7ce04855a1cfb8f77 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Thu, 25 Apr 2019 13:08:26 +1000 Subject: [PATCH 321/339] bpd: support more tagtypes --- beetsplug/bpd/__init__.py | 29 ++++++++-------- docs/plugins/bpd.rst | 3 +- test/test_player.py | 70 +++++++++++++++++++++++++++++++-------- 3 files changed, 72 insertions(+), 30 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index dca3310d8..95598ff18 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -1117,19 +1117,10 @@ class Server(BaseServer): info_lines = [ u'file: ' + item.destination(fragment=True), u'Time: ' + six.text_type(int(item.length)), - u'Title: ' + item.title, - u'Artist: ' + item.artist, - u'Album: ' + item.album, - u'Genre: ' + item.genre, + u'duration: ' + u'{:.3f}'.format(item.length), + u'Id: ' + six.text_type(item.id), ] - track = six.text_type(item.track) - if item.tracktotal: - track += u'/' + six.text_type(item.tracktotal) - info_lines.append(u'Track: ' + track) - - info_lines.append(u'Date: ' + six.text_type(item.year)) - try: pos = self._id_to_index(item.id) info_lines.append(u'Pos: ' + six.text_type(pos)) @@ -1137,7 +1128,9 @@ class Server(BaseServer): # Don't include position if not in playlist. pass - info_lines.append(u'Id: ' + six.text_type(item.id)) + for tagtype, field in self.tagtype_map.items(): + info_lines.append(u'{}: {}'.format( + tagtype, six.text_type(getattr(item, field)))) return info_lines @@ -1341,18 +1334,24 @@ class Server(BaseServer): tagtype_map = { u'Artist': u'artist', + u'ArtistSort': u'artist_sort', u'Album': u'album', u'Title': u'title', u'Track': u'track', u'AlbumArtist': u'albumartist', u'AlbumArtistSort': u'albumartist_sort', - # Name? + u'Label': u'label', u'Genre': u'genre', u'Date': u'year', + u'OriginalDate': u'original_year', u'Composer': u'composer', - # Performer? u'Disc': u'disc', - u'filename': u'path', # Suspect. + u'Comment': u'comments', + u'MUSICBRAINZ_TRACKID': u'mb_trackid', + u'MUSICBRAINZ_ALBUMID': u'mb_albumid', + u'MUSICBRAINZ_ARTISTID': u'mb_artistid', + u'MUSICBRAINZ_ALBUMARTISTID': u'mb_albumartistid', + u'MUSICBRAINZ_RELEASETRACKID': u'mb_releasetrackid', } def cmd_tagtypes(self, conn): diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index 87c931793..8bfe456ad 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -125,8 +125,7 @@ These are some of the known differences between BPD and MPD: * Advanced playback features like cross-fade, ReplayGain and MixRamp are not supported due to BPD's simple audio player backend. * Advanced query syntax is not currently supported. -* Not all tags (fields) are currently exposed to BPD. Clients also can't use - the ``tagtypes`` mask to hide fields. +* Clients can't use the ``tagtypes`` mask to hide fields. * BPD's ``random`` mode is not deterministic and doesn't support priorities. * Mounts and streams are not supported. BPD can only play files from disk. * Stickers are not supported (although this is basically a flexattr in beets diff --git a/test/test_player.py b/test/test_player.py index f22c19261..874a2db52 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -392,8 +392,31 @@ class BPDTest(BPDTestHelper): class BPDQueryTest(BPDTestHelper): test_implements_query = implements({ - 'clearerror', 'currentsong', - }) + 'clearerror', + }) + + def test_cmd_currentsong(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1) + responses = client.send_commands( + ('play',), + ('currentsong',), + ('stop',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual('1', responses[1].data['Id']) + self.assertNotIn('Id', responses[3].data) + + def test_cmd_currentsong_tagtypes(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1) + responses = client.send_commands( + ('play',), + ('currentsong',)) + self._assert_ok(*responses) + self.assertEqual( + BPDConnectionTest.TAGTYPES.union(BPDQueueTest.METADATA), + set(responses[1].data.keys())) def test_cmd_status(self): with self.run_bpd() as client: @@ -749,6 +772,8 @@ class BPDQueueTest(BPDTestHelper): 'swap', 'swapid', 'addtagid', 'cleartagid', }, expectedFailure=True) + METADATA = {'Pos', 'Time', 'Id', 'file', 'duration'} + def test_cmd_add(self): with self.run_bpd() as client: self._bpd_add(client, self.item1) @@ -762,6 +787,15 @@ class BPDQueueTest(BPDTestHelper): ('playlistinfo', '200')) self._assert_failed(responses, bpd.ERROR_ARG, pos=2) + def test_cmd_playlistinfo_tagtypes(self): + with self.run_bpd() as client: + self._bpd_add(client, self.item1) + response = client.send_command('playlistinfo', '0') + self._assert_ok(response) + self.assertEqual( + BPDConnectionTest.TAGTYPES.union(BPDQueueTest.METADATA), + set(response.data.keys())) + def test_cmd_playlistid(self): with self.run_bpd() as client: self._bpd_add(client, self.item1, self.item2) @@ -900,8 +934,24 @@ class BPDStickerTest(BPDTestHelper): class BPDConnectionTest(BPDTestHelper): test_implements_connection = implements({ - 'close', 'kill', 'tagtypes', - }) + 'close', 'kill', + }) + + ALL_MPD_TAGTYPES = { + 'Artist', 'ArtistSort', 'Album', 'AlbumSort', 'AlbumArtist', + 'AlbumArtistSort', 'Title', 'Track', 'Name', 'Genre', 'Date', + 'Composer', 'Performer', 'Comment', 'Disc', 'Label', + 'OriginalDate', 'MUSICBRAINZ_ARTISTID', 'MUSICBRAINZ_ALBUMID', + 'MUSICBRAINZ_ALBUMARTISTID', 'MUSICBRAINZ_TRACKID', + 'MUSICBRAINZ_RELEASETRACKID', 'MUSICBRAINZ_WORKID', + } + UNSUPPORTED_TAGTYPES = { + 'MUSICBRAINZ_WORKID', # not tracked by beets + 'Performer', # not tracked by beets + 'AlbumSort', # not tracked by beets + 'Name', # junk field for internet radio + } + TAGTYPES = ALL_MPD_TAGTYPES.difference(UNSUPPORTED_TAGTYPES) def test_cmd_password(self): with self.run_bpd(password='abc123') as client: @@ -921,19 +971,13 @@ class BPDConnectionTest(BPDTestHelper): response = client.send_command('ping') self._assert_ok(response) - @unittest.skip def test_cmd_tagtypes(self): with self.run_bpd() as client: response = client.send_command('tagtypes') self._assert_ok(response) - self.assertEqual({ - 'Artist', 'ArtistSort', 'Album', 'AlbumSort', 'AlbumArtist', - 'AlbumArtistSort', 'Title', 'Track', 'Name', 'Genre', 'Date', - 'Composer', 'Performer', 'Comment', 'Disc', 'Label', - 'OriginalDate', 'MUSICBRAINZ_ARTISTID', 'MUSICBRAINZ_ALBUMID', - 'MUSICBRAINZ_ALBUMARTISTID', 'MUSICBRAINZ_TRACKID', - 'MUSICBRAINZ_RELEASETRACKID', 'MUSICBRAINZ_WORKID', - }, set(response.data['tag'])) + self.assertEqual( + self.TAGTYPES, + set(response.data['tagtype'])) @unittest.skip def test_tagtypes_mask(self): From dc7e3b9b6a124ff7b1df31683f00aecfc3ef15bd Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Fri, 12 Apr 2019 12:30:40 +1000 Subject: [PATCH 322/339] bpd: support nextsong in status --- beetsplug/bpd/__init__.py | 5 +++++ test/test_player.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 95598ff18..8d90b5cf6 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -412,6 +412,11 @@ class BaseServer(object): current_id = self._item_id(self.playlist[self.current_index]) yield u'song: ' + six.text_type(self.current_index) yield u'songid: ' + six.text_type(current_id) + if len(self.playlist) > self.current_index + 1: + # If there's a next song, report its index too. + next_id = self._item_id(self.playlist[self.current_index + 1]) + yield u'nextsong: ' + six.text_type(self.current_index + 1) + yield u'nextsongid: ' + six.text_type(next_id) if self.error: yield u'error: ' + self.error diff --git a/test/test_player.py b/test/test_player.py index 874a2db52..9bd75519f 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -433,7 +433,8 @@ class BPDQueryTest(BPDTestHelper): } self.assertEqual(fields_not_playing, set(responses[0].data.keys())) fields_playing = fields_not_playing | { - 'song', 'songid', 'time', 'elapsed', 'bitrate', 'duration', 'audio' + 'song', 'songid', 'time', 'elapsed', 'bitrate', 'duration', + 'audio', 'nextsong', 'nextsongid' } self.assertEqual(fields_playing, set(responses[2].data.keys())) From d8be83bc0d504744f6ce699b01040a7465caf1b6 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 15:28:53 +1000 Subject: [PATCH 323/339] bpd: support ranges in playlistid --- beetsplug/bpd/__init__.py | 31 ++++++++++++++++++++++++------- test/test_player.py | 10 +++++++--- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 8d90b5cf6..ab515b388 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -583,23 +583,25 @@ class BaseServer(object): """Indicates supported URL schemes. None by default.""" pass - def cmd_playlistinfo(self, conn, index=-1): + def cmd_playlistinfo(self, conn, index=None): """Gives metadata information about the entire playlist or a single track, given by its index. """ - index = cast_arg(int, index) - if index == -1: + if index is None: for track in self.playlist: yield self._item_info(track) else: + indices = self._parse_range(index, accept_single_number=True) try: - track = self.playlist[index] + tracks = [self.playlist[i] for i in indices] except IndexError: raise ArgumentIndexError() - yield self._item_info(track) + for track in tracks: + yield self._item_info(track) - def cmd_playlistid(self, conn, track_id=-1): - if track_id != -1: + def cmd_playlistid(self, conn, track_id=None): + if track_id is not None: + track_id = cast_arg(int, track_id) track_id = self._id_to_index(track_id) return self.cmd_playlistinfo(conn, track_id) @@ -1139,6 +1141,21 @@ class Server(BaseServer): return info_lines + def _parse_range(self, items, accept_single_number=False): + """Convert a range of positions to a list of item info. + MPD specifies ranges as START:STOP (endpoint excluded) for some + commands. Sometimes a single number can be provided instead. + """ + try: + start, stop = str(items).split(':', 1) + except ValueError: + if accept_single_number: + return [cast_arg(int, items)] + raise BPDError(ERROR_ARG, u'bad range syntax') + start = cast_arg(int, start) + stop = cast_arg(int, stop) + return range(start, stop) + def _item_id(self, item): return item.id diff --git a/test/test_player.py b/test/test_player.py index 9bd75519f..735f9c62c 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -339,8 +339,9 @@ class BPDTestHelper(unittest.TestCase, TestHelper): previous_commands = response[0:pos] self._assert_ok(*previous_commands) response = response[pos] - self.assertEqual(pos, response.err_data[1]) self.assertFalse(response.ok) + if pos is not None: + self.assertEqual(pos, response.err_data[1]) if code is not None: self.assertEqual(code, response.err_data[0]) @@ -781,12 +782,15 @@ class BPDQueueTest(BPDTestHelper): def test_cmd_playlistinfo(self): with self.run_bpd() as client: - self._bpd_add(client, self.item1) + self._bpd_add(client, self.item1, self.item2) responses = client.send_commands( ('playlistinfo',), ('playlistinfo', '0'), + ('playlistinfo', '0:2'), ('playlistinfo', '200')) - self._assert_failed(responses, bpd.ERROR_ARG, pos=2) + self._assert_failed(responses, bpd.ERROR_ARG, pos=3) + self.assertEqual('1', responses[1].data['Id']) + self.assertEqual(['1', '2'], responses[2].data['Id']) def test_cmd_playlistinfo_tagtypes(self): with self.run_bpd() as client: From 62aa358ce7fa8f6a0193009d403ca3936d3df890 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sat, 13 Apr 2019 14:18:54 +1000 Subject: [PATCH 324/339] bpd: bump protocol version to 0.16 --- beetsplug/bpd/__init__.py | 2 +- docs/plugins/bpd.rst | 2 +- test/test_player.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index ab515b388..045bce035 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -39,7 +39,7 @@ from beets import dbcore from mediafile import MediaFile import six -PROTOCOL_VERSION = '0.14.0' +PROTOCOL_VERSION = '0.16.0' BUFSIZE = 1024 HELLO = u'OK MPD %s' % PROTOCOL_VERSION diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index 8bfe456ad..c1a94e972 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -103,7 +103,7 @@ but doesn't support many advanced playback features. Differences from the real MPD ----------------------------- -BPD currently supports version 0.14 of `the MPD protocol`_, but several of the +BPD currently supports version 0.16 of `the MPD protocol`_, but several of the commands and features are "pretend" implementations or have slightly different behaviour to their MPD equivalents. BPD aims to look enough like MPD that it can interact with the ecosystem of clients, but doesn't try to be diff --git a/test/test_player.py b/test/test_player.py index 735f9c62c..959d77eb3 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -363,7 +363,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): class BPDTest(BPDTestHelper): def test_server_hello(self): with self.run_bpd(do_hello=False) as client: - self.assertEqual(client.readline(), b'OK MPD 0.14.0\n') + self.assertEqual(client.readline(), b'OK MPD 0.16.0\n') def test_unknown_cmd(self): with self.run_bpd() as client: From 65432bbb2d82cb402478dea77b2f1199c5ef41d8 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Sun, 2 Jun 2019 23:50:20 +1000 Subject: [PATCH 325/339] Changelog for #3214 --- docs/changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7a74f556b..9e2bfd8ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,12 @@ New features: (the MBID), and ``work_disambig`` (the disambiguation string). Thanks to :user:`dosoe`. :bug:`2580` :bug:`3272` +* :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 + of the MPD protocol. This is enough to get it talking to more complicated + clients like ncmpcpp, but there are still some incompatibilities, largely due + to MPD commands we don't support yet. Let us know if you find an MPD client + that doesn't get along with BPD! + :bug:`3214` :bug:`800` Fixes: @@ -20,6 +26,10 @@ Fixes: objects could seem to reuse values from earlier objects when they were missing a value for a given field. These values are now properly undefined. :bug:`2406` +* :doc:`/plugins/bpd`: Seeking by fractions of a second now works as intended, + fixing crashes in MPD clients like mpDris2 on seek. + The ``playlistid`` command now works properly in its zero-argument form. + :bug:`3214` For plugin developers: From 5a3157d85d76f8e3955e2add763ffb54667d49c5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 2 Jun 2019 20:12:07 -0400 Subject: [PATCH 326/339] Try to make a test more reliable As @arcresu pointed out on Gitter, this openSUSE patch adds a safeguard to this test, which was apparently failing for them: https://build.opensuse.org/package/view_file/openSUSE:Factory/beets/fix_test_command_line_option_relative_to_working_dir.diff?expand=1 Reading the configuration once here to make sure that we're in a clean state seems harmless enough. The culprit is likely that a previous test was modifying the configuration and not properly cleaning up. This change defends against that kind of mistake. --- docs/changelog.rst | 3 +++ test/test_ui.py | 1 + 2 files changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7a74f556b..7bfe10177 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,6 +46,9 @@ For packagers: called `Confuse`_, released as :pypi:`confuse`. Beets now depends on this package. Confuse has existed separately for some time and is used by unrelated projects, but until now we've been bundling a copy within beets. +* We attempted to fix an unreliable test, so a patch to `skip `_ + or `repair `_ + the test may no longer be necessary. .. _MediaFile: https://github.com/beetbox/mediafile .. _Confuse: https://github.com/beetbox/confuse diff --git a/test/test_ui.py b/test/test_ui.py index da3160db7..110e80782 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -918,6 +918,7 @@ class ConfigTest(unittest.TestCase, TestHelper, _common.Assertions): ) def test_command_line_option_relative_to_working_dir(self): + config.read() os.chdir(self.temp_dir) self.run_command('--library', 'foo.db', 'test', lib=None) self.assert_equal_path(config['library'].as_filename(), From aa31fea037ce82ebfafb4dd7d596b7aa36e8b2d6 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 01:07:31 +0200 Subject: [PATCH 327/339] Update a lot of URLs to use HTTPS *All* URLs were checked manually, but only once per domain! I mostly concerned myself with URLs in documentation rather than source code because the latter may or may not have impactful changes, while the former should be straight forward. Changes in addition to simply adding an s: - changed pip and pypi references as their location has changed - MPoD (iOS app) url redirects to Regelian, so I replaced those - updated homebrew references Notable observations: - beets.io does have HTTPS set up properly (via gh-pages) - beatport.py uses the old HTTP url for beatport - as does lyrics.py for lyrics.wikia.com - https://tomahawk-player.org/ expired long ago, but the http page redirects to https regardless - none of the sourceforge subdomains have https (in 2019!) --- .travis.yml | 4 +-- README.rst | 42 ++++++++++++++--------------- README_kr.rst | 46 +++++++++++++++---------------- beets/dbcore/types.py | 2 +- beets/ui/commands.py | 2 +- beetsplug/absubmit.py | 4 +-- beetsplug/fetchart.py | 10 +++++-- beetsplug/keyfinder.py | 2 +- beetsplug/lastimport.py | 2 +- beetsplug/mbsubmit.py | 2 +- beetsplug/thumbnails.py | 4 +-- beetsplug/web/static/beets.js | 2 +- docs/changelog.rst | 48 ++++++++++++++++----------------- docs/dev/index.rst | 2 +- docs/dev/plugins.rst | 12 ++++----- docs/faq.rst | 22 +++++++-------- docs/guides/advanced.rst | 2 +- docs/guides/main.rst | 26 +++++++++--------- docs/guides/tagger.rst | 8 +++--- docs/index.rst | 4 +-- docs/plugins/absubmit.rst | 8 +++--- docs/plugins/acousticbrainz.rst | 2 +- docs/plugins/beatport.rst | 4 +-- docs/plugins/bpd.rst | 14 +++++----- docs/plugins/chroma.rst | 24 ++++++++--------- docs/plugins/convert.rst | 8 +++--- docs/plugins/discogs.rst | 2 +- docs/plugins/embedart.rst | 2 +- docs/plugins/embyupdate.rst | 4 +-- docs/plugins/export.rst | 2 +- docs/plugins/fetchart.rst | 8 +++--- docs/plugins/ftintitle.rst | 2 +- docs/plugins/index.rst | 18 ++++++------- docs/plugins/info.rst | 2 +- docs/plugins/ipfs.rst | 2 +- docs/plugins/keyfinder.rst | 2 +- docs/plugins/kodiupdate.rst | 4 +-- docs/plugins/lastgenre.rst | 8 +++--- docs/plugins/lastimport.rst | 4 +-- docs/plugins/lyrics.rst | 16 +++++------ docs/plugins/mbcollection.rst | 2 +- docs/plugins/mbsubmit.rst | 6 ++--- docs/plugins/metasync.rst | 2 +- docs/plugins/mpdstats.rst | 4 +-- docs/plugins/mpdupdate.rst | 2 +- docs/plugins/plexupdate.rst | 6 ++--- docs/plugins/replaygain.rst | 8 +++--- docs/plugins/sonosupdate.rst | 2 +- docs/plugins/spotify.rst | 4 +-- docs/plugins/subsonicupdate.rst | 2 +- docs/plugins/thumbnails.rst | 2 +- docs/plugins/web.rst | 12 ++++----- docs/reference/cli.rst | 4 +-- docs/reference/config.rst | 10 +++---- docs/reference/pathformat.rst | 12 ++++----- docs/reference/query.rst | 2 +- extra/_beet | 2 +- setup.py | 2 +- test/test_art.py | 6 ++--- 59 files changed, 239 insertions(+), 233 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8376be522..455ab4ca4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,8 @@ matrix: addons: apt: sources: - - sourceline: "deb http://archive.ubuntu.com/ubuntu/ trusty multiverse" - - sourceline: "deb http://archive.ubuntu.com/ubuntu/ trusty-updates multiverse" + - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty multiverse" + - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty-updates multiverse" packages: - bash-completion - gir1.2-gst-plugins-base-1.0 diff --git a/README.rst b/README.rst index a3ea6302f..6b4ebb4fa 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ -.. image:: http://img.shields.io/pypi/v/beets.svg +.. image:: https://img.shields.io/pypi/v/beets.svg :target: https://pypi.python.org/pypi/beets -.. image:: http://img.shields.io/codecov/c/github/beetbox/beets.svg +.. image:: https://img.shields.io/codecov/c/github/beetbox/beets.svg :target: https://codecov.io/github/beetbox/beets .. image:: https://travis-ci.org/beetbox/beets.svg?branch=master @@ -51,28 +51,28 @@ imagine for your music collection. Via `plugins`_, beets becomes a panacea: If beets doesn't do what you want yet, `writing your own plugin`_ is shockingly simple if you know a little Python. -.. _plugins: http://beets.readthedocs.org/page/plugins/ -.. _MPD: http://www.musicpd.org/ -.. _MusicBrainz music collection: http://musicbrainz.org/doc/Collections/ +.. _plugins: https://beets.readthedocs.org/page/plugins/ +.. _MPD: https://www.musicpd.org/ +.. _MusicBrainz music collection: https://musicbrainz.org/doc/Collections/ .. _writing your own plugin: - http://beets.readthedocs.org/page/dev/plugins.html + https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html .. _albums that are missing tracks: - http://beets.readthedocs.org/page/plugins/missing.html + https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: - http://beets.readthedocs.org/page/plugins/duplicates.html + https://beets.readthedocs.org/page/plugins/duplicates.html .. _Transcode audio: - http://beets.readthedocs.org/page/plugins/convert.html -.. _Discogs: http://www.discogs.com/ + https://beets.readthedocs.org/page/plugins/convert.html +.. _Discogs: https://www.discogs.com/ .. _acoustic fingerprints: - http://beets.readthedocs.org/page/plugins/chroma.html -.. _ReplayGain: http://beets.readthedocs.org/page/plugins/replaygain.html -.. _tempos: http://beets.readthedocs.org/page/plugins/acousticbrainz.html -.. _genres: http://beets.readthedocs.org/page/plugins/lastgenre.html -.. _album art: http://beets.readthedocs.org/page/plugins/fetchart.html -.. _lyrics: http://beets.readthedocs.org/page/plugins/lyrics.html -.. _MusicBrainz: http://musicbrainz.org/ + https://beets.readthedocs.org/page/plugins/chroma.html +.. _ReplayGain: https://beets.readthedocs.org/page/plugins/replaygain.html +.. _tempos: https://beets.readthedocs.org/page/plugins/acousticbrainz.html +.. _genres: https://beets.readthedocs.org/page/plugins/lastgenre.html +.. _album art: https://beets.readthedocs.org/page/plugins/fetchart.html +.. _lyrics: https://beets.readthedocs.org/page/plugins/lyrics.html +.. _MusicBrainz: https://musicbrainz.org/ .. _Beatport: https://www.beatport.com Install @@ -81,7 +81,7 @@ Install You can install beets by typing ``pip install beets``. Then check out the `Getting Started`_ guide. -.. _Getting Started: http://beets.readthedocs.org/page/guides/main.html +.. _Getting Started: https://beets.readthedocs.org/page/guides/main.html Contribute ---------- @@ -90,7 +90,7 @@ Check out the `Hacking`_ page on the wiki for tips on how to help out. You might also be interested in the `For Developers`_ section in the docs. .. _Hacking: https://github.com/beetbox/beets/wiki/Hacking -.. _For Developers: http://docs.beets.io/page/dev/ +.. _For Developers: https://beets.readthedocs.io/en/stable/dev/ Read More --------- @@ -99,7 +99,7 @@ Learn more about beets at `its Web site`_. Follow `@b33ts`_ on Twitter for news and updates. .. _its Web site: http://beets.io/ -.. _@b33ts: http://twitter.com/b33ts/ +.. _@b33ts: https://twitter.com/b33ts/ Authors ------- @@ -108,4 +108,4 @@ Beets is by `Adrian Sampson`_ with a supporting cast of thousands. For help, please visit our `forum`_. .. _forum: https://discourse.beets.io -.. _Adrian Sampson: http://www.cs.cornell.edu/~asampson/ +.. _Adrian Sampson: https://www.cs.cornell.edu/~asampson/ diff --git a/README_kr.rst b/README_kr.rst index 18389061c..6bdcf56a6 100644 --- a/README_kr.rst +++ b/README_kr.rst @@ -1,7 +1,7 @@ -.. image:: http://img.shields.io/pypi/v/beets.svg +.. image:: https://img.shields.io/pypi/v/beets.svg :target: https://pypi.python.org/pypi/beets -.. image:: http://img.shields.io/codecov/c/github/beetbox/beets.svg +.. image:: https://img.shields.io/codecov/c/github/beetbox/beets.svg :target: https://codecov.io/github/beetbox/beets .. image:: https://travis-ci.org/beetbox/beets.svg?branch=master @@ -48,28 +48,28 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 만약 Beets에 당신이 원하는게 아직 없다면, 당신이 python을 안다면 `writing your own plugin`_ _은 놀라울정도로 간단하다. -.. _plugins: http://beets.readthedocs.org/page/plugins/ -.. _MPD: http://www.musicpd.org/ -.. _MusicBrainz music collection: http://musicbrainz.org/doc/Collections/ +.. _plugins: https://beets.readthedocs.org/page/plugins/ +.. _MPD: https://www.musicpd.org/ +.. _MusicBrainz music collection: https://musicbrainz.org/doc/Collections/ .. _writing your own plugin: - http://beets.readthedocs.org/page/dev/plugins.html + https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html .. _albums that are missing tracks: - http://beets.readthedocs.org/page/plugins/missing.html + https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: - http://beets.readthedocs.org/page/plugins/duplicates.html + https://beets.readthedocs.org/page/plugins/duplicates.html .. _Transcode audio: - http://beets.readthedocs.org/page/plugins/convert.html -.. _Discogs: http://www.discogs.com/ + https://beets.readthedocs.org/page/plugins/convert.html +.. _Discogs: https://www.discogs.com/ .. _acoustic fingerprints: - http://beets.readthedocs.org/page/plugins/chroma.html -.. _ReplayGain: http://beets.readthedocs.org/page/plugins/replaygain.html -.. _tempos: http://beets.readthedocs.org/page/plugins/acousticbrainz.html -.. _genres: http://beets.readthedocs.org/page/plugins/lastgenre.html -.. _album art: http://beets.readthedocs.org/page/plugins/fetchart.html -.. _lyrics: http://beets.readthedocs.org/page/plugins/lyrics.html -.. _MusicBrainz: http://musicbrainz.org/ + https://beets.readthedocs.org/page/plugins/chroma.html +.. _ReplayGain: https://beets.readthedocs.org/page/plugins/replaygain.html +.. _tempos: https://beets.readthedocs.org/page/plugins/acousticbrainz.html +.. _genres: https://beets.readthedocs.org/page/plugins/lastgenre.html +.. _album art: https://beets.readthedocs.org/page/plugins/fetchart.html +.. _lyrics: https://beets.readthedocs.org/page/plugins/lyrics.html +.. _MusicBrainz: https://musicbrainz.org/ .. _Beatport: https://www.beatport.com 설치 @@ -78,7 +78,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 당신은 ``pip install beets`` 을 사용해서 Beets를 설치할 수 있다. 그리고 `Getting Started`_ 가이드를 확인할 수 있다. -.. _Getting Started: http://beets.readthedocs.org/page/guides/main.html +.. _Getting Started: https://beets.readthedocs.org/page/guides/main.html 컨트리뷰션 ---------- @@ -87,16 +87,16 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 당신은 docs 안에 `For Developers`_ 에도 관심이 있을수 있다. .. _Hacking: https://github.com/beetbox/beets/wiki/Hacking -.. _For Developers: http://docs.beets.io/page/dev/ +.. _For Developers: https://beets.readthedocs.io/en/stable/dev/ Read More --------- -`its Web site`_ 에서 Beets에 대해 조금 더 알아볼 수 있다. +`its Web site`_ 에서 Beets에 대해 조금 더 알아볼 수 있다. 트위터에서 `@b33ts`_ 를 팔로우하면 새 소식을 볼 수 있다. -.. _its Web site: http://beets.io/ -.. _@b33ts: http://twitter.com/b33ts/ +.. _its Web site: https://beets.io/ +.. _@b33ts: https://twitter.com/b33ts/ 저자들 ------- @@ -105,4 +105,4 @@ Read More 돕고 싶다면 `forum`_.를 방문하면 된다. .. _forum: https://discourse.beets.io -.. _Adrian Sampson: http://www.cs.cornell.edu/~asampson/ +.. _Adrian Sampson: https://www.cs.cornell.edu/~asampson/ diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index c37def875..521a5a1ee 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -97,7 +97,7 @@ class Type(object): For fixed fields the type of `value` is determined by the column type affinity given in the `sql` property and the SQL to Python mapping of the database adapter. For more information see: - http://www.sqlite.org/datatype3.html + https://www.sqlite.org/datatype3.html https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types Flexible fields have the type affinity `TEXT`. This means the diff --git a/beets/ui/commands.py b/beets/ui/commands.py index ef4fd144a..53253c1da 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -542,7 +542,7 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, print_(u"No matching release found for {0} tracks." .format(itemcount)) print_(u'For help, see: ' - u'http://beets.readthedocs.org/en/latest/faq.html#nomatch') + u'https://beets.readthedocs.org/en/latest/faq.html#nomatch') sel = ui.input_options(choice_opts) if sel in choice_actions: return choice_actions[sel] diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 9d26ac5db..d9525e1d2 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -73,8 +73,8 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): call([self.extractor]) except OSError: raise ui.UserError( - u'No extractor command found: please install the ' - u'extractor binary from http://acousticbrainz.org/download' + u'No extractor command found: please install the extractor' + u' binary from https://acousticbrainz.org/download' ) except ABSubmitError: # Extractor found, will exit with an error if not called with diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index af1aaa567..a815d4d9b 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -311,7 +311,10 @@ class CoverArtArchive(RemoteArtSource): class Amazon(RemoteArtSource): NAME = u"Amazon" - URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' + if util.SNI_SUPPORTED: + URL = 'https://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' + else: + URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' INDICES = (1, 2) def get(self, album, plugin, paths): @@ -325,7 +328,10 @@ class Amazon(RemoteArtSource): class AlbumArtOrg(RemoteArtSource): NAME = u"AlbumArt.org scraper" - URL = 'http://www.albumart.org/index_detail.php' + if util.SNI_SUPPORTED: + URL = 'https://www.albumart.org/index_detail.php' + else: + URL = 'http://www.albumart.org/index_detail.php' PAT = r'href\s*=\s*"([^>"]*)"[^>]*title\s*=\s*"View larger image"' def get(self, album, plugin, paths): diff --git a/beetsplug/keyfinder.py b/beetsplug/keyfinder.py index a3fbc8211..3a738478e 100644 --- a/beetsplug/keyfinder.py +++ b/beetsplug/keyfinder.py @@ -66,7 +66,7 @@ class KeyFinderPlugin(BeetsPlugin): continue except UnicodeEncodeError: # Workaround for Python 2 Windows bug. - # http://bugs.python.org/issue1759845 + # https://bugs.python.org/issue1759845 self._log.error(u'execution failed for Unicode path: {0!r}', item.path) continue diff --git a/beetsplug/lastimport.py b/beetsplug/lastimport.py index d7b84b0aa..ca97004cf 100644 --- a/beetsplug/lastimport.py +++ b/beetsplug/lastimport.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2016, Rafael Bodill http://github.com/rafi +# Copyright 2016, Rafael Bodill https://github.com/rafi # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the diff --git a/beetsplug/mbsubmit.py b/beetsplug/mbsubmit.py index 02bd5f697..44a476d15 100644 --- a/beetsplug/mbsubmit.py +++ b/beetsplug/mbsubmit.py @@ -19,7 +19,7 @@ This plugin allows the user to print track information in a format that is parseable by the MusicBrainz track parser [1]. Programmatic submitting is not implemented by MusicBrainz yet. -[1] http://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings +[1] https://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings """ from __future__ import division, absolute_import, print_function diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index 04845e880..fe36fbd13 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -160,7 +160,7 @@ class ThumbnailsPlugin(BeetsPlugin): def thumbnail_file_name(self, path): """Compute the thumbnail file name - See http://standards.freedesktop.org/thumbnail-spec/latest/x227.html + See https://standards.freedesktop.org/thumbnail-spec/latest/x227.html """ uri = self.get_uri(path) hash = md5(uri.encode('utf-8')).hexdigest() @@ -168,7 +168,7 @@ class ThumbnailsPlugin(BeetsPlugin): def add_tags(self, album, image_path): """Write required metadata to the thumbnail - See http://standards.freedesktop.org/thumbnail-spec/latest/x142.html + See https://standards.freedesktop.org/thumbnail-spec/latest/x142.html """ mtime = os.stat(album.artpath).st_mtime metadata = {"Thumb::URI": self.get_uri(album.artpath), diff --git a/beetsplug/web/static/beets.js b/beetsplug/web/static/beets.js index 51985c183..97af70110 100644 --- a/beetsplug/web/static/beets.js +++ b/beetsplug/web/static/beets.js @@ -129,7 +129,7 @@ $.fn.player = function(debug) { // Simple selection disable for jQuery. // Cut-and-paste from: -// http://stackoverflow.com/questions/2700000 +// https://stackoverflow.com/questions/2700000 $.fn.disableSelection = function() { $(this).attr('unselectable', 'on') .css('-moz-user-select', 'none') diff --git a/docs/changelog.rst b/docs/changelog.rst index e58325482..a667be780 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1216,7 +1216,7 @@ There are even more new features: don't actually need to be moved. :bug:`1583` .. _Google Code-In: https://codein.withgoogle.com/ -.. _AcousticBrainz: http://acousticbrainz.org/ +.. _AcousticBrainz: https://acousticbrainz.org/ Fixes: @@ -1358,7 +1358,7 @@ Fixes: communication errors. The backend has also been disabled by default, since the API it depends on is currently down. :bug:`1770` -.. _Emby: http://emby.media +.. _Emby: https://emby.media 1.3.15 (October 17, 2015) @@ -1520,8 +1520,8 @@ Fixes: * :doc:`/plugins/convert`: Fix a problem with filename encoding on Windows under Python 3. :bug:`2515` :bug:`2516` -.. _Python bug: http://bugs.python.org/issue16512 -.. _ipfs: http://ipfs.io +.. _Python bug: https://bugs.python.org/issue16512 +.. _ipfs: https://ipfs.io 1.3.13 (April 24, 2015) @@ -1872,7 +1872,7 @@ As usual, there are loads of little fixes and improvements: * The :ref:`config-cmd` command can now use ``$EDITOR`` variables with arguments. -.. _API changes: http://developer.echonest.com/forums/thread/3650 +.. _API changes: https://developer.echonest.com/forums/thread/3650 .. _Plex: https://plex.tv/ .. _musixmatch: https://www.musixmatch.com/ @@ -2352,7 +2352,7 @@ Fixes: * :doc:`/plugins/convert`: Display a useful error message when the FFmpeg executable can't be found. -.. _requests: http://www.python-requests.org/ +.. _requests: https://www.python-requests.org/ 1.3.3 (February 26, 2014) @@ -2534,7 +2534,7 @@ As usual, there are also innumerable little fixes and improvements: .. _Acoustic Attributes: http://developer.echonest.com/acoustic-attributes.html -.. _MPD: http://www.musicpd.org/ +.. _MPD: https://www.musicpd.org/ 1.3.1 (October 12, 2013) @@ -2601,7 +2601,7 @@ And some fixes: * :doc:`/plugins/scrub`: Avoid preserving certain non-standard ID3 tags such as NCON. -.. _Opus: http://www.opus-codec.org/ +.. _Opus: https://www.opus-codec.org/ .. _@Verrus: https://github.com/Verrus @@ -2833,8 +2833,8 @@ And a batch of fixes: * :doc:`/plugins/lyrics`: Lyrics searches should now turn up more results due to some fixes in dealing with special characters. -.. _Discogs: http://discogs.com/ -.. _Beatport: http://www.beatport.com/ +.. _Discogs: https://discogs.com/ +.. _Beatport: https://www.beatport.com/ 1.1.0 (April 29, 2013) @@ -2883,7 +2883,7 @@ will automatically migrate your configuration to the new system. header. Thanks to Uwe L. Korn. * :doc:`/plugins/lastgenre`: Fix an error when using genre canonicalization. -.. _Tomahawk: http://www.tomahawk-player.org/ +.. _Tomahawk: https://tomahawk-player.org/ 1.1b3 (March 16, 2013) ---------------------- @@ -3058,7 +3058,7 @@ Other new stuff: (YAML doesn't like tabs.) * Fix the ``-l`` (log path) command-line option for the ``import`` command. -.. _iTunes Sound Check: http://support.apple.com/kb/HT2425 +.. _iTunes Sound Check: https://support.apple.com/kb/HT2425 1.1b1 (January 29, 2013) ------------------------ @@ -3067,7 +3067,7 @@ This release entirely revamps beets' configuration system. The configuration file is now a `YAML`_ document and is located, along with other support files, in a common directory (e.g., ``~/.config/beets`` on Unix-like systems). -.. _YAML: http://en.wikipedia.org/wiki/YAML +.. _YAML: https://en.wikipedia.org/wiki/YAML * Renamed plugins: The ``rdm`` plugin has been renamed to ``random`` and ``fuzzy_search`` has been renamed to ``fuzzy``. @@ -3229,7 +3229,7 @@ begins today on features for version 1.1. .. _The Echo Nest: http://the.echonest.com/ .. _Tomahawk resolver: http://beets.io/blog/tomahawk-resolver.html .. _mp3gain: http://mp3gain.sourceforge.net/download.php -.. _aacgain: http://aacgain.altosdesign.com +.. _aacgain: https://aacgain.altosdesign.com 1.0b15 (July 26, 2012) ---------------------- @@ -3338,7 +3338,7 @@ fetching cover art for your music, enable this plugin after upgrading to beets database with ``beet import -AWC /path/to/music``. * Fix ``import`` with relative path arguments on Windows. -.. _artist credits: http://wiki.musicbrainz.org/Artist_Credit +.. _artist credits: https://wiki.musicbrainz.org/Artist_Credit 1.0b14 (May 12, 2012) --------------------- @@ -3496,7 +3496,7 @@ to come in the next couple of releases. data. * Fix the ``list`` command in BPD (thanks to Simon Chopin). -.. _Colorama: http://pypi.python.org/pypi/colorama +.. _Colorama: https://pypi.python.org/pypi/colorama 1.0b12 (January 16, 2012) ------------------------- @@ -3609,12 +3609,12 @@ release: one for assigning genres and another for ReplayGain analysis. corrupted. .. _KraYmer: https://github.com/KraYmer -.. _Next Generation Schema: http://musicbrainz.org/doc/XML_Web_Service/Version_2 +.. _Next Generation Schema: https://musicbrainz.org/doc/XML_Web_Service/Version_2 .. _python-musicbrainzngs: https://github.com/alastair/python-musicbrainzngs -.. _acoustid: http://acoustid.org/ +.. _acoustid: https://acoustid.org/ .. _Peter Brunner: https://github.com/Lugoues .. _Simon Chopin: https://github.com/laarmen -.. _albumart.org: http://www.albumart.org/ +.. _albumart.org: https://www.albumart.org/ 1.0b10 (September 22, 2011) --------------------------- @@ -3783,8 +3783,8 @@ below, for a plethora of new features. * Fix a crash on album queries with item-only field names. -.. _xargs: http://en.wikipedia.org/wiki/xargs -.. _unidecode: http://pypi.python.org/pypi/Unidecode/0.04.1 +.. _xargs: https://en.wikipedia.org/wiki/xargs +.. _unidecode: https://pypi.python.org/pypi/Unidecode/0.04.1 1.0b8 (April 28, 2011) ---------------------- @@ -3927,7 +3927,7 @@ new configuration options and the ability to clean up empty directory subtrees. * The old "albumify" plugin for upgrading databases was removed. -.. _as specified by MusicBrainz: http://wiki.musicbrainz.org/ReleaseType +.. _as specified by MusicBrainz: https://wiki.musicbrainz.org/ReleaseType 1.0b6 (January 20, 2011) ------------------------ @@ -4043,7 +4043,7 @@ are also rolled into this release. * Fixed escaping of ``/`` characters in paths on Windows. -.. _!!!: http://musicbrainz.org/artist/f26c72d3-e52c-467b-b651-679c73d8e1a7.html +.. _!!!: https://musicbrainz.org/artist/f26c72d3-e52c-467b-b651-679c73d8e1a7.html 1.0b4 (August 9, 2010) ---------------------- @@ -4232,7 +4232,7 @@ Vorbis) and an option to log untaggable albums during import. removed dependency on the aging ``cmdln`` module in favor of `a hand-rolled solution`_. -.. _a hand-rolled solution: http://gist.github.com/462717 +.. _a hand-rolled solution: https://gist.github.com/462717 1.0b1 (June 17, 2010) --------------------- diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 45640254c..a47d6c8f2 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -7,7 +7,7 @@ in hacking beets itself or creating plugins for it. See also the documentation for `MediaFile`_, the library used by beets to read and write metadata tags in media files. -.. _MediaFile: http://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/ .. toctree:: diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 7ff397bc6..3328654e0 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -15,7 +15,7 @@ structure should look like this:: myawesomeplugin.py .. _Stack Overflow question about namespace packages: - http://stackoverflow.com/questions/1675734/how-do-i-create-a-namespace-package-in-python/1676069#1676069 + https://stackoverflow.com/questions/1675734/how-do-i-create-a-namespace-package-in-python/1676069#1676069 Then, you'll need to put this stuff in ``__init__.py`` to make ``beetsplug`` a namespace package:: @@ -42,7 +42,7 @@ Then, as described above, edit your ``config.yaml`` to include ``plugins: myawesomeplugin`` (substituting the name of the Python module containing your plugin). -.. _virtualenv: http://pypi.python.org/pypi/virtualenv +.. _virtualenv: https://pypi.org/project/virtualenv .. _add_subcommands: @@ -73,7 +73,7 @@ but it defaults to an empty parser (you can extend it later). ``help`` is a description of your command, and ``aliases`` is a list of shorthand versions of your command name. -.. _OptionParser instance: http://docs.python.org/library/optparse.html +.. _OptionParser instance: https://docs.python.org/library/optparse.html You'll need to add a function to your command by saying ``mycommand.func = myfunction``. This function should take the following parameters: ``lib`` (a @@ -81,7 +81,7 @@ beets ``Library`` object) and ``opts`` and ``args`` (command-line options and arguments as returned by `OptionParser.parse_args`_). .. _OptionParser.parse_args: - http://docs.python.org/library/optparse.html#parsing-arguments + https://docs.python.org/library/optparse.html#parsing-arguments The function should use any of the utility functions defined in ``beets.ui``. Try running ``pydoc beets.ui`` to see what's available. @@ -301,7 +301,7 @@ To access this value, say ``self.config['foo'].get()`` at any point in your plugin's code. The `self.config` object is a *view* as defined by the `Confuse`_ library. -.. _Confuse: http://confuse.readthedocs.org/ +.. _Confuse: https://confuse.readthedocs.org/ If you want to access configuration values *outside* of your plugin's section, import the `config` object from the `beets` module. That is, just put ``from @@ -379,7 +379,7 @@ access to file tags. If you have created a descriptor you can add it through your plugins ``add_media_field()`` method. .. automethod:: beets.plugins.BeetsPlugin.add_media_field -.. _MediaFile: http://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/ Here's an example plugin that provides a meaningless new field "foo":: diff --git a/docs/faq.rst b/docs/faq.rst index b7ec10df5..9732a4725 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -6,8 +6,8 @@ Got a question that isn't answered here? Try `IRC`_, the `discussion board`_, or :ref:`filing an issue ` in the bug tracker. .. _IRC: irc://irc.freenode.net/beets -.. _mailing list: http://groups.google.com/group/beets-users -.. _discussion board: http://discourse.beets.io +.. _mailing list: https://groups.google.com/group/beets-users +.. _discussion board: https://discourse.beets.io .. contents:: :local: @@ -94,14 +94,14 @@ the tracks into a single directory to force them to be tagged together. An MBID looks like one of these: -- ``http://musicbrainz.org/release/ded77dcf-7279-457e-955d-625bd3801b87`` +- ``https://musicbrainz.org/release/ded77dcf-7279-457e-955d-625bd3801b87`` - ``d569deba-8c6b-4d08-8c43-d0e5a1b8c7f3`` Beets can recognize either the hex-with-dashes UUID-style string or the full URL that contains it (as of 1.0b11). You can get these IDs by `searching on the MusicBrainz web -site `__ and going to a *release* page (when +site `__ and going to a *release* page (when tagging full albums) or a *recording* page (when tagging singletons). Then, copy the URL of the page and paste it into beets. @@ -119,7 +119,7 @@ Run a command like this:: pip install -U beets -The ``-U`` flag tells `pip `__ to upgrade +The ``-U`` flag tells `pip `__ to upgrade beets to the latest version. If you want a specific version, you can specify with using ``==`` like so:: @@ -163,10 +163,10 @@ on GitHub. `Enter a new issue `__ there to report a bug. Please follow these guidelines when reporting an issue: - Most importantly: if beets is crashing, please `include the - traceback `__. Tracebacks can be more + traceback `__. Tracebacks can be more readable if you put them in a pastebin (e.g., `Gist `__ or - `Hastebin `__), especially when communicating + `Hastebin `__), especially when communicating over IRC or email. - Turn on beets' debug output (using the -v option: for example, ``beet -v import ...``) and include that with your bug report. Look @@ -188,7 +188,7 @@ there to report a bug. Please follow these guidelines when reporting an issue: If you've never reported a bug before, Mozilla has some well-written `general guidelines for good bug -reports `__. +reports `__. .. _find-config: @@ -237,7 +237,7 @@ Why does beets… There are a number of possibilities: - First, make sure the album is in `the MusicBrainz - database `__. You + database `__. You can search on their site to make sure it's cataloged there. (If not, anyone can edit MusicBrainz---so consider adding the data yourself.) - If the album in question is a multi-disc release, see the relevant @@ -320,7 +320,7 @@ it encounters files that *look* like music files (according to their extension) but seem to be broken. Most of the time, this is because the file is corrupted. To check whether the file is intact, try opening it in another media player (e.g., -`VLC `__) to see whether it can +`VLC `__) to see whether it can read the file. You can also use specialized programs for checking file integrity---for example, type ``metaflac --list music.flac`` to check FLAC files. @@ -378,4 +378,4 @@ installed using pip, the command ``pip show -f beets`` can show you where ``beet`` was placed on your system. If you need help extending your ``$PATH``, try `this Super User answer`_. -.. _this Super User answer: http://superuser.com/a/284361/4569 +.. _this Super User answer: https://superuser.com/a/284361/4569 diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index 38cc31d0c..091875c54 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -93,7 +93,7 @@ everything by the Long Winters for listening on the go. The plugin has many more dials you can fiddle with to get your conversions how you like them. Check out :doc:`its documentation `. -.. _ffmpeg: http://www.ffmpeg.org +.. _ffmpeg: https://www.ffmpeg.org Store any data you like diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 563b7ef82..1f5cc4681 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -12,7 +12,7 @@ Installing You will need Python. Beets works on `Python 2.7`_ and Python 3.4 or later. -.. _Python 2.7: http://www.python.org/download/ +.. _Python 2.7: https://www.python.org/download/ * **macOS** v10.7 (Lion) and later include Python 2.7 out of the box. You can opt for Python 3 by installing it via `Homebrew`_: @@ -49,7 +49,7 @@ Beets works on `Python 2.7`_ and Python 3.4 or later. * On **NixOS**, there's a `package `_ you can install with ``nix-env -i beets``. .. _DNF package: https://apps.fedoraproject.org/packages/beets -.. _SlackBuild: http://slackbuilds.org/repository/14.2/multimedia/beets/ +.. _SlackBuild: https://slackbuilds.org/repository/14.2/multimedia/beets/ .. _FreeBSD: http://portsmon.freebsd.org/portoverview.py?category=audio&portname=beets .. _AUR: https://aur.archlinux.org/packages/beets-git/ .. _Debian details: https://tracker.debian.org/pkg/beets @@ -64,14 +64,14 @@ beets`` if you run into permissions problems). To install without pip, download beets from `its PyPI page`_ and run ``python setup.py install`` in the directory therein. -.. _its PyPI page: http://pypi.python.org/pypi/beets#downloads -.. _pip: http://www.pip-installer.org/ +.. _its PyPI page: https://pypi.org/project/beets#downloads +.. _pip: https://pip.pypa.io The best way to upgrade beets to a new version is by running ``pip install -U beets``. You may want to follow `@b33ts`_ on Twitter to hear about progress on new versions. -.. _@b33ts: http://twitter.com/b33ts +.. _@b33ts: https://twitter.com/b33ts Installing on macOS 10.11 and Higher ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -87,7 +87,7 @@ If this happens, you can install beets for the current user only by typing ``~/Library/Python/3.6/bin`` to your ``$PATH``. .. _System Integrity Protection: https://support.apple.com/en-us/HT204899 -.. _Homebrew: http://brew.sh +.. _Homebrew: https://brew.sh Installing on Windows ^^^^^^^^^^^^^^^^^^^^^ @@ -122,10 +122,10 @@ Because I don't use Windows myself, I may have missed something. If you have trouble or you have more detail to contribute here, please direct it to `the mailing list`_. -.. _install Python: http://python.org/download/ +.. _install Python: https://python.org/download/ .. _beets.reg: https://github.com/beetbox/beets/blob/master/extra/beets.reg -.. _install pip: http://www.pip-installer.org/en/latest/installing.html#install-pip -.. _get-pip.py: https://raw.github.com/pypa/pip/master/contrib/get-pip.py +.. _install pip: https://pip.pypa.io/en/stable/installing/ +.. _get-pip.py: https://bootstrap.pypa.io/get-pip.py Configuring @@ -179,7 +179,7 @@ There are approximately six million other configuration options you can set here, including the directory and file naming scheme. See :doc:`/reference/config` for a full reference. -.. _YAML: http://yaml.org/ +.. _YAML: https://yaml.org/ Importing Your Library ---------------------- @@ -300,6 +300,6 @@ import`` gives more specific help about the ``import`` command. Please let me know what you think of beets via `the discussion board`_ or `Twitter`_. -.. _the mailing list: http://groups.google.com/group/beets-users -.. _the discussion board: http://discourse.beets.io -.. _twitter: http://twitter.com/b33ts +.. _the mailing list: https://groups.google.com/group/beets-users +.. _the discussion board: https://discourse.beets.io +.. _twitter: https://twitter.com/b33ts diff --git a/docs/guides/tagger.rst b/docs/guides/tagger.rst index b70857ca5..467d605a4 100644 --- a/docs/guides/tagger.rst +++ b/docs/guides/tagger.rst @@ -272,7 +272,7 @@ Before you jump into acoustic fingerprinting with both feet, though, give beets a try without it. You may be surprised at how well metadata-based matching works. -.. _Chromaprint: http://acoustid.org/chromaprint +.. _Chromaprint: https://acoustid.org/chromaprint Album Art, Lyrics, Genres and Such ---------------------------------- @@ -292,7 +292,7 @@ sure the album is present in `the MusicBrainz database`_. You can search on their site to make sure it's cataloged there. If not, anyone can edit MusicBrainz---so consider adding the data yourself. -.. _the MusicBrainz database: http://musicbrainz.org/ +.. _the MusicBrainz database: https://musicbrainz.org/ If you think beets is ignoring an album that's listed in MusicBrainz, please `file a bug report`_. @@ -305,5 +305,5 @@ I Hope That Makes Sense If we haven't made the process clear, please post on `the discussion board`_ and we'll try to improve this guide. -.. _the mailing list: http://groups.google.com/group/beets-users -.. _the discussion board: http://discourse.beets.io +.. _the mailing list: https://groups.google.com/group/beets-users +.. _the discussion board: https://discourse.beets.io diff --git a/docs/index.rst b/docs/index.rst index 43ba0526a..27fa4740b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,9 +18,9 @@ or `file a bug`_ in the issue tracker. Please let us know where you think this documentation can be improved. .. _beets: http://beets.io/ -.. _the mailing list: http://groups.google.com/group/beets-users +.. _the mailing list: https://groups.google.com/group/beets-users .. _file a bug: https://github.com/beetbox/beets/issues -.. _the discussion board: http://discourse.beets.io +.. _the discussion board: https://discourse.beets.io Contents -------- diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index 30a77d4b0..3221a07b3 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -41,9 +41,9 @@ To configure the plugin, make a ``absubmit:`` section in your configuration file - **extractor**: The absolute path to the `streaming_extractor_music`_ binary. Default: search for the program in your ``$PATH`` -.. _streaming_extractor_music: http://acousticbrainz.org/download -.. _FAQ: http://acousticbrainz.org/faq -.. _pip: http://www.pip-installer.org/ -.. _requests: http://docs.python-requests.org/en/master/ +.. _streaming_extractor_music: https://acousticbrainz.org/download +.. _FAQ: https://acousticbrainz.org/faq +.. _pip: https://pip.pypa.io +.. _requests: https://docs.python-requests.org/en/master/ .. _github: https://github.com/MTG/essentia .. _AcousticBrainz: https://acousticbrainz.org diff --git a/docs/plugins/acousticbrainz.rst b/docs/plugins/acousticbrainz.rst index 5bd514c64..7c24ffe0d 100644 --- a/docs/plugins/acousticbrainz.rst +++ b/docs/plugins/acousticbrainz.rst @@ -4,7 +4,7 @@ AcousticBrainz Plugin The ``acousticbrainz`` plugin gets acoustic-analysis information from the `AcousticBrainz`_ project. -.. _AcousticBrainz: http://acousticbrainz.org/ +.. _AcousticBrainz: https://acousticbrainz.org/ Enable the ``acousticbrainz`` plugin in your configuration (see :ref:`using-plugins`) and run it by typing:: diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index da77cd4cd..709dbb0a8 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -31,6 +31,6 @@ from MusicBrainz and other sources. If you have a Beatport ID or a URL for a release or track you want to tag, you can just enter one of the two at the "enter Id" prompt in the importer. -.. _requests: http://docs.python-requests.org/en/latest/ +.. _requests: https://docs.python-requests.org/en/latest/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib -.. _Beatport: http://beatport.com +.. _Beatport: https://beetport.com diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index c1a94e972..49563a73a 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -6,7 +6,7 @@ implements the MPD protocol, so it's compatible with all the great MPD clients out there. I'm using `Theremin`_, `gmpc`_, `Sonata`_, and `Ario`_ successfully. .. _Theremin: https://theremin.sigterm.eu/ -.. _gmpc: http://gmpc.wikia.com/wiki/Gnome_Music_Player_Client +.. _gmpc: https://gmpc.wikia.com/wiki/Gnome_Music_Player_Client .. _Sonata: http://sonata.berlios.de/ .. _Ario: http://ario-player.sourceforge.net/ @@ -29,8 +29,8 @@ You will also need the various GStreamer plugin packages to make everything work. See the :doc:`/plugins/chroma` documentation for more information on installing GStreamer plugins. -.. _GStreamer WinBuilds: http://www.gstreamer-winbuild.ylatuya.es/ -.. _Homebrew: http://mxcl.github.com/homebrew/ +.. _GStreamer WinBuilds: https://www.gstreamer-winbuild.ylatuya.es/ +.. _Homebrew: https://brew.sh Usage ----- @@ -44,7 +44,7 @@ Then, you can run BPD by invoking:: Fire up your favorite MPD client to start playing music. The MPD site has `a long list of available clients`_. Here are my favorites: -.. _a long list of available clients: http://mpd.wikia.com/wiki/Clients +.. _a long list of available clients: https://mpd.wikia.com/wiki/Clients * Linux: `gmpc`_, `Sonata`_ @@ -52,9 +52,9 @@ long list of available clients`_. Here are my favorites: * Windows: I don't know. Get in touch if you have a recommendation. -* iPhone/iPod touch: `MPoD`_ +* iPhone/iPod touch: `Rigelian`_ -.. _MPoD: http://www.katoemba.net/makesnosenseatall/mpod/ +.. _Rigelian: https://www.rigelian.net/ One nice thing about MPD's (and thus BPD's) client-server architecture is that the client can just as easily on a different computer from the server as it can @@ -109,7 +109,7 @@ behaviour to their MPD equivalents. BPD aims to look enough like MPD that it can interact with the ecosystem of clients, but doesn't try to be a fully-fledged MPD replacement in terms of its playback capabilities. -.. _the MPD protocol: http://www.musicpd.org/doc/protocol/ +.. _the MPD protocol: https://www.musicpd.org/doc/protocol/ These are some of the known differences between BPD and MPD: diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index 617d8cf69..1b86073b8 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -8,8 +8,8 @@ information at all (or have completely incorrect data). This plugin uses an open-source fingerprinting technology called `Chromaprint`_ and its associated Web service, called `Acoustid`_. -.. _Chromaprint: http://acoustid.org/chromaprint -.. _acoustid: http://acoustid.org/ +.. _Chromaprint: https://acoustid.org/chromaprint +.. _acoustid: https://acoustid.org/ Turning on fingerprinting can increase the accuracy of the autotagger---especially on files with very poor metadata---but it comes at a @@ -31,7 +31,7 @@ First, install pyacoustid itself. You can do this using `pip`_, like so:: $ pip install pyacoustid -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io Then, you will need to install `Chromaprint`_, either as a dynamic library or in the form of a command-line tool (``fpcalc``). @@ -45,7 +45,7 @@ The simplest way to get up and running, especially on Windows, is to means something like ``C:\\Program Files``. On OS X or Linux, put the executable somewhere like ``/usr/local/bin``. -.. _download: http://acoustid.org/chromaprint +.. _download: https://acoustid.org/chromaprint Installing the Library '''''''''''''''''''''' @@ -56,7 +56,7 @@ site has links to packages for major Linux distributions. If you use `Homebrew`_ on Mac OS X, you can install the library with ``brew install chromaprint``. -.. _Homebrew: http://mxcl.github.com/homebrew/ +.. _Homebrew: https://brew.sh/ You will also need a mechanism for decoding audio files supported by the `audioread`_ library: @@ -78,12 +78,12 @@ You will also need a mechanism for decoding audio files supported by the * On Windows, builds are provided by `GStreamer`_ .. _audioread: https://github.com/beetbox/audioread -.. _pyacoustid: http://github.com/beetbox/pyacoustid -.. _FFmpeg: http://ffmpeg.org/ -.. _MAD: http://spacepants.org/src/pymad/ -.. _pymad: http://www.underbit.com/products/mad/ -.. _Core Audio: http://developer.apple.com/technologies/mac/audio-and-video.html -.. _Gstreamer: http://gstreamer.freedesktop.org/ +.. _pyacoustid: https://github.com/beetbox/pyacoustid +.. _FFmpeg: https://ffmpeg.org/ +.. _MAD: https://spacepants.org/src/pymad/ +.. _pymad: https://www.underbit.com/products/mad/ +.. _Core Audio: https://developer.apple.com/technologies/mac/audio-and-video.html +.. _Gstreamer: https://gstreamer.freedesktop.org/ .. _PyGObject: https://wiki.gnome.org/Projects/PyGObject To decode audio formats (MP3, FLAC, etc.) with GStreamer, you'll need the @@ -132,4 +132,4 @@ Then, run ``beet submit``. (You can also provide a query to submit a subset of your library.) The command will use stored fingerprints if they're available; otherwise it will fingerprint each file before submitting it. -.. _get an API key: http://acoustid.org/api-key +.. _get an API key: https://acoustid.org/api-key diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 92545af30..59670c269 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -14,7 +14,7 @@ To use the ``convert`` plugin, first enable it in your configuration (see :ref:`using-plugins`). By default, the plugin depends on `FFmpeg`_ to transcode the audio, so you might want to install it. -.. _FFmpeg: http://ffmpeg.org +.. _FFmpeg: https://ffmpeg.org Usage @@ -170,6 +170,6 @@ can use the :doc:`/plugins/replaygain` to do this analysis. See the LAME options and a thorough discussion of MP3 encoding. .. _documentation: http://lame.sourceforge.net/using.php -.. _HydrogenAudio wiki: http://wiki.hydrogenaud.io/index.php?title=LAME -.. _gapless: http://wiki.hydrogenaud.io/index.php?title=Gapless_playback -.. _LAME: http://lame.sourceforge.net/ +.. _HydrogenAudio wiki: https://wiki.hydrogenaud.io/index.php?title=LAME +.. _gapless: https://wiki.hydrogenaud.io/index.php?title=Gapless_playback +.. _LAME: https://lame.sourceforge.net/ diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index a02b34590..622a085b4 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -4,7 +4,7 @@ Discogs Plugin The ``discogs`` plugin extends the autotagger's search capabilities to include matches from the `Discogs`_ database. -.. _Discogs: http://discogs.com +.. _Discogs: https://discogs.com Installation ------------ diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index 2a34a59e8..cc2fe6fc8 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -69,7 +69,7 @@ Note: ``compare_threshold`` option requires `ImageMagick`_, and ``maxwidth`` requires either `ImageMagick`_ or `Pillow`_. .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ .. _PHASH: http://www.fmwconcepts.com/misc_tests/perceptual_hash_test_results_510/ Manually Embedding and Extracting Art diff --git a/docs/plugins/embyupdate.rst b/docs/plugins/embyupdate.rst index d820f5c6b..626fafa9d 100644 --- a/docs/plugins/embyupdate.rst +++ b/docs/plugins/embyupdate.rst @@ -17,8 +17,8 @@ To use the ``embyupdate`` plugin you need to install the `requests`_ library wit With that all in place, you'll see beets send the "update" command to your Emby server every time you change your beets library. -.. _Emby: http://emby.media/ -.. _requests: http://docs.python-requests.org/en/latest/ +.. _Emby: https://emby.media/ +.. _requests: https://docs.python-requests.org/en/latest/ Configuration ------------- diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 4326ccb16..d712dfc8b 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -4,7 +4,7 @@ Export Plugin The ``export`` plugin lets you get data from the items and export the content as `JSON`_. -.. _JSON: http://www.json.org +.. _JSON: https://www.json.org Enable the ``export`` plugin (see :ref:`using-plugins` for help). Then, type ``beet export`` followed by a :doc:`query ` to get the data from your library. For example, run this:: diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 8af7f686a..f23fec765 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -11,7 +11,7 @@ To use the ``fetchart`` plugin, first enable it in your configuration (see The plugin uses `requests`_ to fetch album art from the Web. -.. _requests: http://docs.python-requests.org/en/latest/ +.. _requests: https://docs.python-requests.org/en/latest/ Fetching Album Art During Import -------------------------------- @@ -81,7 +81,7 @@ or `Pillow`_. .. _beets custom search engine: https://cse.google.com.au:443/cse/publicurl?cx=001442825323518660753:hrh5ch1gjzm .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ Here's an example that makes plugin select only images that contain ``front`` or ``back`` keywords in their filenames and prioritizes the iTunes source over @@ -135,7 +135,7 @@ On some versions of Windows, the program can be shadowed by a system-provided environment variable so that ImageMagick comes first or use Pillow instead. .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ .. _album-art-sources: @@ -191,7 +191,7 @@ Optionally, you can `define a custom search engine`_. Get your search engine's token and use it for your ``google_engine`` configuration option. The default engine searches the entire web for cover art. -.. _define a custom search engine: http://www.google.com/cse/all +.. _define a custom search engine: https://www.google.com/cse/all Note that the Google custom search API is limited to 100 queries per day. After that, the fetchart plugin will fall back on other declared data sources. diff --git a/docs/plugins/ftintitle.rst b/docs/plugins/ftintitle.rst index 8a080b3e2..66c9ecd69 100644 --- a/docs/plugins/ftintitle.rst +++ b/docs/plugins/ftintitle.rst @@ -41,4 +41,4 @@ your entire collection. Use the ``-d`` flag to remove featured artists (equivalent of the ``drop`` config option). -.. _MusicBrainz style: http://musicbrainz.org/doc/Style +.. _MusicBrainz style: https://musicbrainz.org/doc/Style diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index f95a6285d..1c8a8d417 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -107,7 +107,7 @@ Autotagger Extensions * :doc:`fromfilename`: Guess metadata for untagged tracks from their filenames. -.. _Discogs: http://www.discogs.com/ +.. _Discogs: https://www.discogs.com/ Metadata -------- @@ -136,7 +136,7 @@ Metadata * :doc:`zero`: Nullify fields by pattern or unconditionally. .. _KeyFinder: http://www.ibrahimshaath.co.uk/keyfinder/ -.. _streaming_extractor_music: http://acousticbrainz.org/download +.. _streaming_extractor_music: https://acousticbrainz.org/download Path Formats ------------ @@ -169,10 +169,10 @@ Interoperability * :doc:`thumbnails`: Get thumbnails with the cover art on your album folders. -.. _Emby: http://emby.media -.. _Plex: http://plex.tv -.. _Kodi: http://kodi.tv -.. _Sonos: http://sonos.com +.. _Emby: https://emby.media +.. _Plex: https://plex.tv +.. _Kodi: https://kodi.tv +.. _Sonos: https://sonos.com Miscellaneous ------------- @@ -194,14 +194,14 @@ Miscellaneous * :doc:`mbcollection`: Maintain your MusicBrainz collection list. * :doc:`mbsubmit`: Print an album's tracks in a MusicBrainz-friendly format. * :doc:`missing`: List missing tracks. -* `mstream`_: A music streaming server + webapp that can be used alongside beets. +* `mstream`_: A music streaming server + webapp that can be used alongside beets. * :doc:`random`: Randomly choose albums and tracks from your library. * :doc:`spotify`: Create Spotify playlists from the Beets library. * :doc:`types`: Declare types for flexible attributes. * :doc:`web`: An experimental Web-based GUI for beets. -.. _MPD: http://www.musicpd.org/ -.. _MPD clients: http://mpd.wikia.com/wiki/Clients +.. _MPD: https://www.musicpd.org/ +.. _MPD clients: https://mpd.wikia.com/wiki/Clients .. _mstream: https://github.com/IrosTheBeggar/mStream .. _other-plugins: diff --git a/docs/plugins/info.rst b/docs/plugins/info.rst index 238a957ff..d628fb4ad 100644 --- a/docs/plugins/info.rst +++ b/docs/plugins/info.rst @@ -42,4 +42,4 @@ Additional command-line options include: * ``--keys-only`` or ``-k``: Show the name of the tags without the values. .. _id3v2: http://id3v2.sourceforge.net -.. _mp3info: http://www.ibiblio.org/mp3info/ +.. _mp3info: https://www.ibiblio.org/mp3info/ diff --git a/docs/plugins/ipfs.rst b/docs/plugins/ipfs.rst index 141143ae7..5bf8ca906 100644 --- a/docs/plugins/ipfs.rst +++ b/docs/plugins/ipfs.rst @@ -4,7 +4,7 @@ IPFS Plugin The ``ipfs`` plugin makes it easy to share your library and music with friends. The plugin uses `ipfs`_ for storing the library and file content. -.. _ipfs: http://ipfs.io/ +.. _ipfs: https://ipfs.io/ Installation ------------ diff --git a/docs/plugins/keyfinder.rst b/docs/plugins/keyfinder.rst index 856939ecc..878830f29 100644 --- a/docs/plugins/keyfinder.rst +++ b/docs/plugins/keyfinder.rst @@ -29,4 +29,4 @@ configuration file. The available options are: `initial_key` value. Default: ``no``. -.. _KeyFinder: http://www.ibrahimshaath.co.uk/keyfinder/ +.. _KeyFinder: https://www.ibrahimshaath.co.uk/keyfinder/ diff --git a/docs/plugins/kodiupdate.rst b/docs/plugins/kodiupdate.rst index a1ec04775..e60f503f2 100644 --- a/docs/plugins/kodiupdate.rst +++ b/docs/plugins/kodiupdate.rst @@ -26,8 +26,8 @@ In Kodi's interface, navigate to System/Settings/Network/Services and choose "Al With that all in place, you'll see beets send the "update" command to your Kodi host every time you change your beets library. -.. _Kodi: http://kodi.tv/ -.. _requests: http://docs.python-requests.org/en/latest/ +.. _Kodi: https://kodi.tv/ +.. _requests: https://docs.python-requests.org/en/latest/ Configuration ------------- diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index c3d5f97ec..5fcdd2254 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -7,8 +7,8 @@ importing and autotagging music, beets does not assign a genre. The to your albums and items. .. _does not contain genre information: - http://musicbrainz.org/doc/General_FAQ#Why_does_MusicBrainz_not_support_genre_information.3F -.. _Last.fm: http://last.fm/ + https://musicbrainz.org/doc/General_FAQ#Why_does_MusicBrainz_not_support_genre_information.3F +.. _Last.fm: https://last.fm/ Installation ------------ @@ -34,7 +34,7 @@ The genre list file should contain one genre per line. Blank lines are ignored. For the curious, the default genre list is generated by a `script that scrapes Wikipedia`_. -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io .. _pylast: https://github.com/pylast/pylast .. _script that scrapes Wikipedia: https://gist.github.com/1241307 .. _internal whitelist: https://raw.githubusercontent.com/beetbox/beets/master/beetsplug/lastgenre/genres.txt @@ -72,7 +72,7 @@ nothing would ever be matched to a more generic node since all the specific subgenres are in the whitelist to begin with. -.. _YAML: http://www.yaml.org/ +.. _YAML: https://www.yaml.org/ .. _tree of nested genre names: https://raw.githubusercontent.com/beetbox/beets/master/beetsplug/lastgenre/genres-tree.yaml diff --git a/docs/plugins/lastimport.rst b/docs/plugins/lastimport.rst index 8006d6bbb..1c12b8616 100644 --- a/docs/plugins/lastimport.rst +++ b/docs/plugins/lastimport.rst @@ -6,7 +6,7 @@ library into beets' database. You can later create :doc:`smart playlists ` by querying ``play_count`` and do other fun stuff with this field. -.. _Last.fm: http://last.fm +.. _Last.fm: https://last.fm Installation ------------ @@ -23,7 +23,7 @@ Next, add your Last.fm username to your beets configuration file:: lastfm: user: beetsfanatic -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io .. _pylast: https://github.com/pylast/pylast Importing Play Counts diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 799bd0325..fac07ad87 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -5,9 +5,9 @@ The ``lyrics`` plugin fetches and stores song lyrics from databases on the Web. Namely, the current version of the plugin uses `Lyric Wiki`_, `Musixmatch`_, `Genius.com`_, and, optionally, the Google custom search API. -.. _Lyric Wiki: http://lyrics.wikia.com/ +.. _Lyric Wiki: https://lyrics.wikia.com/ .. _Musixmatch: https://www.musixmatch.com/ -.. _Genius.com: http://genius.com/ +.. _Genius.com: https://genius.com/ Fetch Lyrics During Import @@ -26,7 +26,7 @@ already have them. The lyrics will be stored in the beets database. If the ``import.write`` config option is on, then the lyrics will also be written to the files' tags. -.. _requests: http://docs.python-requests.org/en/latest/ +.. _requests: https://docs.python-requests.org/en/latest/ Configuration @@ -105,11 +105,11 @@ A minimal ``conf.py`` and ``index.rst`` files are created the first time the command is run. They are not overwritten on subsequent runs, so you can safely modify these files to customize the output. -.. _Sphinx: http://www.sphinx-doc.org/ +.. _Sphinx: https://www.sphinx-doc.org/ .. _reStructuredText: http://docutils.sourceforge.net/rst.html Sphinx supports various `builders -`_, but here are a +`_, but here are a few suggestions. * Build an HTML version:: @@ -148,13 +148,13 @@ Optionally, you can `define a custom search engine`_. Get your search engine's token and use it for your ``google_engine_ID`` configuration option. By default, beets use a list of sources known to be scrapeable. -.. _define a custom search engine: http://www.google.com/cse/all +.. _define a custom search engine: https://www.google.com/cse/all Note that the Google custom search API is limited to 100 queries per day. After that, the lyrics plugin will fall back on other declared data sources. -.. _pip: http://www.pip-installer.org/ -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ +.. _pip: https://pip.pypa.io +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ Activate Genius Lyrics ---------------------- diff --git a/docs/plugins/mbcollection.rst b/docs/plugins/mbcollection.rst index 803d34904..113855bce 100644 --- a/docs/plugins/mbcollection.rst +++ b/docs/plugins/mbcollection.rst @@ -4,7 +4,7 @@ MusicBrainz Collection Plugin The ``mbcollection`` plugin lets you submit your catalog to MusicBrainz to maintain your `music collection`_ list there. -.. _music collection: http://musicbrainz.org/doc/Collections +.. _music collection: https://musicbrainz.org/doc/Collections To begin, just enable the ``mbcollection`` plugin in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/mbsubmit.rst b/docs/plugins/mbsubmit.rst index 5c13375ba..70e14662d 100644 --- a/docs/plugins/mbsubmit.rst +++ b/docs/plugins/mbsubmit.rst @@ -5,7 +5,7 @@ The ``mbsubmit`` plugin provides an extra prompt choice during an import session that prints the tracks of the current album in a format that is parseable by MusicBrainz's `track parser`_. -.. _track parser: http://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings +.. _track parser: https://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings Usage ----- @@ -15,7 +15,7 @@ and select the ``Print tracks`` choice which is by default displayed when no strong recommendations are found for the album:: No matching release found for 3 tracks. - For help, see: http://beets.readthedocs.org/en/latest/faq.html#nomatch + For help, see: https://beets.readthedocs.org/en/latest/faq.html#nomatch [U]se as-is, as Tracks, Group albums, Skip, Enter search, enter Id, aBort, Print tracks? p 01. An Obscure Track - An Obscure Artist (3:37) @@ -23,7 +23,7 @@ strong recommendations are found for the album:: 03. The Third Track - Another Obscure Artist (3:02) No matching release found for 3 tracks. - For help, see: http://beets.readthedocs.org/en/latest/faq.html#nomatch + For help, see: https://beets.readthedocs.org/en/latest/faq.html#nomatch [U]se as-is, as Tracks, Group albums, Skip, Enter search, enter Id, aBort, Print tracks? diff --git a/docs/plugins/metasync.rst b/docs/plugins/metasync.rst index 6703d3c19..691550595 100644 --- a/docs/plugins/metasync.rst +++ b/docs/plugins/metasync.rst @@ -22,7 +22,7 @@ Enable the ``metasync`` plugin in your configuration (see To synchronize with Amarok, you'll need the `dbus-python`_ library. There are packages for most major Linux distributions. -.. _dbus-python: http://dbus.freedesktop.org/releases/dbus-python/ +.. _dbus-python: https://dbus.freedesktop.org/releases/dbus-python/ Configuration diff --git a/docs/plugins/mpdstats.rst b/docs/plugins/mpdstats.rst index 2e5e78c36..b769e7468 100644 --- a/docs/plugins/mpdstats.rst +++ b/docs/plugins/mpdstats.rst @@ -9,7 +9,7 @@ habits from `MPD`_. It collects the following information about tracks: * last_played: UNIX timestamp when you last played this track. * rating: A rating based on *play_count* and *skip_count*. -.. _MPD: http://www.musicpd.org/ +.. _MPD: https://www.musicpd.org/ Installing Dependencies ----------------------- @@ -23,7 +23,7 @@ Install the library from `pip`_, like so:: Add the ``mpdstats`` plugin to your configuration (see :ref:`using-plugins`). -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io Usage ----- diff --git a/docs/plugins/mpdupdate.rst b/docs/plugins/mpdupdate.rst index 7ac647536..01a6a9fe7 100644 --- a/docs/plugins/mpdupdate.rst +++ b/docs/plugins/mpdupdate.rst @@ -4,7 +4,7 @@ MPDUpdate Plugin ``mpdupdate`` is a very simple plugin for beets that lets you automatically update `MPD`_'s index whenever you change your beets library. -.. _MPD: http://www.musicpd.org/ +.. _MPD: https://www.musicpd.org/ To use ``mpdupdate`` plugin, enable it in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index 4ac047660..f9312280a 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -21,11 +21,11 @@ To use the ``plexupdate`` plugin you need to install the `requests`_ library wit pip install requests -With that all in place, you'll see beets send the "update" command to your Plex +With that all in place, you'll see beets send the "update" command to your Plex server every time you change your beets library. -.. _Plex: http://plex.tv/ -.. _requests: http://docs.python-requests.org/en/latest/ +.. _Plex: https://plex.tv/ +.. _requests: https://docs.python-requests.org/en/latest/ .. _documentation about tokens: https://support.plex.tv/hc/en-us/articles/204059436-Finding-your-account-token-X-Plex-Token Configuration diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 825f279e2..728f1846e 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -4,7 +4,7 @@ ReplayGain Plugin This plugin adds support for `ReplayGain`_, a technique for normalizing audio playback levels. -.. _ReplayGain: http://wiki.hydrogenaudio.org/index.php?title=ReplayGain +.. _ReplayGain: https://wiki.hydrogenaudio.org/index.php?title=ReplayGain Installation @@ -27,7 +27,7 @@ install GStreamer and plugins for compatibility with your audio files. You will need at least GStreamer 1.0 and `PyGObject 3.x`_ (a.k.a. ``python-gi``). .. _PyGObject 3.x: https://pygobject.readthedocs.io/en/latest/ -.. _GStreamer: http://gstreamer.freedesktop.org/ +.. _GStreamer: https://gstreamer.freedesktop.org/ Then, enable the ``replaygain`` plugin (see :ref:`using-plugins`) and specify the GStreamer backend by adding this to your configuration file:: @@ -47,8 +47,8 @@ command-line tool or the `aacgain`_ fork thereof. Here are some hints: * On Windows, download and install the original `mp3gain`_. .. _mp3gain: http://mp3gain.sourceforge.net/download.php -.. _aacgain: http://aacgain.altosdesign.com -.. _Homebrew: http://mxcl.github.com/homebrew/ +.. _aacgain: https://aacgain.altosdesign.com +.. _Homebrew: https://brew.sh Then, enable the plugin (see :ref:`using-plugins`) and specify the "command" backend in your configuration file:: diff --git a/docs/plugins/sonosupdate.rst b/docs/plugins/sonosupdate.rst index 97a13bd07..cae69d554 100644 --- a/docs/plugins/sonosupdate.rst +++ b/docs/plugins/sonosupdate.rst @@ -14,5 +14,5 @@ To use the ``sonosupdate`` plugin you need to install the `soco`_ library with:: With that all in place, you'll see beets send the "update" command to your Sonos controller every time you change your beets library. -.. _Sonos: http://sonos.com/ +.. _Sonos: https://sonos.com/ .. _soco: http://python-soco.com diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index 3f4c6c43d..5d6ae8f47 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -31,8 +31,8 @@ Here's an example:: $ beet spotify "In The Lonely Hour" Processing 14 tracks... - http://open.spotify.com/track/19w0OHr8SiZzRhjpnjctJ4 - http://open.spotify.com/track/3PRLM4FzhplXfySa4B7bxS + https://open.spotify.com/track/19w0OHr8SiZzRhjpnjctJ4 + https://open.spotify.com/track/3PRLM4FzhplXfySa4B7bxS [...] Command-line options include: diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index daf4a0cfb..2d9331b7c 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -4,7 +4,7 @@ SubsonicUpdate Plugin ``subsonicupdate`` is a very simple plugin for beets that lets you automatically update `Subsonic`_'s index whenever you change your beets library. -.. _Subsonic: http://www.subsonic.org +.. _Subsonic: https://www.subsonic.org To use ``subsonicupdate`` plugin, enable it in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/thumbnails.rst b/docs/plugins/thumbnails.rst index c2a28d091..0f46e04e8 100644 --- a/docs/plugins/thumbnails.rst +++ b/docs/plugins/thumbnails.rst @@ -13,7 +13,7 @@ as the :doc:`/plugins/fetchart`. You'll need 2 additional python packages: `ImageMagick`_ or `Pillow`_. .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ Configuration ------------- diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 35287acc8..d3ae668ce 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -19,13 +19,13 @@ The Web interface depends on `Flask`_. To get it, just run ``pip install flask``. Then enable the ``web`` plugin in your configuration (see :ref:`using-plugins`). -.. _Flask: http://flask.pocoo.org/ +.. _Flask: https://flask.pocoo.org/ If you need CORS (it's disabled by default---see :ref:`web-cors`, below), then you also need `flask-cors`_. Just type ``pip install flask-cors``. .. _flask-cors: https://github.com/CoryDolphin/flask-cors -.. _CORS: http://en.wikipedia.org/wiki/Cross-origin_resource_sharing +.. _CORS: https://en.wikipedia.org/wiki/Cross-origin_resource_sharing Run the Server @@ -78,8 +78,8 @@ The Web backend is built using a simple REST+JSON API with the excellent `Flask`_ library. The frontend is a single-page application written with `Backbone.js`_. This allows future non-Web clients to use the same backend API. -.. _Flask: http://flask.pocoo.org/ -.. _Backbone.js: http://backbonejs.org +.. _Flask: https://flask.pocoo.org/ +.. _Backbone.js: https://backbonejs.org Eventually, to make the Web player really viable, we should use a Flash fallback for unsupported formats/browsers. There are a number of options for this: @@ -88,8 +88,8 @@ for unsupported formats/browsers. There are a number of options for this: * `html5media`_ * `MediaElement.js`_ -.. _audio.js: http://kolber.github.com/audiojs/ -.. _html5media: http://html5media.info/ +.. _audio.js: https://kolber.github.io/audiojs/ +.. _html5media: https://html5media.info/ .. _MediaElement.js: http://mediaelementjs.com/ .. _web-cors: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 7b9e9eb72..2fc7c7b31 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -217,7 +217,7 @@ beatles`` prints out the number of tracks on each Beatles album. In Unix shells, remember to enclose the template argument in single quotes to avoid environment variable expansion. -.. _xargs: http://en.wikipedia.org/wiki/Xargs +.. _xargs: https://en.wikipedia.org/wiki/Xargs .. _remove-cmd: @@ -498,6 +498,6 @@ defines some bash-specific functions to make this work without errors:: See Also -------- - ``http://beets.readthedocs.org/`` + ``https://beets.readthedocs.org/`` :manpage:`beetsconfig(5)` diff --git a/docs/reference/config.rst b/docs/reference/config.rst index a96e3dfb3..687f6c3f9 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -31,7 +31,7 @@ In YAML, you will need to use spaces (not tabs!) to indent some lines. If you have questions about more sophisticated syntax, take a look at the `YAML`_ documentation. -.. _YAML: http://yaml.org/ +.. _YAML: https://yaml.org/ The rest of this page enumerates the dizzying litany of configuration options available in beets. You might also want to see an @@ -167,7 +167,7 @@ equivalent to wrapping all your path templates in the ``%asciify{}`` Default: ``no``. -.. _unidecode module: http://pypi.python.org/pypi/Unidecode +.. _unidecode module: https://pypi.org/project/Unidecode .. _art-filename: @@ -314,7 +314,7 @@ standard output. It's also used to read messages from the standard input. By default, this is determined automatically from the locale environment variables. -.. _known to python: http://docs.python.org/2/library/codecs.html#standard-encodings +.. _known to python: https://docs.python.org/2/library/codecs.html#standard-encodings .. _clutter: @@ -688,7 +688,7 @@ to one request per second. .. _your own MusicBrainz database: https://musicbrainz.org/doc/MusicBrainz_Server/Setup .. _main server: https://musicbrainz.org/ -.. _limited: http://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting +.. _limited: https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting .. _Building search indexes: https://musicbrainz.org/doc/MusicBrainz_Server/Setup#Building_search_indexes .. _searchlimit: @@ -981,6 +981,6 @@ Here's an example file:: See Also -------- - ``http://beets.readthedocs.org/`` + ``https://beets.readthedocs.org/`` :manpage:`beet(1)` diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index 79998a9e1..9213cae4b 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -23,7 +23,7 @@ a dollars sign. As with `Python template strings`_, ``${title}`` is equivalent to ``$title``; you can use this if you need to separate a field name from the text that follows it. -.. _Python template strings: http://docs.python.org/library/string.html#template-strings +.. _Python template strings: https://docs.python.org/library/string.html#template-strings A Note About Artists @@ -38,7 +38,7 @@ tracks in a "Talking Heads" directory and one in a "Tom Tom Club" directory. You probably don't want that! So use ``$albumartist``. .. _Stop Making Sense: - http://musicbrainz.org/release/798dcaab-0f1a-4f02-a9cb-61d5b0ddfd36.html + https://musicbrainz.org/release/798dcaab-0f1a-4f02-a9cb-61d5b0ddfd36.html As a convenience, however, beets allows ``$albumartist`` to fall back to the value for ``$artist`` and vice-versa if one tag is present but the other is not. @@ -89,8 +89,8 @@ These functions are built in to beets: without ``$``. Note that this doesn't work with built-in :ref:`itemfields`, as they are always defined. -.. _unidecode module: http://pypi.python.org/pypi/Unidecode -.. _strftime: http://docs.python.org/2/library/time.html#time.strftime +.. _unidecode module: https://pypi.org/project/Unidecode +.. _strftime: https://docs.python.org/3/library/time.html#time.strftime Plugins can extend beets with more template functions (see :ref:`templ_plugins`). @@ -228,8 +228,8 @@ Ordinary metadata: * disctitle * encoder -.. _artist credit: http://wiki.musicbrainz.org/Artist_Credit -.. _list of type names: http://musicbrainz.org/doc/Release_Group/Type +.. _artist credit: https://wiki.musicbrainz.org/Artist_Credit +.. _list of type names: https://musicbrainz.org/doc/Release_Group/Type Audio information: diff --git a/docs/reference/query.rst b/docs/reference/query.rst index d103d9aec..5c16f610b 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -122,7 +122,7 @@ expressions, such as ``()[]|``. To type those characters, you'll need to escape them (e.g., with backslashes or quotation marks, depending on your shell). -.. _Python's built-in implementation: http://docs.python.org/library/re.html +.. _Python's built-in implementation: https://docs.python.org/library/re.html .. _numericquery: diff --git a/extra/_beet b/extra/_beet index 56c86d036..a8c9083be 100644 --- a/extra/_beet +++ b/extra/_beet @@ -1,6 +1,6 @@ #compdef beet -# zsh completion for beets music library manager and MusicBrainz tagger: http://beets.radbox.org/ +# zsh completion for beets music library manager and MusicBrainz tagger: http://beets.io/ # Default values for BEETS_LIBRARY & BEETS_CONFIG needed for the cache checking function. # They will be updated under the assumption that the config file is in the same directory as the library. diff --git a/setup.py b/setup.py index 7c209f019..30db5272d 100755 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ setup( # badfiles: mp3val and flac # bpd: python-gi and GStreamer 1.0+ # embedart: ImageMagick - # absubmit: extractor binary from http://acousticbrainz.org/download + # absubmit: extractor binary from https://acousticbrainz.org/download # keyfinder: KeyFinder # replaygain: python-gi and GStreamer 1.0+ or mp3gain/aacgain # or Python Audio Tools diff --git a/test/test_art.py b/test/test_art.py index 556222f48..f4b3a6e62 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -159,9 +159,9 @@ class FSArtTest(UseThePlugin): class CombinedTest(FetchImageHelper, UseThePlugin): ASIN = 'xxxx' MBID = 'releaseid' - AMAZON_URL = 'http://images.amazon.com/images/P/{0}.01.LZZZZZZZ.jpg' \ + AMAZON_URL = 'https://images.amazon.com/images/P/{0}.01.LZZZZZZZ.jpg' \ .format(ASIN) - AAO_URL = 'http://www.albumart.org/index_detail.php?asin={0}' \ + AAO_URL = 'https://www.albumart.org/index_detail.php?asin={0}' \ .format(ASIN) CAA_URL = 'coverartarchive.org/release/{0}/front' \ .format(MBID) @@ -240,7 +240,7 @@ class CombinedTest(FetchImageHelper, UseThePlugin): class AAOTest(UseThePlugin): ASIN = 'xxxx' - AAO_URL = 'http://www.albumart.org/index_detail.php?asin={0}'.format(ASIN) + AAO_URL = 'https://www.albumart.org/index_detail.php?asin={0}'.format(ASIN) def setUp(self): super(AAOTest, self).setUp() From da8f4a294e5f72a0a1dd25a1b601e181b3a34af9 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 4 Jun 2019 12:01:40 +1000 Subject: [PATCH 328/339] Add repology badge This is just to advertise that beets is available in distros. The badge links to a list of distro packages for beets and the current versions they have available, which is useful for users and contributors. --- README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6b4ebb4fa..8f8705abd 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,9 @@ .. image:: https://travis-ci.org/beetbox/beets.svg?branch=master :target: https://travis-ci.org/beetbox/beets +.. image:: https://repology.org/badge/tiny-repos/beets.svg + :target: https://repology.org/project/beets/versions + beets ===== @@ -78,10 +81,12 @@ shockingly simple if you know a little Python. Install ------- -You can install beets by typing ``pip install beets``. Then check out the -`Getting Started`_ guide. +You can install beets by typing ``pip install beets``. +Beets has also been packaged in the `software repositories`_ of several distributions. +Check out the `Getting Started`_ guide for more information. .. _Getting Started: https://beets.readthedocs.org/page/guides/main.html +.. _software repositories: https://repology.org/project/beets/versions Contribute ---------- From 0e65800fbc1485e84ea74091b9a4f03f2b8040ba Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 4 Jun 2019 13:18:36 +1000 Subject: [PATCH 329/339] Expand library API docs --- docs/dev/api.rst | 221 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 177 insertions(+), 44 deletions(-) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index d9e68481d..09b9b2f85 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -1,12 +1,11 @@ -API Documentation -================= +Library Database API +==================== .. currentmodule:: beets.library -This page describes the internal API of beets' core. It's a work in -progress---since beets is an application first and a library second, its API -has been mainly undocumented until recently. Please file bugs if you run -across incomplete or incorrect docs here. +This page describes the internal API of beets' core database features. It +doesn't exhaustively document the API, but is aimed at giving an overview of +the architecture to orient anyone who wants to dive into the code. The :class:`Library` object is the central repository for data in beets. It represents a database containing songs, which are :class:`Item` instances, and @@ -15,8 +14,24 @@ groups of items, which are :class:`Album` instances. The Library Class ----------------- +The :class:`Library` is typically instantiated as a singleton. A single +invocation of beets usually has only one :class:`Library`. It's powered by +:class:`dbcore.Database` under the hood, which handles the `SQLite`_ +abstraction, something like a very minimal `ORM`_. The library is also +responsible for handling queries to retrieve stored objects. + .. autoclass:: Library(path, directory[, path_formats[, replacements]]) + .. automethod:: __init__ + + You can add new items or albums to the library: + + .. automethod:: add + + .. automethod:: add_album + + And there are methods for querying the database: + .. automethod:: items .. automethod:: albums @@ -25,60 +40,178 @@ The Library Class .. automethod:: get_album - .. automethod:: add - - .. automethod:: add_album + Any modifications must go through a :class:`Transaction` which you get can + using this method: .. automethod:: transaction +.. _SQLite: http://sqlite.org/ +.. _ORM: http://en.wikipedia.org/wiki/Object-relational_mapping + + +Model Classes +------------- + +The two model entities in beets libraries, :class:`Item` and :class:`Album`, +share a base class, :class:`LibModel`, that provides common functionality and +ORM-like abstraction. + +Model base +'''''''''' + +Models use dirty-flags to track when the object's metadata goes out of +sync with the database. The dirty dictionary maps field names to booleans +indicating whether the field has been written since the object was last +synchronized (via load or store) with the database. + +.. autoclass:: LibModel + + .. automethod:: all_keys + + .. automethod:: __init__ + + .. autoattribute:: _types + + .. autoattribute:: _fields + + There are CRUD-like methods for interacting with the database: + + .. automethod:: store + + .. automethod:: load + + .. automethod:: remove + + .. automethod:: add + + The fields model classes can be accessed using attributes (dots, as in + ``item.artist``) or items (brackets, as in ``item['artist']``). + The base class :class:`dbcore.Model` has a ``dict``-like interface, so + normal the normal mapping API is supported: + + .. automethod:: keys + + .. automethod:: update + + .. automethod:: items + + .. automethod:: get + +Item +'''' + +Each :class:`Item` object represents a song or track. (We use the more generic +term item because, one day, beets might support non-music media.) An item can +either be purely abstract, in which case it's just a bag of metadata fields, +or it can have an associated file (indicated by ``item.path``). + +In terms of the underlying SQLite database, items are backed by a single table +called items with one column per metadata fields. The metadata fields currently +in use are listed in ``library.py`` in ``Item._fields``. + +To read and write a file's tags, we use the `MediaFile`_ library. +To make changes to either the database or the tags on a file, you +update an item's fields (e.g., ``item.title = "Let It Be"``) and then call +``item.write()``. + +.. _MediaFile: http://mediafile.readthedocs.io/ + +.. autoclass:: Item + + .. automethod:: __init__ + + .. automethod:: from_path + + .. automethod:: get_album + + .. automethod:: destination + + The methods ``read()`` and ``write()`` are complementary: one reads a + file's tags and updates the item's metadata fields accordingly while the + other takes the item's fields and writes them to the file's tags. + + .. automethod:: read + + .. automethod:: write + + .. automethod:: try_write + + .. automethod:: try_sync + + The :class:`Item` class supplements the normal model interface so that they + interacting with the filesystem as well: + + .. automethod:: move + + .. automethod:: remove + + Items also track their modification times (mtimes) to help detect when they + become out of sync with on-disk metadata. + + .. automethod:: current_mtime + +Album +''''' + +An :class:`Album` is a collection of Items in the database. Every item in the +database has either zero or one associated albums (accessible via +``item.album_id``). An item that has no associated album is called a +singleton. + +An :class:`Album` object keeps track of album-level metadata, which is (mostly) +a subset of the track-level metadata. The album-level metadata fields are +listed in ``Album._fields``. +For those fields that are both item-level and album-level (e.g., ``year`` or +``albumartist``), every item in an album should share the same value. Albums +use an SQLite table called ``albums``, in which each column is an album +metadata field. + +.. autoclass:: Album + + .. automethod:: __init__ + + .. automethod:: item_dir + + To get or change an album's metadata, use its fields (e.g., + ``print(album.year)`` or ``album.year = 2012``). Changing fields in this + way updates the album itself and also changes the same field in all + associated items: + + .. autoattribute:: item_keys + + .. automethod:: store + + .. automethod:: try_sync + + .. automethod:: move + + .. automethod:: remove + + Albums also manage album art, image files that are associated with each + album: + + .. automethod:: set_art + + .. automethod:: move_art + + .. automethod:: art_destination + Transactions '''''''''''' The :class:`Library` class provides the basic methods necessary to access and manipulate its contents. To perform more complicated operations atomically, or to interact directly with the underlying SQLite database, you must use a -*transaction*. For example:: +*transaction* (see this `blog post`_ for motivation). For example:: lib = Library() with lib.transaction() as tx: items = lib.items(query) lib.add_album(list(items)) +.. _blog post: http://beets.io/blog/sqlite-nightmare.html + .. currentmodule:: beets.dbcore.db .. autoclass:: Transaction :members: - -Model Classes -------------- - -The two model entities in beets libraries, :class:`Item` and :class:`Album`, -share a base class, :class:`Model`, that provides common functionality and -ORM-like abstraction. - -The fields model classes can be accessed using attributes (dots, as in -``item.artist``) or items (brackets, as in ``item['artist']``). The -:class:`Model` base class provides some methods that resemble `dict` -objects. - -Model base -'''''''''' - -.. currentmodule:: beets.dbcore - -.. autoclass:: Model - :members: - -Item -'''' - -.. currentmodule:: beets.library - -.. autoclass:: Item - :members: - -Album -''''' - -.. autoclass:: Album - :members: From 984aa223c69b8c0831231919647d3bba96cb4839 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:03:36 +1000 Subject: [PATCH 330/339] docs: highlight model field API --- docs/dev/api.rst | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index 09b9b2f85..62dfd392c 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -45,16 +45,22 @@ responsible for handling queries to retrieve stored objects. .. automethod:: transaction -.. _SQLite: http://sqlite.org/ -.. _ORM: http://en.wikipedia.org/wiki/Object-relational_mapping +.. _SQLite: https://sqlite.org/ +.. _ORM: https://en.wikipedia.org/wiki/Object-relational_mapping Model Classes ------------- The two model entities in beets libraries, :class:`Item` and :class:`Album`, -share a base class, :class:`LibModel`, that provides common functionality and -ORM-like abstraction. +share a base class, :class:`LibModel`, that provides common functionality. That +class itself specialises :class:`dbcore.Model` which provides an ORM-like +abstraction. + +To get or change the metadata of a model (an item or album), either access its +attributes (e.g., ``print(album.year)`` or ``album.year = 2012``) or use the +``dict``-like interface (e.g. ``item['artist']``). + Model base '''''''''' @@ -84,8 +90,6 @@ synchronized (via load or store) with the database. .. automethod:: add - The fields model classes can be accessed using attributes (dots, as in - ``item.artist``) or items (brackets, as in ``item['artist']``). The base class :class:`dbcore.Model` has a ``dict``-like interface, so normal the normal mapping API is supported: @@ -114,7 +118,7 @@ To make changes to either the database or the tags on a file, you update an item's fields (e.g., ``item.title = "Let It Be"``) and then call ``item.write()``. -.. _MediaFile: http://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/ .. autoclass:: Item @@ -157,6 +161,8 @@ An :class:`Album` is a collection of Items in the database. Every item in the database has either zero or one associated albums (accessible via ``item.album_id``). An item that has no associated album is called a singleton. +Changing fields on an album (e.g. ``album.year = 2012``) updates the album +itself and also changes the same field in all associated items. An :class:`Album` object keeps track of album-level metadata, which is (mostly) a subset of the track-level metadata. The album-level metadata fields are @@ -172,10 +178,8 @@ metadata field. .. automethod:: item_dir - To get or change an album's metadata, use its fields (e.g., - ``print(album.year)`` or ``album.year = 2012``). Changing fields in this - way updates the album itself and also changes the same field in all - associated items: + Albums extend the normal model interface to also forward changes to their + items: .. autoattribute:: item_keys From e27c6e480b4920ce4a7aa8c966e94fa3659a57e0 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:10:10 +1000 Subject: [PATCH 331/339] docs: add query API reference --- docs/dev/api.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index 62dfd392c..6ccdfd57e 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -219,3 +219,32 @@ to interact directly with the underlying SQLite database, you must use a .. autoclass:: Transaction :members: + + +Queries +------- + +To access albums and items in a library, we use :doc:`/reference/query`. +In beets, the :class:`Query` abstract base class represents a criterion that +matches items or albums in the database. +Every subclass of :class:`Query` must implement two methods, which implement +two different ways of identifying matching items/albums. + +The ``clause()`` method should return an SQLite ``WHERE`` clause that matches +appropriate albums/items. This allows for efficient batch queries. +Correspondingly, the ``match(item)`` method should take an :class:`Item` object +and return a boolean, indicating whether or not a specific item matches the +criterion. This alternate implementation allows clients to determine whether +items that have already been fetched from the database match the query. + +There are many different types of queries. Just as an example, +:class:`FieldQuery` determines whether a certain field matches a certain value +(an equality query). +:class:`AndQuery` (like its abstract superclass, :class:`CollectionQuery`) +takes a set of other query objects and bundles them together, matching only +albums/items that match all constituent queries. + +Beets has a human-writable plain-text query syntax that can be parsed into +:class:`Query` objects. Calling ``AndQuery.from_strings`` parses a list of +query parts into a query object that can then be used with :class:`Library` +objects. From 918024a4658a82828b1e83a7e6a093d6c0f31fc9 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:16:12 +1000 Subject: [PATCH 332/339] docs: document mtime management --- docs/dev/api.rst | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index 6ccdfd57e..ec81c169b 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -120,6 +120,38 @@ update an item's fields (e.g., ``item.title = "Let It Be"``) and then call .. _MediaFile: https://mediafile.readthedocs.io/ +Items also track their modification times (mtimes) to help detect when they +become out of sync with on-disk metadata, mainly to speed up the +:ref:`update-cmd` (which needs to check whether the database is in sync with +the filesystem). This feature turns out to be sort of complicated. + +For any :class:`Item`, there are two mtimes: the on-disk mtime (maintained by +the OS) and the database mtime (maintained by beets). Correspondingly, there is +on-disk metadata (ID3 tags, for example) and DB metadata. The goal with the +mtime is to ensure that the on-disk and DB mtimes match when the on-disk and DB +metadata are in sync; this lets beets do a quick mtime check and avoid +rereading files in some circumstances. + +Specifically, beets attempts to maintain the following invariant: + + If the on-disk metadata differs from the DB metadata, then the on-disk + mtime must be greater than the DB mtime. + +As a result, it is always valid for the DB mtime to be zero (assuming that real +disk mtimes are always positive). However, whenever possible, beets tries to +set ``db_mtime = disk_mtime`` at points where it knows the metadata is +synchronized. When it is possible that the metadata is out of sync, beets can +then just set ``db_mtime = 0`` to return to a consistent state. + +This leads to the following implementation policy: + + * On every write of disk metadata (``Item.write()``), the DB mtime is updated + to match the post-write disk mtime. + * Same for metadata reads (``Item.read()``). + * On every modification to DB metadata (``item.field = ...``), the DB mtime + is reset to zero. + + .. autoclass:: Item .. automethod:: __init__ @@ -130,6 +162,8 @@ update an item's fields (e.g., ``item.title = "Let It Be"``) and then call .. automethod:: destination + .. automethod:: current_mtime + The methods ``read()`` and ``write()`` are complementary: one reads a file's tags and updates the item's metadata fields accordingly while the other takes the item's fields and writes them to the file's tags. @@ -149,11 +183,6 @@ update an item's fields (e.g., ``item.title = "Let It Be"``) and then call .. automethod:: remove - Items also track their modification times (mtimes) to help detect when they - become out of sync with on-disk metadata. - - .. automethod:: current_mtime - Album ''''' From de78151eea5928aab1bd434efb5792b4df9501dc Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:18:46 +1000 Subject: [PATCH 333/339] docs: rename api -> library --- docs/dev/index.rst | 2 +- docs/dev/{api.rst => library.rst} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/dev/{api.rst => library.rst} (100%) diff --git a/docs/dev/index.rst b/docs/dev/index.rst index a47d6c8f2..ebeccc535 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -12,4 +12,4 @@ and write metadata tags in media files. .. toctree:: plugins - api + library diff --git a/docs/dev/api.rst b/docs/dev/library.rst similarity index 100% rename from docs/dev/api.rst rename to docs/dev/library.rst From 6769da29ae105d5d12fd71cb7b8eaba629742d6a Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:28:06 +1000 Subject: [PATCH 334/339] docs: add dev importer and cli text from wiki --- docs/dev/cli.rst | 9 +++++++++ docs/dev/importer.rst | 19 +++++++++++++++++++ docs/dev/index.rst | 2 ++ 3 files changed, 30 insertions(+) create mode 100644 docs/dev/cli.rst create mode 100644 docs/dev/importer.rst diff --git a/docs/dev/cli.rst b/docs/dev/cli.rst new file mode 100644 index 000000000..77d3af5a5 --- /dev/null +++ b/docs/dev/cli.rst @@ -0,0 +1,9 @@ +Providing a CLI +=============== + +The ``beets.ui`` module houses interactions with the user via a terminal, the +:doc:`/reference/cli`. +The main function is called when the user types beet on the command line. +The CLI functionality is organized into commands, some of which are built-in +and some of which are provided by plugins. The built-in commands are all +implemented in the ``beets.ui.commands`` submodule. diff --git a/docs/dev/importer.rst b/docs/dev/importer.rst new file mode 100644 index 000000000..5182c7134 --- /dev/null +++ b/docs/dev/importer.rst @@ -0,0 +1,19 @@ +Music Importer +============== + +The importer component is responsible for the user-centric workflow that adds +music to a library. This is one of the first aspects that a user experiences +when using beets: it finds music in the filesystem, groups it into albums, +finds corresponding metadata in MusicBrainz, asks the user for intervention, +applies changes, and moves/copies files. A description of its user interface is +given in :doc:`/guides/tagger`. + +The workflow is implemented in the ``beets.importer`` module and is +distinct from the core logic for matching MusicBrainz metadata (in the +``beets.autotag`` module). The workflow is also decoupled from the command-line +interface with the hope that, eventually, other (graphical) interfaces can be +bolted onto the same importer implementation. + +The importer is multithreaded and follows the pipeline pattern. Each pipeline +stage is a Python coroutine. The ``beets.util.pipeline`` module houses +a generic, reusable implementation of a multithreaded pipeline. diff --git a/docs/dev/index.rst b/docs/dev/index.rst index ebeccc535..f1465494d 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -13,3 +13,5 @@ and write metadata tags in media files. plugins library + importer + cli From 670046dd9ab2b8fce6df3f22822be4d7ec2a0971 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 22:55:12 +0200 Subject: [PATCH 335/339] Build https URLS for beatport releases I'm not sure where these are used, but the website supports https and the API url already uses https, so this should be a safe call and not require a util.SNI_SUPPORTED check. --- beetsplug/beatport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index da59bef87..ab50f46f2 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -224,7 +224,7 @@ class BeatportRelease(BeatportObject): if 'category' in data: self.category = data['category'] if 'slug' in data: - self.url = "http://beatport.com/release/{0}/{1}".format( + self.url = "https://beatport.com/release/{0}/{1}".format( data['slug'], data['id']) @@ -252,8 +252,8 @@ class BeatportTrack(BeatportObject): except ValueError: pass if 'slug' in data: - self.url = "http://beatport.com/track/{0}/{1}".format(data['slug'], - data['id']) + self.url = "https://beatport.com/track/{0}/{1}" \ + .format(data['slug'], data['id']) self.track_number = data.get('trackNumber') From 1a23eab8b6a5c1f721300d5298e1caddd6cf533f Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 23:00:52 +0200 Subject: [PATCH 336/339] Use https for lyrics.wikia.com, when supported --- beetsplug/lyrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 9e44eeef6..16699d9d3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -55,6 +55,7 @@ except ImportError: from beets import plugins from beets import ui +from beets import util import beets DIV_RE = re.compile(r'<(/?)div>?', re.I) @@ -406,7 +407,10 @@ class Genius(Backend): class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" - URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' + if util.SNI_SUPPORTED: + URL_PATTERN = 'https://lyrics.wikia.com/%s:%s' + else: + URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' def fetch(self, artist, title): url = self.build_url(artist, title) From c144141e9a8d7dd3374d1f077f6f1787033b5223 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 23:06:58 +0200 Subject: [PATCH 337/339] Update a few more http URLs to https that I missed Should really be all now (pending the next commit). --- beets/ui/__init__.py | 4 ++-- beets/util/__init__.py | 4 ++-- beetsplug/spotify.py | 2 +- docs/plugins/smartplaylist.rst | 2 +- docs/plugins/web.rst | 2 +- test/test_mb.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index a88ed9aef..aae3a1769 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -204,7 +204,7 @@ def input_(prompt=None): """ # raw_input incorrectly sends prompts to stderr, not stdout, so we # use print_() explicitly to display prompts. - # http://bugs.python.org/issue1927 + # https://bugs.python.org/issue1927 if prompt: print_(prompt, end=u' ') @@ -929,7 +929,7 @@ class CommonOptionsParser(optparse.OptionParser, object): # # This is a fairly generic subcommand parser for optparse. It is # maintained externally here: -# http://gist.github.com/462717 +# https://gist.github.com/462717 # There you will also find a better description of the code and a more # succinct example program. diff --git a/beets/util/__init__.py b/beets/util/__init__.py index e2348cf6e..162502eb1 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -412,7 +412,7 @@ def syspath(path, prefix=True): path = path.decode(encoding, 'replace') # Add the magic prefix if it isn't already there. - # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx + # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX): if path.startswith(u'\\\\'): # UNC path. Final path should look like \\?\UNC\... @@ -563,7 +563,7 @@ def unique_path(path): # Note: The Windows "reserved characters" are, of course, allowed on # Unix. They are forbidden here because they cause problems on Samba # shares, which are sufficiently common as to cause frequent problems. -# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx +# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx CHAR_REPLACE = [ (re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere. (re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix). diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index f6df91bb3..d8d7637d6 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -22,7 +22,7 @@ class SpotifyPlugin(BeetsPlugin): # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' - open_track_url = 'http://open.spotify.com/track/' + open_track_url = 'https://open.spotify.com/track/' search_url = 'https://api.spotify.com/v1/search' album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index 8ccbd0091..e68217657 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -5,7 +5,7 @@ Smart Playlist Plugin beets queries every time your library changes. This plugin is specifically created to work well with `MPD's`_ playlist functionality. -.. _MPD's: http://www.musicpd.org/ +.. _MPD's: https://www.musicpd.org/ To use it, enable the ``smartplaylist`` plugin in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index d3ae668ce..d416b1b7d 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -90,7 +90,7 @@ for unsupported formats/browsers. There are a number of options for this: .. _audio.js: https://kolber.github.io/audiojs/ .. _html5media: https://html5media.info/ -.. _MediaElement.js: http://mediaelementjs.com/ +.. _MediaElement.js: https://mediaelementjs.com/ .. _web-cors: diff --git a/test/test_mb.py b/test/test_mb.py index d5cb7c468..de1ffd9a7 100644 --- a/test/test_mb.py +++ b/test/test_mb.py @@ -459,7 +459,7 @@ class ParseIDTest(_common.TestCase): def test_parse_id_url_finds_id(self): id_string = "28e32c71-1450-463e-92bf-e0a46446fc11" - id_url = "http://musicbrainz.org/entity/%s" % id_string + id_url = "https://musicbrainz.org/entity/%s" % id_string out = mb._parse_id(id_url) self.assertEqual(out, id_string) From 9631616b538b1ebe60efe032c87d736b2744bda6 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 23:08:18 +0200 Subject: [PATCH 338/339] Replace a couple URLs that don't point to anything I'm unsure regarding the pygst tutorial, so I just added another URL of the best resource I could find with a quick web search. --- beets/ui/__init__.py | 2 +- beetsplug/bpd/gstplayer.py | 3 ++- docs/reference/cli.rst | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index aae3a1769..aec0e80a9 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -475,7 +475,7 @@ def human_seconds_short(interval): # Colorization. # ANSI terminal colorization code heavily inspired by pygments: -# http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py +# https://bitbucket.org/birkenfeld/pygments-main/src/default/pygments/console.py # (pygments is by Tim Hatch, Armin Ronacher, et al.) COLOR_ESCAPE = "\x1b[" DARK_COLORS = { diff --git a/beetsplug/bpd/gstplayer.py b/beetsplug/bpd/gstplayer.py index 8d4e7c9ff..3ba293bf2 100644 --- a/beetsplug/bpd/gstplayer.py +++ b/beetsplug/bpd/gstplayer.py @@ -64,7 +64,8 @@ class GstPlayer(object): """ # Set up the Gstreamer player. From the pygst tutorial: - # http://pygstdocs.berlios.de/pygst-tutorial/playbin.html + # https://pygstdocs.berlios.de/pygst-tutorial/playbin.html (gone) + # https://brettviren.github.io/pygst-tutorial-org/pygst-tutorial.html #### # Updated to GStreamer 1.0 with: # https://wiki.ubuntu.com/Novacut/GStreamer1.0 diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 2fc7c7b31..e17d5b42f 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -453,7 +453,7 @@ available via your package manager. On OS X, you can install it via Homebrew with ``brew install bash-completion``; Homebrew will give you instructions for sourcing the script. -.. _bash-completion: http://bash-completion.alioth.debian.org/ +.. _bash-completion: https://github.com/scop/bash-completion .. _bash: https://www.gnu.org/software/bash/ The completion script suggests names of subcommands and (after typing From 728203e15af1e741ee6f0f118d57dbfc7d5e0487 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Thu, 6 Jun 2019 15:34:15 +0200 Subject: [PATCH 339/339] beets.io now supports HTTPS See https://github.com/beetbox/beets/pull/3297. --- README.rst | 2 +- README_kr.rst | 4 ++-- beets/autotag/mb.py | 2 +- beetsplug/beatport.py | 4 ++-- beetsplug/discogs.py | 2 +- docs/changelog.rst | 6 +++--- docs/dev/library.rst | 2 +- docs/guides/advanced.rst | 2 +- docs/guides/main.rst | 4 ++-- docs/index.rst | 2 +- extra/_beet | 16 ++++++++-------- setup.py | 2 +- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index 8f8705abd..f9be39c52 100644 --- a/README.rst +++ b/README.rst @@ -103,7 +103,7 @@ Read More Learn more about beets at `its Web site`_. Follow `@b33ts`_ on Twitter for news and updates. -.. _its Web site: http://beets.io/ +.. _its Web site: https://beets.io/ .. _@b33ts: https://twitter.com/b33ts/ Authors diff --git a/README_kr.rst b/README_kr.rst index 6bdcf56a6..25dd052d8 100644 --- a/README_kr.rst +++ b/README_kr.rst @@ -34,7 +34,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 - 필요하는 메타 데이터를 계산하거나 패치 할 때: `album art`_, `lyrics`_, `genres`_, `tempos`_, `ReplayGain`_ levels, or `acoustic fingerprints`_. -- `MusicBrainz`_, `Discogs`_,`Beatport`_로부터 메타데이터를 가져오거나, +- `MusicBrainz`_, `Discogs`_,`Beatport`_로부터 메타데이터를 가져오거나, 노래 제목이나 음향 특징으로 메타데이터를 추측한다 - `Transcode audio`_ 당신이 좋아하는 어떤 포맷으로든 변경한다. - 당신의 라이브러리에서 `duplicate tracks and albums`_ 이나 `albums that are missing tracks`_ 를 검사한다. @@ -45,7 +45,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 - 명령어로부터 음악 파일의 메타데이터를 분석할 수 있다. - `MPD`_ 프로토콜을 사용하여 음악 플레이어로 음악을 들으면, 엄청나게 다양한 인터페이스로 작동한다. -만약 Beets에 당신이 원하는게 아직 없다면, +만약 Beets에 당신이 원하는게 아직 없다면, 당신이 python을 안다면 `writing your own plugin`_ _은 놀라울정도로 간단하다. .. _plugins: https://beets.readthedocs.org/page/plugins/ diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index b8f91f440..1a6e0b1f1 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -39,7 +39,7 @@ else: SKIPPED_TRACKS = ['[data track]'] musicbrainzngs.set_useragent('beets', beets.__version__, - 'http://beets.io/') + 'https://beets.io/') class MusicBrainzAPIError(util.HumanReadableException): diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index ab50f46f2..3462f118a 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -34,7 +34,7 @@ import confuse AUTH_ERRORS = (TokenRequestDenied, TokenMissing, VerifierMissing) -USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) +USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) class BeatportAPIError(Exception): @@ -109,7 +109,7 @@ class BeatportClient(object): :rtype: (unicode, unicode) tuple """ self.api.parse_authorization_response( - "http://beets.io/auth?" + auth_data) + "https://beets.io/auth?" + auth_data) access_data = self.api.fetch_access_token( self._make_url('/identity/1/oauth/access-token')) return access_data['oauth_token'], access_data['oauth_token_secret'] diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 68b4b5a95..6a0a9c531 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -37,7 +37,7 @@ import traceback from string import ascii_lowercase -USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) +USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) # Exceptions that discogs_client should really handle but does not. CONNECTION_ERRORS = (ConnectionError, socket.error, http_client.HTTPException, diff --git a/docs/changelog.rst b/docs/changelog.rst index a667be780..264a82107 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1241,7 +1241,7 @@ Fixes: * :doc:`/plugins/replaygain`: Fix a crash using the Python Audio Tools backend. :bug:`1873` -.. _beets.io: http://beets.io/ +.. _beets.io: https://beets.io/ .. _Beetbox: https://github.com/beetbox @@ -2639,7 +2639,7 @@ previous versions would spit out a warning and then list your entire library. There's more detail than you could ever need `on the beets blog`_. -.. _on the beets blog: http://beets.io/blog/flexattr.html +.. _on the beets blog: https://beets.io/blog/flexattr.html 1.2.2 (August 27, 2013) @@ -3227,7 +3227,7 @@ begins today on features for version 1.1. unintentionally loading the plugins they contain. .. _The Echo Nest: http://the.echonest.com/ -.. _Tomahawk resolver: http://beets.io/blog/tomahawk-resolver.html +.. _Tomahawk resolver: https://beets.io/blog/tomahawk-resolver.html .. _mp3gain: http://mp3gain.sourceforge.net/download.php .. _aacgain: https://aacgain.altosdesign.com diff --git a/docs/dev/library.rst b/docs/dev/library.rst index ec81c169b..77e218b93 100644 --- a/docs/dev/library.rst +++ b/docs/dev/library.rst @@ -242,7 +242,7 @@ to interact directly with the underlying SQLite database, you must use a items = lib.items(query) lib.add_album(list(items)) -.. _blog post: http://beets.io/blog/sqlite-nightmare.html +.. _blog post: https://beets.io/blog/sqlite-nightmare.html .. currentmodule:: beets.dbcore.db diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index 091875c54..f4f8d3cd9 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -127,7 +127,7 @@ And, unlike :ref:`built-in fields `, such fields can be removed:: Read more than you ever wanted to know about the *flexible attributes* feature `on the beets blog`_. -.. _on the beets blog: http://beets.io/blog/flexattr.html +.. _on the beets blog: https://beets.io/blog/flexattr.html Choose a path style manually for some music diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 1f5cc4681..2f05634d9 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -4,7 +4,7 @@ Getting Started Welcome to `beets`_! This guide will help you begin using it to make your music collection better. -.. _beets: http://beets.io/ +.. _beets: https://beets.io/ Installing ---------- @@ -43,7 +43,7 @@ Beets works on `Python 2.7`_ and Python 3.4 or later. * On **Fedora** 22 or later, there is a `DNF package`_:: $ sudo dnf install beets beets-plugins beets-doc - + * On **Solus**, run ``eopkg install beets``. * On **NixOS**, there's a `package `_ you can install with ``nix-env -i beets``. diff --git a/docs/index.rst b/docs/index.rst index 27fa4740b..4919147ce 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ Freenode, drop by `the discussion board`_, send email to `the mailing list`_, or `file a bug`_ in the issue tracker. Please let us know where you think this documentation can be improved. -.. _beets: http://beets.io/ +.. _beets: https://beets.io/ .. _the mailing list: https://groups.google.com/group/beets-users .. _file a bug: https://github.com/beetbox/beets/issues .. _the discussion board: https://discourse.beets.io diff --git a/extra/_beet b/extra/_beet index a8c9083be..129c0485e 100644 --- a/extra/_beet +++ b/extra/_beet @@ -1,6 +1,6 @@ #compdef beet -# zsh completion for beets music library manager and MusicBrainz tagger: http://beets.io/ +# zsh completion for beets music library manager and MusicBrainz tagger: https://beets.io/ # Default values for BEETS_LIBRARY & BEETS_CONFIG needed for the cache checking function. # They will be updated under the assumption that the config file is in the same directory as the library. @@ -34,7 +34,7 @@ _beet_check_cache () { # useful: argument to _regex_arguments for matching any word local matchany=/$'[^\0]##\0'/ # arguments to _regex_arguments for completing files and directories -local -a files dirs +local -a files dirs files=("$matchany" ':file:file:_files') dirs=("$matchany" ':dir:directory:_dirs') @@ -73,7 +73,7 @@ if ! _retrieve_cache beetscmds || _cache_invalid beetscmds; then # create completion function for queries _regex_arguments _beet_query "$matchany" \# \( "$matchquery" ":query:query string:$queryelem" \) \# local "beets_query"="$(which _beet_query)" - # arguments for _regex_arguments for completing lists of queries and modifications + # arguments for _regex_arguments for completing lists of queries and modifications beets_query_args=( \( "$matchquery" ":query:query string:{_beet_query}" \) \# ) beets_modify_args=( \( "$matchmodify" ":modify:modify string:$modifyelem" \) \# ) # now build arguments for _beet and _beet_help completion functions @@ -82,7 +82,7 @@ if ! _retrieve_cache beetscmds || _cache_invalid beetscmds; then subcmd="${i[(w)1]}" # remove first word and parenthesised alias, replace : with -, [ with (, ] with ), and remove single quotes cmddesc="${${${${${i[(w)2,-1]##\(*\) #}//:/-}//\[/(}//\]/)}//\'/}" - # update arguments needed for creating _beet + # update arguments needed for creating _beet beets_regex_words_subcmds+=(/"${subcmd}"$'\0'/ ":subcmds:subcommands:((${subcmd}:${cmddesc// /\ }))") beets_regex_words_subcmds+=(\( "${matchany}" ":option:option:{_beet_subcmd ${subcmd}}" \) \# \|) # update arguments needed for creating _beet_help @@ -137,7 +137,7 @@ _beet_subcmd_options() { fi ;; (LOG) - local -a files + local -a files files=("$matchany" ':file:file:_files') regex_words+=("$opt:$optdesc:\$files") ;; @@ -180,7 +180,7 @@ _beet_subcmd() { if [[ ! $(type _beet_${subcmd} | grep function) =~ function ]]; then if ! _retrieve_cache "beets${subcmd}" || _cache_invalid "beets${subcmd}"; then local matchany=/$'[^\0]##\0'/ - local -a files + local -a files files=("$matchany" ':file:file:_files') # get arguments for completing subcommand options _beet_subcmd_options "$subcmd" @@ -197,7 +197,7 @@ _beet_subcmd() { (fields|migrate|version|config) _regex_arguments _beet_${subcmd} "${matchany}" /"${subcmd}"$'\0'/ "${options[@]}" ;; - (help) + (help) _regex_words subcmds "subcommands" "${beets_regex_words_help[@]}" _regex_arguments _beet_help "${matchany}" /$'help\0'/ "${options[@]}" "${reply[@]}" ;; @@ -232,6 +232,6 @@ zstyle ":completion:${curcontext}:" tag-order '! options' # Execute the completion function _beet "$@" -# Local Variables: +# Local Variables: # mode:shell-script # End: diff --git a/setup.py b/setup.py index 30db5272d..1078d6cc9 100755 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( description='music tagger and library organizer', author='Adrian Sampson', author_email='adrian@radbox.org', - url='http://beets.io/', + url='https://beets.io/', license='MIT', platforms='ALL', long_description=_read('README.rst'),