From ca8c557840389f85c2f8a4803d12853caf1a658c Mon Sep 17 00:00:00 2001 From: Nathan Musoke Date: Tue, 9 May 2017 14:19:01 +1200 Subject: [PATCH 001/613] bugfix: Python3ify the IPFS plugin Paths were being constructed in a Python 3-incompatible way by concating bytes and strings. Do this more carefully by encoding and decoding of binary and strings. --- beetsplug/ipfs.py | 12 ++++++------ docs/changelog.rst | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/beetsplug/ipfs.py b/beetsplug/ipfs.py index 9a9d6aa50..88e305213 100644 --- a/beetsplug/ipfs.py +++ b/beetsplug/ipfs.py @@ -190,7 +190,7 @@ class IPFSPlugin(BeetsPlugin): else: lib_name = _hash lib_root = os.path.dirname(lib.path) - remote_libs = lib_root + "/remotes" + remote_libs = os.path.join(lib_root, b"remotes") if not os.path.exists(remote_libs): try: os.makedirs(remote_libs) @@ -198,7 +198,7 @@ class IPFSPlugin(BeetsPlugin): msg = "Could not create {0}. Error: {1}".format(remote_libs, e) self._log.error(msg) return False - path = remote_libs + "/" + lib_name + ".db" + path = os.path.join(remote_libs, lib_name.encode() + b".db") if not os.path.exists(path): cmd = "ipfs get {0} -o".format(_hash).split() cmd.append(path) @@ -209,7 +209,7 @@ class IPFSPlugin(BeetsPlugin): return False # add all albums from remotes into a combined library - jpath = remote_libs + "/joined.db" + jpath = os.path.join(remote_libs, b"joined.db") jlib = library.Library(jpath) nlib = library.Library(path) for album in nlib.albums(): @@ -237,7 +237,7 @@ class IPFSPlugin(BeetsPlugin): return for album in albums: - ui.print_(format(album, fmt), " : ", album.ipfs) + ui.print_(format(album, fmt), " : ", album.ipfs.decode()) def query(self, lib, args): rlib = self.get_remote_lib(lib) @@ -246,8 +246,8 @@ class IPFSPlugin(BeetsPlugin): def get_remote_lib(self, lib): lib_root = os.path.dirname(lib.path) - remote_libs = lib_root + "/remotes" - path = remote_libs + "/joined.db" + remote_libs = os.path.join(lib_root, b"remotes") + path = os.path.join(remote_libs, b"joined.db") if not os.path.isfile(path): raise IOError return library.Library(path) diff --git a/docs/changelog.rst b/docs/changelog.rst index bf0d7d254..b80df36c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -102,6 +102,7 @@ Fixes: error message. Thanks to :user:`Mary011196`. :bug:`1676` :bug:`2508` * :doc:`/plugins/web`: Avoid a crash when sending binary data, such as Chromaprint fingerprints, in music attributes. :bug:`2542` :bug:`2532` +* :doc:`/plugins/ipfs`: Fix Python 3 compatibility. Two plugins had backends removed due to bitrot: From c5319274ca4e8322c250d0b7a9ae9ac036ec4432 Mon Sep 17 00:00:00 2001 From: Nathan Musoke Date: Tue, 9 May 2017 15:38:50 +1200 Subject: [PATCH 002/613] IPFS plugin: Add note to check hashes carefully In the future, just checking that a hash begins with "Qm" and has length 46 will likely not be sufficient. --- beetsplug/ipfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/ipfs.py b/beetsplug/ipfs.py index 88e305213..0e9b0c1f1 100644 --- a/beetsplug/ipfs.py +++ b/beetsplug/ipfs.py @@ -147,6 +147,8 @@ class IPFSPlugin(BeetsPlugin): def ipfs_get(self, lib, query): query = query[0] # Check if query is a hash + # TODO: generalize to other hashes; probably use a multihash + # implementation if query.startswith("Qm") and len(query) == 46: self.ipfs_get_from_hash(lib, query) else: From 3d842db8d8507828ec1e0f6f961ac3e710fea683 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Wed, 6 Feb 2019 09:27:24 +0100 Subject: [PATCH 003/613] Added per disc album_gain support --- beetsplug/replaygain.py | 92 +++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 31 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ac45aa4f8..868800d9b 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -83,7 +83,7 @@ class Backend(object): def compute_track_gain(self, items): raise NotImplementedError() - def compute_album_gain(self, album): + def compute_album_gain(self, items): # TODO: implement album gain in terms of track gain of the # individual tracks which can be used for any backend. raise NotImplementedError() @@ -125,15 +125,14 @@ class Bs1770gainBackend(Backend): output = self.compute_gain(items, False) return output - def compute_album_gain(self, album): + def compute_album_gain(self, items): """Computes the album gain of the given album, returns an AlbumGain object. """ # TODO: What should be done when not all tracks in the album are # supported? - supported_items = album.items() - output = self.compute_gain(supported_items, True) + output = self.compute_gain(items, True) if not output: raise ReplayGainError(u'no output from bs1770gain') @@ -316,15 +315,15 @@ class CommandBackend(Backend): output = self.compute_gain(supported_items, False) return output - def compute_album_gain(self, album): + def compute_album_gain(self, items): """Computes the album gain of the given album, returns an AlbumGain object. """ # TODO: What should be done when not all tracks in the album are # supported? - supported_items = list(filter(self.format_supported, album.items())) - if len(supported_items) != len(album.items()): + supported_items = list(filter(self.format_supported, items)) + if len(supported_items) != len(items): self._log.debug(u'tracks are of unsupported format') return AlbumGain(None, []) @@ -521,8 +520,8 @@ class GStreamerBackend(Backend): return ret - def compute_album_gain(self, album): - items = list(album.items()) + def compute_album_gain(self, items): + items = list(items) self.compute(items, True) if len(self._file_tags) != len(items): raise ReplayGainError(u"Some items in album did not receive tags") @@ -777,22 +776,20 @@ class AudioToolsBackend(Backend): item.artist, item.title, rg_track_gain, rg_track_peak) return Gain(gain=rg_track_gain, peak=rg_track_peak) - def compute_album_gain(self, album): + def compute_album_gain(self, items): """Compute ReplayGain values for the requested album and its items. :rtype: :class:`AlbumGain` """ - self._log.debug(u'Analysing album {0}', album) - # The first item is taken and opened to get the sample rate to # initialize the replaygain object. The object is used for all the # tracks in the album to get the album values. - item = list(album.items())[0] + item = list(items)[0] audiofile = self.open_audio_file(item) rg = self.init_replaygain(audiofile, item) track_gains = [] - for item in album.items(): + for item in items: audiofile = self.open_audio_file(item) rg_track_gain, rg_track_peak = self._title_gain(rg, audiofile) track_gains.append( @@ -836,9 +833,11 @@ class ReplayGainPlugin(BeetsPlugin): 'backend': u'command', 'targetlevel': 89, 'r128': ['Opus'], + 'per_disc': False }) self.overwrite = self.config['overwrite'].get(bool) + self.per_disc = self.config['per_disc'].get(bool) backend_name = self.config['backend'].as_str() if backend_name not in self.backends: raise ui.UserError( @@ -919,6 +918,21 @@ class ReplayGainPlugin(BeetsPlugin): self._log.debug(u'applied r128 album gain {0}', album.r128_album_gain) + def store_album_gain_item_level(self, items, album_gain): + for item in items: + item.rg_album_gain = album_gain.gain + item.rg_album_peak = album_gain.peak + item.store() + self._log.debug(u'applied album gain {0}, peak {1}', + item.rg_album_gain, item.rg_album_peak) + + def store_album_r128_gain_item_level(self, items, album_gain): + for item in items: + item.r128_album_gain = album_gain.gain + item.store() + self._log.debug(u'applied r128 album gain {0}', + item.r128_album_gain) + def handle_album(self, album, write, force=False): """Compute album and track replay gain store it in all of the album's items. @@ -946,29 +960,45 @@ class ReplayGainPlugin(BeetsPlugin): backend_instance = self.r128_backend_instance store_track_gain = self.store_track_r128_gain store_album_gain = self.store_album_r128_gain + store_album_gain_item_level = self.store_album_r128_gain_item_level else: backend_instance = self.backend_instance store_track_gain = self.store_track_gain store_album_gain = self.store_album_gain + store_album_gain_item_level = self.store_album_gain_item_level - try: - album_gain = backend_instance.compute_album_gain(album) - if len(album_gain.track_gains) != len(album.items()): - raise ReplayGainError( - u"ReplayGain backend failed " - u"for some tracks in album {0}".format(album) - ) + discs = dict() + if self.per_disc: + for item in album.items(): + if discs.get(item.disc) is None: + discs[item.disc] = [] + discs[item.disc].append(item) + else: + discs[1] = album.items() - store_album_gain(album, album_gain.album_gain) - for item, track_gain in zip(album.items(), album_gain.track_gains): - store_track_gain(item, track_gain) - if write: - item.try_write() - except ReplayGainError as e: - self._log.info(u"ReplayGain error: {0}", e) - except FatalReplayGainError as e: - raise ui.UserError( - u"Fatal replay gain error: {0}".format(e)) + for discnumber in discs: + try: + items = discs[discnumber] + album_gain = backend_instance.compute_album_gain(items) + if len(album_gain.track_gains) != len(items): + raise ReplayGainError( + u"ReplayGain backend failed " + u"for some tracks in album {0}".format(album) + ) + + if len(items) == len(album.items()): + store_album_gain(album, album_gain.album_gain) + else: + store_album_gain_item_level(items, album_gain.album_gain) + for item, track_gain in zip(items, album_gain.track_gains): + store_track_gain(item, track_gain) + if write: + item.try_write() + except ReplayGainError as e: + self._log.info(u"ReplayGain error: {0}", e) + except FatalReplayGainError as e: + raise ui.UserError( + u"Fatal replay gain error: {0}".format(e)) def handle_track(self, item, write, force=False): """Compute track replay gain and store it in the item. From 1619761bd6bcd4ac4f3aa8d04e615cbb8f9f4535 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Wed, 6 Feb 2019 09:38:03 +0100 Subject: [PATCH 004/613] Updated docs with per_disc ReplayGain configuration. --- docs/plugins/replaygain.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index ad0e50e22..3f1667c8f 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -93,6 +93,8 @@ configuration file. The available options are: integer values instead of the common ``REPLAYGAIN_`` tags with floating point values. Requires the "ffmpeg" backend. Default: ``Opus``. +- **per_disc**: Calculate album ReplayGain on disc level instead of album level. + Default: ``no`` These options only work with the "command" backend: From 31326ebb20438f96702824f78df67764b58718fa Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Wed, 6 Feb 2019 10:06:48 +0100 Subject: [PATCH 005/613] Simplified album ReplayGain code --- beetsplug/replaygain.py | 43 +++++++++++------------------------------ 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index dba1f6baa..660bf8c1a 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -904,34 +904,18 @@ class ReplayGainPlugin(BeetsPlugin): self._log.debug(u'applied r128 track gain {0}', item.r128_track_gain) - def store_album_gain(self, album, album_gain): - album.rg_album_gain = album_gain.gain - album.rg_album_peak = album_gain.peak - album.store() - + def store_album_gain(self, item, album_gain): + item.rg_album_gain = album_gain.gain + item.rg_album_peak = album_gain.peak + item.store() self._log.debug(u'applied album gain {0}, peak {1}', - album.rg_album_gain, album.rg_album_peak) + item.rg_album_gain, item.rg_album_peak) - def store_album_r128_gain(self, album, album_gain): - album.r128_album_gain = int(round(album_gain.gain * pow(2, 8))) - album.store() - - self._log.debug(u'applied r128 album gain {0}', album.r128_album_gain) - - def store_album_gain_item_level(self, items, album_gain): - for item in items: - item.rg_album_gain = album_gain.gain - item.rg_album_peak = album_gain.peak - item.store() - self._log.debug(u'applied album gain {0}, peak {1}', - item.rg_album_gain, item.rg_album_peak) - - def store_album_r128_gain_item_level(self, items, album_gain): - for item in items: - item.r128_album_gain = album_gain.gain - item.store() - self._log.debug(u'applied r128 album gain {0}', - item.r128_album_gain) + def store_album_r128_gain(self, item, album_gain): + item.r128_album_gain = album_gain.gain + item.store() + self._log.debug(u'applied r128 album gain {0}', + item.r128_album_gain) def handle_album(self, album, write, force=False): """Compute album and track replay gain store it in all of the @@ -960,12 +944,10 @@ class ReplayGainPlugin(BeetsPlugin): backend_instance = self.r128_backend_instance store_track_gain = self.store_track_r128_gain store_album_gain = self.store_album_r128_gain - store_album_gain_item_level = self.store_album_r128_gain_item_level else: backend_instance = self.backend_instance store_track_gain = self.store_track_gain store_album_gain = self.store_album_gain - store_album_gain_item_level = self.store_album_gain_item_level discs = dict() if self.per_disc: @@ -986,12 +968,9 @@ class ReplayGainPlugin(BeetsPlugin): u"for some tracks in album {0}".format(album) ) - if len(items) == len(album.items()): - store_album_gain(album, album_gain.album_gain) - else: - store_album_gain_item_level(items, album_gain.album_gain) for item, track_gain in zip(items, album_gain.track_gains): store_track_gain(item, track_gain) + store_album_gain(item, album_gain.album_gain) if write: item.try_write() except ReplayGainError as e: From 24f02cb5cd6cc51c5efd3d064ee9fcc0988628ec Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Wed, 6 Feb 2019 10:12:06 +0100 Subject: [PATCH 006/613] ReplayGain refactoring --- beetsplug/replaygain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 660bf8c1a..2122be7a4 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -958,9 +958,8 @@ class ReplayGainPlugin(BeetsPlugin): else: discs[1] = album.items() - for discnumber in discs: + for discnumber, items in discs.items(): try: - items = discs[discnumber] album_gain = backend_instance.compute_album_gain(items) if len(album_gain.track_gains) != len(items): raise ReplayGainError( From 413147d3c98562452c18fa84ac9944dfb4067681 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Wed, 6 Feb 2019 12:42:58 +0100 Subject: [PATCH 007/613] ReplayGain: Updated changelog with per_disc option. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cab4a1e0..e5d1507a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -65,6 +65,10 @@ New features: :bug:`3123` * :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which passes that flag to ipfs. Thanks to :user:`wildthyme`. +* :doc:`/plugins/replaygain`: The plugin now supports a ``per_disc`` option + which enables calculation of album ReplayGain on disc level instead of album + level. + Thanks to :user:`samuelnilsson` Changes: From 93007bfdd5a305a83c6a55457e1d19574444c280 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Wed, 6 Feb 2019 13:17:34 +0100 Subject: [PATCH 008/613] ReplayGain: fixed error caused by per_disc option --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 2122be7a4..40d228490 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -802,7 +802,7 @@ class AudioToolsBackend(Backend): # album values. rg_album_gain, rg_album_peak = rg.album_gain() self._log.debug(u'ReplayGain for album {0}: {1:.2f}, {2:.2f}', - album, rg_album_gain, rg_album_peak) + items[0].album, rg_album_gain, rg_album_peak) return AlbumGain( Gain(gain=rg_album_gain, peak=rg_album_peak), From 8fcff5ddc7bfef1394bedc39aa83614a7710d0ee Mon Sep 17 00:00:00 2001 From: Peter Koondial Date: Sun, 5 May 2019 11:11:27 +0200 Subject: [PATCH 009/613] Adding styles to discogs plugin --- beets/autotag/__init__.py | 1 + beets/config_default.yaml | 1 + beets/library.py | 4 +++- beetsplug/discogs.py | 10 +++++++++- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index a71b9b0a6..48901f425 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -155,6 +155,7 @@ def apply_metadata(album_info, mapping): 'script', 'language', 'country', + 'style', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', diff --git a/beets/config_default.yaml b/beets/config_default.yaml index cf9ae6bf9..538753bb7 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -131,6 +131,7 @@ match: track_index: 1.0 track_length: 2.0 track_id: 5.0 + style: 5.0 preferred: countries: [] media: [] diff --git a/beets/library.py b/beets/library.py index 16db1e974..97ae4589c 100644 --- a/beets/library.py +++ b/beets/library.py @@ -436,6 +436,7 @@ class Item(LibModel): 'albumartist_sort': types.STRING, 'albumartist_credit': types.STRING, 'genre': types.STRING, + 'style': types.STRING, 'lyricist': types.STRING, 'composer': types.STRING, 'composer_sort': types.STRING, @@ -495,7 +496,7 @@ class Item(LibModel): } _search_fields = ('artist', 'title', 'comments', - 'album', 'albumartist', 'genre') + 'album', 'albumartist') _types = { 'data_source': types.STRING, @@ -915,6 +916,7 @@ class Album(LibModel): 'albumartist_credit': types.STRING, 'album': types.STRING, 'genre': types.STRING, + 'style': types.STRING, 'year': types.PaddedInt(4), 'month': types.PaddedInt(2), 'day': types.PaddedInt(2), diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 5a2bf57e0..efc9f2b0e 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -302,6 +302,13 @@ class DiscogsPlugin(BeetsPlugin): mediums = [t.medium for t in tracks] country = result.data.get('country') data_url = result.data.get('uri') + style = result.data.get('styles') + if style is None: + self._log.info('Style not Found') + elif len(style) == 0: + return style + else: + style = ' - '.join(sorted(style)) # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. @@ -339,7 +346,8 @@ class DiscogsPlugin(BeetsPlugin): day=None, label=label, mediums=len(set(mediums)), artist_sort=None, releasegroup_id=master_id, catalognum=catalogno, script=None, language=None, - country=country, albumstatus=None, media=media, + country=country, style=style, + albumstatus=None, media=media, albumdisambig=None, artist_credit=None, original_year=original_year, original_month=None, original_day=None, data_source='Discogs', From 295efde7b43e1cf389136cae4362d971e93a5a7f Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 5 May 2019 11:23:27 +0200 Subject: [PATCH 010/613] re-adding genre --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 97ae4589c..c7fbe48cf 100644 --- a/beets/library.py +++ b/beets/library.py @@ -496,7 +496,7 @@ class Item(LibModel): } _search_fields = ('artist', 'title', 'comments', - 'album', 'albumartist') + 'album', 'albumartist', 'genre') _types = { 'data_source': types.STRING, From 6ffbd5af45ac88a584c07ee22fa0fbabf5c2a45e Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 5 May 2019 11:44:24 +0200 Subject: [PATCH 011/613] adding styles to hook and returning Style not Defined if no style set --- beets/autotag/hooks.py | 5 +++-- beetsplug/discogs.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index ec7047b7c..f822cdfde 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -79,7 +79,7 @@ class AlbumInfo(object): albumtype=None, va=False, year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, albumstatus=None, media=None, + language=None, country=None, style=None, albumstatus=None, media=None, albumdisambig=None, releasegroupdisambig=None, artist_credit=None, original_year=None, original_month=None, original_day=None, data_source=None, data_url=None): @@ -102,6 +102,7 @@ class AlbumInfo(object): self.script = script self.language = language self.country = country + self.style = style self.albumstatus = albumstatus self.media = media self.albumdisambig = albumdisambig @@ -121,7 +122,7 @@ class AlbumInfo(object): constituent `TrackInfo` objects, are decoded to Unicode. """ for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', - 'catalognum', 'script', 'language', 'country', + 'catalognum', 'script', 'language', 'country', 'style', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', 'artist_credit', 'media']: value = getattr(self, fld) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index efc9f2b0e..0865b691b 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -303,8 +303,10 @@ class DiscogsPlugin(BeetsPlugin): country = result.data.get('country') data_url = result.data.get('uri') style = result.data.get('styles') + print('style', style) if style is None: self._log.info('Style not Found') + return "Style not Defined" elif len(style) == 0: return style else: From 080680c950432898a47979e96f64a50622ddf50e Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 14:35:51 +0200 Subject: [PATCH 012/613] add parentwork plugin, first try --- beetsplug/parentwork.py | 193 +++++++++++++++++++++++++++++++++++++ docs/plugins/index.rst | 2 + docs/plugins/parentwork.py | 60 ++++++++++++ test/test_parentwork.py | 93 ++++++++++++++++++ 4 files changed, 348 insertions(+) create mode 100644 beetsplug/parentwork.py create mode 100644 docs/plugins/parentwork.py create mode 100644 test/test_parentwork.py diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py new file mode 100644 index 000000000..8a8c0c12a --- /dev/null +++ b/beetsplug/parentwork.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2017, Dorian Soergel. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Gets work title, disambiguation, parent work and its disambiguation, +composer, composer sort name and performers +""" + +from __future__ import division, absolute_import, print_function + +from beets import ui +from beets.plugins import BeetsPlugin + +import musicbrainzngs + + +def work_father(mb_workid, work_date=None): + """ This function finds the id of the father work given its id""" + work_info = musicbrainzngs.get_work_by_id(mb_workid, + includes=["work-rels", + "artist-rels"]) + if 'artist-relation-list' in work_info['work'] and work_date is None: + for artist in work_info['work']['artist-relation-list']: + if artist['type'] == 'composer': + if 'end' in artist.keys(): + work_date = artist['end'] + + if 'work-relation-list' in work_info['work']: + for work_father in work_info['work']['work-relation-list']: + if work_father['type'] == 'parts' \ + and work_father.get('direction') == 'backward': + father_id = work_father['work']['id'] + return father_id, work_date + return None, work_date + + +def work_parent(mb_workid): + """This function finds the parentwork id of a work given its id. """ + work_date = None + while True: + (new_mb_workid, work_date) = work_father(mb_workid, work_date) + if not new_mb_workid: + return mb_workid, work_date + mb_workid = new_mb_workid + return mb_workid, work_date + + +def find_parentwork(mb_workid): + """This function gives the work relationships (dict) of a parentwork + given the id of the work""" + parent_id, work_date = work_parent(mb_workid) + work_info = musicbrainzngs.get_work_by_id(parent_id, + includes=["artist-rels"]) + return work_info, work_date + + +class ParentWorkPlugin(BeetsPlugin): + def __init__(self): + super(ParentWorkPlugin, self).__init__() + + self.config.add({ + 'auto': False, + 'force': False, + }) + + self._command = ui.Subcommand( + 'parentwork', + help=u'Fetches parent works, composers and dates') + + self._command.parser.add_option( + u'-f', u'--force', dest='force', + action='store_true', default=None, + help=u'Re-fetches all parent works') + + if self.config['auto']: + self.import_stages = [self.imported] + + def commands(self): + + def func(lib, opts, args): + self.config.set_args(opts) + force_parent = self.config['force'].get(bool) + write = ui.should_write() + + for item in lib.items(ui.decargs(args)): + self.find_work(item, force_parent) + item.store() + if write: + item.try_write() + + self._command.func = func + return [self._command] + + def imported(self, session, task): + """Import hook for fetching parent works automatically. + """ + force_parent = self.config['force'].get(bool) + + for item in task.imported_items(): + self.find_work(item, force_parent) + item.store() + + def get_info(self, item, work_info): + """Given the parentwork info dict, this function fetches + parent_composer, parent_composer_sort, parentwork, + parentwork_disambig, mb_workid and composer_ids""" + + parent_composer = [] + parent_composer_sort = [] + + composer_exists = False + if 'artist-relation-list' in work_info['work']: + for artist in work_info['work']['artist-relation-list']: + if artist['type'] == 'composer': + parent_composer.append(artist['artist']['name']) + parent_composer_sort.append(artist['artist']['sort-name']) + if not composer_exists: + self._log.info(item.artist + ' - ' + item.title) + self._log.debug( + "no composer, add one at https://musicbrainz.org/work/" + + work_info['work']['id']) + parentwork = work_info['work']['title'] + mb_parentworkid = work_info['work']['id'] + if 'disambiguation' in work_info['work']: + parentwork_disambig = work_info['work']['disambiguation'] + else: + parentwork_disambig.append('') + return parentwork, mb_parentworkid, parentwork_disambig, + parent_composer, parent_composer_sort + + def find_work(self, item, force): + + recording_id = item.mb_trackid + try: + item.parentwork + hasparent = True + except AttributeError: + hasparent = False + hasawork = True + if not item.mb_workid: + self._log.info("No work attached, recording id: " + + recording_id) + self._log.info(item.artist + ' - ' + item.title) + self._log.info("add one at https://musicbrainz.org" + + "/recording/" + recording_id) + hasawork = False + found = False + + if hasawork and (force or (not hasparent)): + try: + work_info, work_date = find_parentwork(item.mb_workid) + (parentwork, mb_parentworkid, parentwork_disambig, + parent_composer, + parent_composer_sort) = self.get_info(item, work_info) + found = True + except musicbrainzngs.musicbrainz.WebServiceError: + self._log.debug("Work unreachable") + found = False + elif parentwork: + self._log.debug("Work already in library, not necessary fetching") + return + + if found: + self._log.debug("Finished searching work for: " + + item.artist + ' - ' + item.title) + self._log.debug("Work fetched: " + parentwork + + ' - ' + u', '.join(parent_composer)) + item['parentwork'] = parentwork + item['parentwork_disambig'] = parentwork_disambig + item['mb_parentworkid'] = mb_parentworkid + item['parent_composer'] = u'' + item['parent_composer'] = u', '.join(parent_composer) + item['parent_composer_sort'] = u'' + item['parent_composer_sort'] = u', '.join(parent_composer_sort) + if work_date: + item['work_date'] = work_date + ui.show_model_changes( + item, fields=['parentwork', 'parentwork_disambig', + 'mb_parentworkid', 'parent_composer', + 'parent_composer_sort', 'work_date']) + + item.store() diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index e75e2f810..b962f7a10 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -80,6 +80,7 @@ like this:: missing mpdstats mpdupdate + parentwork permissions play playlist @@ -131,6 +132,7 @@ Metadata * :doc:`metasync`: Fetch metadata from local or remote sources * :doc:`mpdstats`: Connect to `MPD`_ and update the beets library with play statistics (last_played, play_count, skip_count, rating). +* :doc:`parentwork`: Fetch work titles and works they are part of. * :doc:`replaygain`: Calculate volume normalization for players that support it. * :doc:`scrub`: Clean extraneous metadata from music files. * :doc:`zero`: Nullify fields by pattern or unconditionally. diff --git a/docs/plugins/parentwork.py b/docs/plugins/parentwork.py new file mode 100644 index 000000000..d64934b88 --- /dev/null +++ b/docs/plugins/parentwork.py @@ -0,0 +1,60 @@ +Parentwork Plugin +================= + +The ``parentwork`` plugin fetches the work title, parentwork title and +parentwork composer. + +In the MusicBrainz database, a recording can be associated with a work. A +work can itself be associated with another work, for example one being part +of the other (what I call the father work). This plugin looks the work id +from the library and then looks up the father, then the father of the father +and so on until it reaches the top. The work at the top is what I call the +parentwork. This plugin is especially designed for classical music. For +classical music, just fetching the work title as in MusicBrainz is not +satisfying, because MusicBrainz has separate works for, for example, all the +movements of a symphony. This plugin aims to solve this problem by not only +fetching the work itself from MusicBrainz but also its parentwork which would +be, in this case, the whole symphony. + +This plugin adds five tags: + +- **parentwork**: The title of the parentwork. +- **mb_parentworkid**: The musicbrainz id of the parentwork. +- **parentwork_disambig**: The disambiguation of the parentwork title. +- **parent_composer**: The composer of the parentwork. +- **parent_composer_sort**: The sort name of the parentwork composer. +- **work_date**: THe composition date of the work, or the first parent work + that has a composition date. Format: yyyy-mm-dd. + +To fill in the parentwork tag and the associated parent** tags, in case there +are several works on the recording, it fills it with the results of the first +work and then appends the results of the second work only if they differ from +the ones already there. This is to care for cases of, for example, an opera +recording that contains several scenes of the opera: neither the parentwork +nor all the associated tags will be duplicated. +If there are several works linked to a recording, they all get a +disambiguation (empty as default) and if all disambiguations are empty, the +disambiguation field is left empty, else the disambiguation field can look +like ``,disambig,,`` (if there are four works and only the second has a +disambiguation) if only the second work has a disambiguation. This may +seem clumsy but it allows to identify which of the four works the +disambiguation belongs to. + +To use the ``parentwork`` plugin, enable it in your configuration (see +:ref:`using-plugins`). + +Configuration +------------- + +To configure the plugin, make a ``parentwork:`` section in your +configuration file. The available options are: + +- **force**: As a default, ``parentwork`` only fetches work info for + recordings that do not already have a ``parentwork`` tag. If ``force`` + is enabled, it fetches it for all recordings. + Default: ``no`` + +- **auto**: If enabled, automatically fetches works at import. It takes quite + some time, because beets is restricted to one musicbrainz query per second. + Default: ``no`` + diff --git a/test/test_parentwork.py b/test/test_parentwork.py new file mode 100644 index 000000000..44545c63e --- /dev/null +++ b/test/test_parentwork.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2017, Dorian Soergel +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Tests for the 'parentwork' plugin.""" + +from __future__ import division, absolute_import, print_function + +from mock import patch +import unittest +from test.helper import TestHelper + +from beets.library import Item +from beetsplug import parentwork + + +@patch('beets.util.command_output') +class ParentWorkTest(unittest.TestCase, TestHelper): + def setUp(self): + """Set up configuration""" + self.setup_beets() + self.load_plugins('parentwork') + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() + + def test_normal_case(self, command_output): + item = Item(path='/file', + mb_workid=u'e27bda6e-531e-36d3-9cd7-b8ebc18e8c53') + item.add(self.lib) + + command_output.return_value = u'32c8943f-1b27-3a23-8660-4567f4847c94' + self.run_command('parentwork') + + item.load() + self.assertEqual(item['mb_parentworkid'], + u'32c8943f-1b27-3a23-8660-4567f4847c94') + + def test_force(self, command_output): + self.config['parentwork']['force'] = True + item = Item(path='/file', + mb_workid=u'e27bda6e-531e-36d3-9cd7-b8ebc18e8c53', + mb_parentworkid=u'XXX') + item.add(self.lib) + + command_output.return_value = u'32c8943f-1b27-3a23-8660-4567f4847c94' + self.run_command('parentwork') + + item.load() + self.assertEqual(item['mb_parentworkid'], + u'32c8943f-1b27-3a23-8660-4567f4847c94') + + def test_no_force(self, command_output): + self.config['parentwork']['force'] = True + item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-\ + b8ebc18e8c53', mb_parentworkid=u'XXX') + item.add(self.lib) + + command_output.return_value = u'32c8943f-1b27-3a23-8660-4567f4847c94' + self.run_command('parentwork') + + item.load() + self.assertEqual(item['mb_parentworkid'], u'XXX') + + # test different cases, still with Matthew Passion Ouverture or Mozart + # requiem + + def test_father_work(self, command_output): + mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' + self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', + parentwork.work_father(mb_workid)[0]) + self.assertEqual(u'45afb3b2-18ac-4187-bc72-beb1b1c194ba', + parentwork.work_parent(mb_workid)[0]) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 6d6c1a16473212be6f28dccec4711de515e6ee6b Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 15:04:00 +0200 Subject: [PATCH 013/613] fixes for disambiguation --- beetsplug/parentwork.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 8a8c0c12a..869e8ea96 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -134,10 +134,11 @@ class ParentWorkPlugin(BeetsPlugin): mb_parentworkid = work_info['work']['id'] if 'disambiguation' in work_info['work']: parentwork_disambig = work_info['work']['disambiguation'] + return [parentwork, mb_parentworkid, parent_composer, + parent_composer_sort, parentwork_disambig] else: - parentwork_disambig.append('') - return parentwork, mb_parentworkid, parentwork_disambig, - parent_composer, parent_composer_sort + return [parentwork, mb_parentworkid, parent_composer, + parent_composer_sort, None] def find_work(self, item, force): @@ -147,27 +148,29 @@ class ParentWorkPlugin(BeetsPlugin): hasparent = True except AttributeError: hasparent = False - hasawork = True if not item.mb_workid: self._log.info("No work attached, recording id: " + recording_id) self._log.info(item.artist + ' - ' + item.title) self._log.info("add one at https://musicbrainz.org" + "/recording/" + recording_id) - hasawork = False + return found = False - - if hasawork and (force or (not hasparent)): + if force or (not hasparent): try: work_info, work_date = find_parentwork(item.mb_workid) - (parentwork, mb_parentworkid, parentwork_disambig, - parent_composer, - parent_composer_sort) = self.get_info(item, work_info) + parent_info = self.get_info(item, work_info) + parentwork = parent_info[0] + mb_parentworkid = parent_info[1] + parent_composer = parent_info[2] + parent_composer_sort = parent_info[3] + parentwork_disambig = parent_info[4] + found = True except musicbrainzngs.musicbrainz.WebServiceError: self._log.debug("Work unreachable") found = False - elif parentwork: + elif hasparent: self._log.debug("Work already in library, not necessary fetching") return @@ -177,7 +180,8 @@ class ParentWorkPlugin(BeetsPlugin): self._log.debug("Work fetched: " + parentwork + ' - ' + u', '.join(parent_composer)) item['parentwork'] = parentwork - item['parentwork_disambig'] = parentwork_disambig + if parentwork_disambig: + item['parentwork_disambig'] = parentwork_disambig item['mb_parentworkid'] = mb_parentworkid item['parent_composer'] = u'' item['parent_composer'] = u', '.join(parent_composer) From b28d6850596009466be20a1b3c202bf4ee9fff03 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 15:13:55 +0200 Subject: [PATCH 014/613] wrong file name for parentwork documentation --- docs/plugins/{parentwork.py => parentwork.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/plugins/{parentwork.py => parentwork.rst} (100%) diff --git a/docs/plugins/parentwork.py b/docs/plugins/parentwork.rst similarity index 100% rename from docs/plugins/parentwork.py rename to docs/plugins/parentwork.rst From 638e9d5dc866d1765a5e9f91628ad3589c8f8e4f Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 17:17:06 +0200 Subject: [PATCH 015/613] style changes, docstrings --- beetsplug/parentwork.py | 113 ++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 869e8ea96..4520edea2 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -25,8 +25,8 @@ from beets.plugins import BeetsPlugin import musicbrainzngs -def work_father(mb_workid, work_date=None): - """ This function finds the id of the father work given its id""" +def work_father_id(mb_workid, work_date=None): + """ Given a mb_workid, find the id one of the works the work is part of""" work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", "artist-rels"]) @@ -45,21 +45,21 @@ def work_father(mb_workid, work_date=None): return None, work_date -def work_parent(mb_workid): - """This function finds the parentwork id of a work given its id. """ +def work_parent_id(mb_workid): + """Find the parentwork id of a work given its id. """ work_date = None while True: - (new_mb_workid, work_date) = work_father(mb_workid, work_date) + new_mb_workid, work_date = work_father_id(mb_workid, work_date) if not new_mb_workid: return mb_workid, work_date mb_workid = new_mb_workid return mb_workid, work_date -def find_parentwork(mb_workid): - """This function gives the work relationships (dict) of a parentwork - given the id of the work""" - parent_id, work_date = work_parent(mb_workid) +def find_parentwork_info(mb_workid): + """Return the work relationships (dict) of a parentwork given the id of + the work""" + parent_id, work_date = work_parent_id(mb_workid) work_info = musicbrainzngs.get_work_by_id(parent_id, includes=["artist-rels"]) return work_info, work_date @@ -112,12 +112,13 @@ class ParentWorkPlugin(BeetsPlugin): item.store() def get_info(self, item, work_info): - """Given the parentwork info dict, this function fetches - parent_composer, parent_composer_sort, parentwork, - parentwork_disambig, mb_workid and composer_ids""" + """Given the parentwork info dict, fetch parent_composer, + parent_composer_sort, parentwork, parentwork_disambig, mb_workid and + composer_ids. """ parent_composer = [] parent_composer_sort = [] + parentwork_info = {} composer_exists = False if 'artist-relation-list' in work_info['work']: @@ -125,73 +126,73 @@ class ParentWorkPlugin(BeetsPlugin): if artist['type'] == 'composer': parent_composer.append(artist['artist']['name']) parent_composer_sort.append(artist['artist']['sort-name']) + + parentwork_info['parent_composer'] = u', '.join(parent_composer) + parentwork_info['parent_composer_sort'] = u', '.join( + parent_composer_sort) + if not composer_exists: self._log.info(item.artist + ' - ' + item.title) self._log.debug( "no composer, add one at https://musicbrainz.org/work/" + work_info['work']['id']) - parentwork = work_info['work']['title'] - mb_parentworkid = work_info['work']['id'] + + parentwork_info['parentwork'] = work_info['work']['title'] + parentwork_info['mb_parentworkid'] = work_info['work']['id'] + if 'disambiguation' in work_info['work']: - parentwork_disambig = work_info['work']['disambiguation'] - return [parentwork, mb_parentworkid, parent_composer, - parent_composer_sort, parentwork_disambig] + parentwork_info['parentwork_disambig'] = work_info[ + 'work']['disambiguation'] + else: - return [parentwork, mb_parentworkid, parent_composer, - parent_composer_sort, None] + parentwork_info['parentwork_disambig'] = None + + return parentwork_info def find_work(self, item, force): + """ Finds the parentwork of a recording and populates the tags + accordingly. - recording_id = item.mb_trackid - try: - item.parentwork + Namely, the tags parentwork, parentwork_disambig, mb_parentworkid, + parent_composer, parent_composer_sort and work_date are populated. """ + + if hasattr(item, 'parentwork'): hasparent = True - except AttributeError: + else: hasparent = False if not item.mb_workid: self._log.info("No work attached, recording id: " + - recording_id) + item.mb_trackid) self._log.info(item.artist + ' - ' + item.title) self._log.info("add one at https://musicbrainz.org" + - "/recording/" + recording_id) + "/recording/" + item.mb_trackid) return - found = False if force or (not hasparent): try: - work_info, work_date = find_parentwork(item.mb_workid) - parent_info = self.get_info(item, work_info) - parentwork = parent_info[0] - mb_parentworkid = parent_info[1] - parent_composer = parent_info[2] - parent_composer_sort = parent_info[3] - parentwork_disambig = parent_info[4] - - found = True + work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError: self._log.debug("Work unreachable") - found = False + return + parent_info = self.get_info(item, work_info) + elif hasparent: self._log.debug("Work already in library, not necessary fetching") return - if found: - self._log.debug("Finished searching work for: " + - item.artist + ' - ' + item.title) - self._log.debug("Work fetched: " + parentwork + - ' - ' + u', '.join(parent_composer)) - item['parentwork'] = parentwork - if parentwork_disambig: - item['parentwork_disambig'] = parentwork_disambig - item['mb_parentworkid'] = mb_parentworkid - item['parent_composer'] = u'' - item['parent_composer'] = u', '.join(parent_composer) - item['parent_composer_sort'] = u'' - item['parent_composer_sort'] = u', '.join(parent_composer_sort) - if work_date: - item['work_date'] = work_date - ui.show_model_changes( - item, fields=['parentwork', 'parentwork_disambig', - 'mb_parentworkid', 'parent_composer', - 'parent_composer_sort', 'work_date']) + self._log.debug("Finished searching work for: " + + item.artist + ' - ' + item.title) + self._log.debug("Work fetched: " + parent_info['parentwork'] + + ' - ' + parent_info['parent_composer']) - item.store() + for key, value in parent_info.items(): + if value: + item[key] = value + + if work_date: + item['work_date'] = work_date + ui.show_model_changes( + item, fields=['parentwork', 'parentwork_disambig', + 'mb_parentworkid', 'parent_composer', + 'parent_composer_sort', 'work_date']) + + item.store() From acf447b4b01f8c9296c05245975e954d612966a7 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 17:52:39 +0200 Subject: [PATCH 016/613] adapt tests, correct docstrings --- beetsplug/parentwork.py | 9 +++++---- test/test_parentwork.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 4520edea2..e03a142d4 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -26,7 +26,8 @@ import musicbrainzngs def work_father_id(mb_workid, work_date=None): - """ Given a mb_workid, find the id one of the works the work is part of""" + """ Given a mb_workid, find the id one of the works the work is part of + and the first composition date it encounters. """ work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", "artist-rels"]) @@ -46,7 +47,7 @@ def work_father_id(mb_workid, work_date=None): def work_parent_id(mb_workid): - """Find the parentwork id of a work given its id. """ + """Find the parentwork id and composition date of a work given its id. """ work_date = None while True: new_mb_workid, work_date = work_father_id(mb_workid, work_date) @@ -57,8 +58,8 @@ def work_parent_id(mb_workid): def find_parentwork_info(mb_workid): - """Return the work relationships (dict) of a parentwork given the id of - the work""" + """Return the work relationships (dict) and composition date of a + parentwork given the id of the work""" parent_id, work_date = work_parent_id(mb_workid) work_info = musicbrainzngs.get_work_by_id(parent_id, includes=["artist-rels"]) diff --git a/test/test_parentwork.py b/test/test_parentwork.py index 44545c63e..985723fb5 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -80,9 +80,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): def test_father_work(self, command_output): mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', - parentwork.work_father(mb_workid)[0]) + parentwork.work_father_id(mb_workid)[0]) self.assertEqual(u'45afb3b2-18ac-4187-bc72-beb1b1c194ba', - parentwork.work_parent(mb_workid)[0]) + parentwork.work_parent_id(mb_workid)[0]) def suite(): From e6da3e149815b93d99b3a7fbf400f0845b55cd3c Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 18:28:53 +0200 Subject: [PATCH 017/613] move _command into command --- beetsplug/parentwork.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index e03a142d4..94d32f41e 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -75,15 +75,6 @@ class ParentWorkPlugin(BeetsPlugin): 'force': False, }) - self._command = ui.Subcommand( - 'parentwork', - help=u'Fetches parent works, composers and dates') - - self._command.parser.add_option( - u'-f', u'--force', dest='force', - action='store_true', default=None, - help=u'Re-fetches all parent works') - if self.config['auto']: self.import_stages = [self.imported] @@ -99,9 +90,17 @@ class ParentWorkPlugin(BeetsPlugin): item.store() if write: item.try_write() + command = ui.Subcommand( + 'parentwork', + help=u'Fetches parent works, composers and dates') - self._command.func = func - return [self._command] + command.parser.add_option( + u'-f', u'--force', dest='force', + action='store_true', default=None, + help=u'Re-fetches all parent works') + + command.func = func + return [command] def imported(self, session, task): """Import hook for fetching parent works automatically. From a10ad548c9d5acf84c29c5bb33fef24247f44120 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 31 May 2019 20:54:15 +0200 Subject: [PATCH 018/613] logging if no parent composer --- beetsplug/parentwork.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 94d32f41e..6350a7200 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -181,8 +181,11 @@ class ParentWorkPlugin(BeetsPlugin): self._log.debug("Finished searching work for: " + item.artist + ' - ' + item.title) - self._log.debug("Work fetched: " + parent_info['parentwork'] + - ' - ' + parent_info['parent_composer']) + if parent_info['parent_composer']: + self._log.debug("Work fetched: " + parent_info['parentwork'] + + ' - ' + parent_info['parent_composer']) + else: + self._log.debug("Work fetched: " + parent_info['parentwork']) for key, value in parent_info.items(): if value: From feafc66f96d51c0e26687abf150ee39b79a2990e Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 16:58:07 +0200 Subject: [PATCH 019/613] fixing parentwork but no parent composer --- beetsplug/parentwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 6350a7200..41b4fa461 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -181,7 +181,7 @@ class ParentWorkPlugin(BeetsPlugin): self._log.debug("Finished searching work for: " + item.artist + ' - ' + item.title) - if parent_info['parent_composer']: + if hasattr(item, 'parentwork'): self._log.debug("Work fetched: " + parent_info['parentwork'] + ' - ' + parent_info['parent_composer']) else: From 369629bea5059c7955ffaaeb4214be5aac0885e8 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 17:20:36 +0200 Subject: [PATCH 020/613] clarifying docstrings --- beetsplug/parentwork.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 41b4fa461..00c6635db 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -13,8 +13,8 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Gets work title, disambiguation, parent work and its disambiguation, -composer, composer sort name and performers +"""Gets parent work, its disambiguation and id, composer, composer sort name +and work composition date """ from __future__ import division, absolute_import, print_function @@ -27,7 +27,10 @@ import musicbrainzngs def work_father_id(mb_workid, work_date=None): """ Given a mb_workid, find the id one of the works the work is part of - and the first composition date it encounters. """ + and the first composition date it encounters. + + For a give work, hat we call father_work is the work it is part of. + The parent_work is the furthest ancestor.""" work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", "artist-rels"]) From 1177222c6f4c5660afa52276ec9c559dbf6d6bca Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 17:45:57 +0200 Subject: [PATCH 021/613] flake8 --- beetsplug/parentwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 00c6635db..78b7e129a 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -29,7 +29,7 @@ def work_father_id(mb_workid, work_date=None): """ Given a mb_workid, find the id one of the works the work is part of and the first composition date it encounters. - For a give work, hat we call father_work is the work it is part of. + For a give work, hat we call father_work is the work it is part of. The parent_work is the furthest ancestor.""" work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", From 380003a2fb23ea9cbfb1f148dd1db2af18209950 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 20:01:02 +0200 Subject: [PATCH 022/613] fix documentation --- docs/plugins/parentwork.rst | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index d64934b88..d76070b33 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -26,20 +26,6 @@ This plugin adds five tags: - **work_date**: THe composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. -To fill in the parentwork tag and the associated parent** tags, in case there -are several works on the recording, it fills it with the results of the first -work and then appends the results of the second work only if they differ from -the ones already there. This is to care for cases of, for example, an opera -recording that contains several scenes of the opera: neither the parentwork -nor all the associated tags will be duplicated. -If there are several works linked to a recording, they all get a -disambiguation (empty as default) and if all disambiguations are empty, the -disambiguation field is left empty, else the disambiguation field can look -like ``,disambig,,`` (if there are four works and only the second has a -disambiguation) if only the second work has a disambiguation. This may -seem clumsy but it allows to identify which of the four works the -disambiguation belongs to. - To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From 92d005ab30c9351745bffb8d5f37079dba04acf0 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 20:05:34 +0200 Subject: [PATCH 023/613] renaming functions --- beetsplug/parentwork.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 78b7e129a..753f1fa5f 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -25,12 +25,9 @@ from beets.plugins import BeetsPlugin import musicbrainzngs -def work_father_id(mb_workid, work_date=None): +def direct_parent_id(mb_workid, work_date=None): """ Given a mb_workid, find the id one of the works the work is part of - and the first composition date it encounters. - - For a give work, hat we call father_work is the work it is part of. - The parent_work is the furthest ancestor.""" + and the first composition date it encounters.""" work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", "artist-rels"]) @@ -53,7 +50,7 @@ def work_parent_id(mb_workid): """Find the parentwork id and composition date of a work given its id. """ work_date = None while True: - new_mb_workid, work_date = work_father_id(mb_workid, work_date) + new_mb_workid, work_date = direct_parent_id(mb_workid, work_date) if not new_mb_workid: return mb_workid, work_date mb_workid = new_mb_workid From b3b59f8452004694c64d9769123bbb693d4bd391 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 20:14:43 +0200 Subject: [PATCH 024/613] rename functions in test --- test/test_parentwork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_parentwork.py b/test/test_parentwork.py index 985723fb5..885773906 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -80,9 +80,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): def test_father_work(self, command_output): mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', - parentwork.work_father_id(mb_workid)[0]) + parentwork.direct_parent_id(mb_workid)[0]) self.assertEqual(u'45afb3b2-18ac-4187-bc72-beb1b1c194ba', - parentwork.work_parent_id(mb_workid)[0]) + parentwork.direct_parent_id(mb_workid)[0]) def suite(): From a71c381bb5f800522e961d02a80a8994448357e8 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Mon, 3 Jun 2019 20:33:49 +0200 Subject: [PATCH 025/613] rename functions in test --- test/test_parentwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_parentwork.py b/test/test_parentwork.py index 885773906..8447fcdd1 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -82,7 +82,7 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', parentwork.direct_parent_id(mb_workid)[0]) self.assertEqual(u'45afb3b2-18ac-4187-bc72-beb1b1c194ba', - parentwork.direct_parent_id(mb_workid)[0]) + parentwork.work_parent_id(mb_workid)[0]) def suite(): From aa31fea037ce82ebfafb4dd7d596b7aa36e8b2d6 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 01:07:31 +0200 Subject: [PATCH 026/613] Update a lot of URLs to use HTTPS *All* URLs were checked manually, but only once per domain! I mostly concerned myself with URLs in documentation rather than source code because the latter may or may not have impactful changes, while the former should be straight forward. Changes in addition to simply adding an s: - changed pip and pypi references as their location has changed - MPoD (iOS app) url redirects to Regelian, so I replaced those - updated homebrew references Notable observations: - beets.io does have HTTPS set up properly (via gh-pages) - beatport.py uses the old HTTP url for beatport - as does lyrics.py for lyrics.wikia.com - https://tomahawk-player.org/ expired long ago, but the http page redirects to https regardless - none of the sourceforge subdomains have https (in 2019!) --- .travis.yml | 4 +-- README.rst | 42 ++++++++++++++--------------- README_kr.rst | 46 +++++++++++++++---------------- beets/dbcore/types.py | 2 +- beets/ui/commands.py | 2 +- beetsplug/absubmit.py | 4 +-- beetsplug/fetchart.py | 10 +++++-- beetsplug/keyfinder.py | 2 +- beetsplug/lastimport.py | 2 +- beetsplug/mbsubmit.py | 2 +- beetsplug/thumbnails.py | 4 +-- beetsplug/web/static/beets.js | 2 +- docs/changelog.rst | 48 ++++++++++++++++----------------- docs/dev/index.rst | 2 +- docs/dev/plugins.rst | 12 ++++----- docs/faq.rst | 22 +++++++-------- docs/guides/advanced.rst | 2 +- docs/guides/main.rst | 26 +++++++++--------- docs/guides/tagger.rst | 8 +++--- docs/index.rst | 4 +-- docs/plugins/absubmit.rst | 8 +++--- docs/plugins/acousticbrainz.rst | 2 +- docs/plugins/beatport.rst | 4 +-- docs/plugins/bpd.rst | 14 +++++----- docs/plugins/chroma.rst | 24 ++++++++--------- docs/plugins/convert.rst | 8 +++--- docs/plugins/discogs.rst | 2 +- docs/plugins/embedart.rst | 2 +- docs/plugins/embyupdate.rst | 4 +-- docs/plugins/export.rst | 2 +- docs/plugins/fetchart.rst | 8 +++--- docs/plugins/ftintitle.rst | 2 +- docs/plugins/index.rst | 18 ++++++------- docs/plugins/info.rst | 2 +- docs/plugins/ipfs.rst | 2 +- docs/plugins/keyfinder.rst | 2 +- docs/plugins/kodiupdate.rst | 4 +-- docs/plugins/lastgenre.rst | 8 +++--- docs/plugins/lastimport.rst | 4 +-- docs/plugins/lyrics.rst | 16 +++++------ docs/plugins/mbcollection.rst | 2 +- docs/plugins/mbsubmit.rst | 6 ++--- docs/plugins/metasync.rst | 2 +- docs/plugins/mpdstats.rst | 4 +-- docs/plugins/mpdupdate.rst | 2 +- docs/plugins/plexupdate.rst | 6 ++--- docs/plugins/replaygain.rst | 8 +++--- docs/plugins/sonosupdate.rst | 2 +- docs/plugins/spotify.rst | 4 +-- docs/plugins/subsonicupdate.rst | 2 +- docs/plugins/thumbnails.rst | 2 +- docs/plugins/web.rst | 12 ++++----- docs/reference/cli.rst | 4 +-- docs/reference/config.rst | 10 +++---- docs/reference/pathformat.rst | 12 ++++----- docs/reference/query.rst | 2 +- extra/_beet | 2 +- setup.py | 2 +- test/test_art.py | 6 ++--- 59 files changed, 239 insertions(+), 233 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8376be522..455ab4ca4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,8 @@ matrix: addons: apt: sources: - - sourceline: "deb http://archive.ubuntu.com/ubuntu/ trusty multiverse" - - sourceline: "deb http://archive.ubuntu.com/ubuntu/ trusty-updates multiverse" + - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty multiverse" + - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty-updates multiverse" packages: - bash-completion - gir1.2-gst-plugins-base-1.0 diff --git a/README.rst b/README.rst index a3ea6302f..6b4ebb4fa 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ -.. image:: http://img.shields.io/pypi/v/beets.svg +.. image:: https://img.shields.io/pypi/v/beets.svg :target: https://pypi.python.org/pypi/beets -.. image:: http://img.shields.io/codecov/c/github/beetbox/beets.svg +.. image:: https://img.shields.io/codecov/c/github/beetbox/beets.svg :target: https://codecov.io/github/beetbox/beets .. image:: https://travis-ci.org/beetbox/beets.svg?branch=master @@ -51,28 +51,28 @@ imagine for your music collection. Via `plugins`_, beets becomes a panacea: If beets doesn't do what you want yet, `writing your own plugin`_ is shockingly simple if you know a little Python. -.. _plugins: http://beets.readthedocs.org/page/plugins/ -.. _MPD: http://www.musicpd.org/ -.. _MusicBrainz music collection: http://musicbrainz.org/doc/Collections/ +.. _plugins: https://beets.readthedocs.org/page/plugins/ +.. _MPD: https://www.musicpd.org/ +.. _MusicBrainz music collection: https://musicbrainz.org/doc/Collections/ .. _writing your own plugin: - http://beets.readthedocs.org/page/dev/plugins.html + https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html .. _albums that are missing tracks: - http://beets.readthedocs.org/page/plugins/missing.html + https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: - http://beets.readthedocs.org/page/plugins/duplicates.html + https://beets.readthedocs.org/page/plugins/duplicates.html .. _Transcode audio: - http://beets.readthedocs.org/page/plugins/convert.html -.. _Discogs: http://www.discogs.com/ + https://beets.readthedocs.org/page/plugins/convert.html +.. _Discogs: https://www.discogs.com/ .. _acoustic fingerprints: - http://beets.readthedocs.org/page/plugins/chroma.html -.. _ReplayGain: http://beets.readthedocs.org/page/plugins/replaygain.html -.. _tempos: http://beets.readthedocs.org/page/plugins/acousticbrainz.html -.. _genres: http://beets.readthedocs.org/page/plugins/lastgenre.html -.. _album art: http://beets.readthedocs.org/page/plugins/fetchart.html -.. _lyrics: http://beets.readthedocs.org/page/plugins/lyrics.html -.. _MusicBrainz: http://musicbrainz.org/ + https://beets.readthedocs.org/page/plugins/chroma.html +.. _ReplayGain: https://beets.readthedocs.org/page/plugins/replaygain.html +.. _tempos: https://beets.readthedocs.org/page/plugins/acousticbrainz.html +.. _genres: https://beets.readthedocs.org/page/plugins/lastgenre.html +.. _album art: https://beets.readthedocs.org/page/plugins/fetchart.html +.. _lyrics: https://beets.readthedocs.org/page/plugins/lyrics.html +.. _MusicBrainz: https://musicbrainz.org/ .. _Beatport: https://www.beatport.com Install @@ -81,7 +81,7 @@ Install You can install beets by typing ``pip install beets``. Then check out the `Getting Started`_ guide. -.. _Getting Started: http://beets.readthedocs.org/page/guides/main.html +.. _Getting Started: https://beets.readthedocs.org/page/guides/main.html Contribute ---------- @@ -90,7 +90,7 @@ Check out the `Hacking`_ page on the wiki for tips on how to help out. You might also be interested in the `For Developers`_ section in the docs. .. _Hacking: https://github.com/beetbox/beets/wiki/Hacking -.. _For Developers: http://docs.beets.io/page/dev/ +.. _For Developers: https://beets.readthedocs.io/en/stable/dev/ Read More --------- @@ -99,7 +99,7 @@ Learn more about beets at `its Web site`_. Follow `@b33ts`_ on Twitter for news and updates. .. _its Web site: http://beets.io/ -.. _@b33ts: http://twitter.com/b33ts/ +.. _@b33ts: https://twitter.com/b33ts/ Authors ------- @@ -108,4 +108,4 @@ Beets is by `Adrian Sampson`_ with a supporting cast of thousands. For help, please visit our `forum`_. .. _forum: https://discourse.beets.io -.. _Adrian Sampson: http://www.cs.cornell.edu/~asampson/ +.. _Adrian Sampson: https://www.cs.cornell.edu/~asampson/ diff --git a/README_kr.rst b/README_kr.rst index 18389061c..6bdcf56a6 100644 --- a/README_kr.rst +++ b/README_kr.rst @@ -1,7 +1,7 @@ -.. image:: http://img.shields.io/pypi/v/beets.svg +.. image:: https://img.shields.io/pypi/v/beets.svg :target: https://pypi.python.org/pypi/beets -.. image:: http://img.shields.io/codecov/c/github/beetbox/beets.svg +.. image:: https://img.shields.io/codecov/c/github/beetbox/beets.svg :target: https://codecov.io/github/beetbox/beets .. image:: https://travis-ci.org/beetbox/beets.svg?branch=master @@ -48,28 +48,28 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 만약 Beets에 당신이 원하는게 아직 없다면, 당신이 python을 안다면 `writing your own plugin`_ _은 놀라울정도로 간단하다. -.. _plugins: http://beets.readthedocs.org/page/plugins/ -.. _MPD: http://www.musicpd.org/ -.. _MusicBrainz music collection: http://musicbrainz.org/doc/Collections/ +.. _plugins: https://beets.readthedocs.org/page/plugins/ +.. _MPD: https://www.musicpd.org/ +.. _MusicBrainz music collection: https://musicbrainz.org/doc/Collections/ .. _writing your own plugin: - http://beets.readthedocs.org/page/dev/plugins.html + https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html .. _albums that are missing tracks: - http://beets.readthedocs.org/page/plugins/missing.html + https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: - http://beets.readthedocs.org/page/plugins/duplicates.html + https://beets.readthedocs.org/page/plugins/duplicates.html .. _Transcode audio: - http://beets.readthedocs.org/page/plugins/convert.html -.. _Discogs: http://www.discogs.com/ + https://beets.readthedocs.org/page/plugins/convert.html +.. _Discogs: https://www.discogs.com/ .. _acoustic fingerprints: - http://beets.readthedocs.org/page/plugins/chroma.html -.. _ReplayGain: http://beets.readthedocs.org/page/plugins/replaygain.html -.. _tempos: http://beets.readthedocs.org/page/plugins/acousticbrainz.html -.. _genres: http://beets.readthedocs.org/page/plugins/lastgenre.html -.. _album art: http://beets.readthedocs.org/page/plugins/fetchart.html -.. _lyrics: http://beets.readthedocs.org/page/plugins/lyrics.html -.. _MusicBrainz: http://musicbrainz.org/ + https://beets.readthedocs.org/page/plugins/chroma.html +.. _ReplayGain: https://beets.readthedocs.org/page/plugins/replaygain.html +.. _tempos: https://beets.readthedocs.org/page/plugins/acousticbrainz.html +.. _genres: https://beets.readthedocs.org/page/plugins/lastgenre.html +.. _album art: https://beets.readthedocs.org/page/plugins/fetchart.html +.. _lyrics: https://beets.readthedocs.org/page/plugins/lyrics.html +.. _MusicBrainz: https://musicbrainz.org/ .. _Beatport: https://www.beatport.com 설치 @@ -78,7 +78,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 당신은 ``pip install beets`` 을 사용해서 Beets를 설치할 수 있다. 그리고 `Getting Started`_ 가이드를 확인할 수 있다. -.. _Getting Started: http://beets.readthedocs.org/page/guides/main.html +.. _Getting Started: https://beets.readthedocs.org/page/guides/main.html 컨트리뷰션 ---------- @@ -87,16 +87,16 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 당신은 docs 안에 `For Developers`_ 에도 관심이 있을수 있다. .. _Hacking: https://github.com/beetbox/beets/wiki/Hacking -.. _For Developers: http://docs.beets.io/page/dev/ +.. _For Developers: https://beets.readthedocs.io/en/stable/dev/ Read More --------- -`its Web site`_ 에서 Beets에 대해 조금 더 알아볼 수 있다. +`its Web site`_ 에서 Beets에 대해 조금 더 알아볼 수 있다. 트위터에서 `@b33ts`_ 를 팔로우하면 새 소식을 볼 수 있다. -.. _its Web site: http://beets.io/ -.. _@b33ts: http://twitter.com/b33ts/ +.. _its Web site: https://beets.io/ +.. _@b33ts: https://twitter.com/b33ts/ 저자들 ------- @@ -105,4 +105,4 @@ Read More 돕고 싶다면 `forum`_.를 방문하면 된다. .. _forum: https://discourse.beets.io -.. _Adrian Sampson: http://www.cs.cornell.edu/~asampson/ +.. _Adrian Sampson: https://www.cs.cornell.edu/~asampson/ diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index c37def875..521a5a1ee 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -97,7 +97,7 @@ class Type(object): For fixed fields the type of `value` is determined by the column type affinity given in the `sql` property and the SQL to Python mapping of the database adapter. For more information see: - http://www.sqlite.org/datatype3.html + https://www.sqlite.org/datatype3.html https://docs.python.org/2/library/sqlite3.html#sqlite-and-python-types Flexible fields have the type affinity `TEXT`. This means the diff --git a/beets/ui/commands.py b/beets/ui/commands.py index ef4fd144a..53253c1da 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -542,7 +542,7 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, print_(u"No matching release found for {0} tracks." .format(itemcount)) print_(u'For help, see: ' - u'http://beets.readthedocs.org/en/latest/faq.html#nomatch') + u'https://beets.readthedocs.org/en/latest/faq.html#nomatch') sel = ui.input_options(choice_opts) if sel in choice_actions: return choice_actions[sel] diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 9d26ac5db..d9525e1d2 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -73,8 +73,8 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): call([self.extractor]) except OSError: raise ui.UserError( - u'No extractor command found: please install the ' - u'extractor binary from http://acousticbrainz.org/download' + u'No extractor command found: please install the extractor' + u' binary from https://acousticbrainz.org/download' ) except ABSubmitError: # Extractor found, will exit with an error if not called with diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index af1aaa567..a815d4d9b 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -311,7 +311,10 @@ class CoverArtArchive(RemoteArtSource): class Amazon(RemoteArtSource): NAME = u"Amazon" - URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' + if util.SNI_SUPPORTED: + URL = 'https://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' + else: + URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg' INDICES = (1, 2) def get(self, album, plugin, paths): @@ -325,7 +328,10 @@ class Amazon(RemoteArtSource): class AlbumArtOrg(RemoteArtSource): NAME = u"AlbumArt.org scraper" - URL = 'http://www.albumart.org/index_detail.php' + if util.SNI_SUPPORTED: + URL = 'https://www.albumart.org/index_detail.php' + else: + URL = 'http://www.albumart.org/index_detail.php' PAT = r'href\s*=\s*"([^>"]*)"[^>]*title\s*=\s*"View larger image"' def get(self, album, plugin, paths): diff --git a/beetsplug/keyfinder.py b/beetsplug/keyfinder.py index a3fbc8211..3a738478e 100644 --- a/beetsplug/keyfinder.py +++ b/beetsplug/keyfinder.py @@ -66,7 +66,7 @@ class KeyFinderPlugin(BeetsPlugin): continue except UnicodeEncodeError: # Workaround for Python 2 Windows bug. - # http://bugs.python.org/issue1759845 + # https://bugs.python.org/issue1759845 self._log.error(u'execution failed for Unicode path: {0!r}', item.path) continue diff --git a/beetsplug/lastimport.py b/beetsplug/lastimport.py index d7b84b0aa..ca97004cf 100644 --- a/beetsplug/lastimport.py +++ b/beetsplug/lastimport.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # This file is part of beets. -# Copyright 2016, Rafael Bodill http://github.com/rafi +# Copyright 2016, Rafael Bodill https://github.com/rafi # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the diff --git a/beetsplug/mbsubmit.py b/beetsplug/mbsubmit.py index 02bd5f697..44a476d15 100644 --- a/beetsplug/mbsubmit.py +++ b/beetsplug/mbsubmit.py @@ -19,7 +19,7 @@ This plugin allows the user to print track information in a format that is parseable by the MusicBrainz track parser [1]. Programmatic submitting is not implemented by MusicBrainz yet. -[1] http://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings +[1] https://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings """ from __future__ import division, absolute_import, print_function diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index 04845e880..fe36fbd13 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -160,7 +160,7 @@ class ThumbnailsPlugin(BeetsPlugin): def thumbnail_file_name(self, path): """Compute the thumbnail file name - See http://standards.freedesktop.org/thumbnail-spec/latest/x227.html + See https://standards.freedesktop.org/thumbnail-spec/latest/x227.html """ uri = self.get_uri(path) hash = md5(uri.encode('utf-8')).hexdigest() @@ -168,7 +168,7 @@ class ThumbnailsPlugin(BeetsPlugin): def add_tags(self, album, image_path): """Write required metadata to the thumbnail - See http://standards.freedesktop.org/thumbnail-spec/latest/x142.html + See https://standards.freedesktop.org/thumbnail-spec/latest/x142.html """ mtime = os.stat(album.artpath).st_mtime metadata = {"Thumb::URI": self.get_uri(album.artpath), diff --git a/beetsplug/web/static/beets.js b/beetsplug/web/static/beets.js index 51985c183..97af70110 100644 --- a/beetsplug/web/static/beets.js +++ b/beetsplug/web/static/beets.js @@ -129,7 +129,7 @@ $.fn.player = function(debug) { // Simple selection disable for jQuery. // Cut-and-paste from: -// http://stackoverflow.com/questions/2700000 +// https://stackoverflow.com/questions/2700000 $.fn.disableSelection = function() { $(this).attr('unselectable', 'on') .css('-moz-user-select', 'none') diff --git a/docs/changelog.rst b/docs/changelog.rst index e58325482..a667be780 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1216,7 +1216,7 @@ There are even more new features: don't actually need to be moved. :bug:`1583` .. _Google Code-In: https://codein.withgoogle.com/ -.. _AcousticBrainz: http://acousticbrainz.org/ +.. _AcousticBrainz: https://acousticbrainz.org/ Fixes: @@ -1358,7 +1358,7 @@ Fixes: communication errors. The backend has also been disabled by default, since the API it depends on is currently down. :bug:`1770` -.. _Emby: http://emby.media +.. _Emby: https://emby.media 1.3.15 (October 17, 2015) @@ -1520,8 +1520,8 @@ Fixes: * :doc:`/plugins/convert`: Fix a problem with filename encoding on Windows under Python 3. :bug:`2515` :bug:`2516` -.. _Python bug: http://bugs.python.org/issue16512 -.. _ipfs: http://ipfs.io +.. _Python bug: https://bugs.python.org/issue16512 +.. _ipfs: https://ipfs.io 1.3.13 (April 24, 2015) @@ -1872,7 +1872,7 @@ As usual, there are loads of little fixes and improvements: * The :ref:`config-cmd` command can now use ``$EDITOR`` variables with arguments. -.. _API changes: http://developer.echonest.com/forums/thread/3650 +.. _API changes: https://developer.echonest.com/forums/thread/3650 .. _Plex: https://plex.tv/ .. _musixmatch: https://www.musixmatch.com/ @@ -2352,7 +2352,7 @@ Fixes: * :doc:`/plugins/convert`: Display a useful error message when the FFmpeg executable can't be found. -.. _requests: http://www.python-requests.org/ +.. _requests: https://www.python-requests.org/ 1.3.3 (February 26, 2014) @@ -2534,7 +2534,7 @@ As usual, there are also innumerable little fixes and improvements: .. _Acoustic Attributes: http://developer.echonest.com/acoustic-attributes.html -.. _MPD: http://www.musicpd.org/ +.. _MPD: https://www.musicpd.org/ 1.3.1 (October 12, 2013) @@ -2601,7 +2601,7 @@ And some fixes: * :doc:`/plugins/scrub`: Avoid preserving certain non-standard ID3 tags such as NCON. -.. _Opus: http://www.opus-codec.org/ +.. _Opus: https://www.opus-codec.org/ .. _@Verrus: https://github.com/Verrus @@ -2833,8 +2833,8 @@ And a batch of fixes: * :doc:`/plugins/lyrics`: Lyrics searches should now turn up more results due to some fixes in dealing with special characters. -.. _Discogs: http://discogs.com/ -.. _Beatport: http://www.beatport.com/ +.. _Discogs: https://discogs.com/ +.. _Beatport: https://www.beatport.com/ 1.1.0 (April 29, 2013) @@ -2883,7 +2883,7 @@ will automatically migrate your configuration to the new system. header. Thanks to Uwe L. Korn. * :doc:`/plugins/lastgenre`: Fix an error when using genre canonicalization. -.. _Tomahawk: http://www.tomahawk-player.org/ +.. _Tomahawk: https://tomahawk-player.org/ 1.1b3 (March 16, 2013) ---------------------- @@ -3058,7 +3058,7 @@ Other new stuff: (YAML doesn't like tabs.) * Fix the ``-l`` (log path) command-line option for the ``import`` command. -.. _iTunes Sound Check: http://support.apple.com/kb/HT2425 +.. _iTunes Sound Check: https://support.apple.com/kb/HT2425 1.1b1 (January 29, 2013) ------------------------ @@ -3067,7 +3067,7 @@ This release entirely revamps beets' configuration system. The configuration file is now a `YAML`_ document and is located, along with other support files, in a common directory (e.g., ``~/.config/beets`` on Unix-like systems). -.. _YAML: http://en.wikipedia.org/wiki/YAML +.. _YAML: https://en.wikipedia.org/wiki/YAML * Renamed plugins: The ``rdm`` plugin has been renamed to ``random`` and ``fuzzy_search`` has been renamed to ``fuzzy``. @@ -3229,7 +3229,7 @@ begins today on features for version 1.1. .. _The Echo Nest: http://the.echonest.com/ .. _Tomahawk resolver: http://beets.io/blog/tomahawk-resolver.html .. _mp3gain: http://mp3gain.sourceforge.net/download.php -.. _aacgain: http://aacgain.altosdesign.com +.. _aacgain: https://aacgain.altosdesign.com 1.0b15 (July 26, 2012) ---------------------- @@ -3338,7 +3338,7 @@ fetching cover art for your music, enable this plugin after upgrading to beets database with ``beet import -AWC /path/to/music``. * Fix ``import`` with relative path arguments on Windows. -.. _artist credits: http://wiki.musicbrainz.org/Artist_Credit +.. _artist credits: https://wiki.musicbrainz.org/Artist_Credit 1.0b14 (May 12, 2012) --------------------- @@ -3496,7 +3496,7 @@ to come in the next couple of releases. data. * Fix the ``list`` command in BPD (thanks to Simon Chopin). -.. _Colorama: http://pypi.python.org/pypi/colorama +.. _Colorama: https://pypi.python.org/pypi/colorama 1.0b12 (January 16, 2012) ------------------------- @@ -3609,12 +3609,12 @@ release: one for assigning genres and another for ReplayGain analysis. corrupted. .. _KraYmer: https://github.com/KraYmer -.. _Next Generation Schema: http://musicbrainz.org/doc/XML_Web_Service/Version_2 +.. _Next Generation Schema: https://musicbrainz.org/doc/XML_Web_Service/Version_2 .. _python-musicbrainzngs: https://github.com/alastair/python-musicbrainzngs -.. _acoustid: http://acoustid.org/ +.. _acoustid: https://acoustid.org/ .. _Peter Brunner: https://github.com/Lugoues .. _Simon Chopin: https://github.com/laarmen -.. _albumart.org: http://www.albumart.org/ +.. _albumart.org: https://www.albumart.org/ 1.0b10 (September 22, 2011) --------------------------- @@ -3783,8 +3783,8 @@ below, for a plethora of new features. * Fix a crash on album queries with item-only field names. -.. _xargs: http://en.wikipedia.org/wiki/xargs -.. _unidecode: http://pypi.python.org/pypi/Unidecode/0.04.1 +.. _xargs: https://en.wikipedia.org/wiki/xargs +.. _unidecode: https://pypi.python.org/pypi/Unidecode/0.04.1 1.0b8 (April 28, 2011) ---------------------- @@ -3927,7 +3927,7 @@ new configuration options and the ability to clean up empty directory subtrees. * The old "albumify" plugin for upgrading databases was removed. -.. _as specified by MusicBrainz: http://wiki.musicbrainz.org/ReleaseType +.. _as specified by MusicBrainz: https://wiki.musicbrainz.org/ReleaseType 1.0b6 (January 20, 2011) ------------------------ @@ -4043,7 +4043,7 @@ are also rolled into this release. * Fixed escaping of ``/`` characters in paths on Windows. -.. _!!!: http://musicbrainz.org/artist/f26c72d3-e52c-467b-b651-679c73d8e1a7.html +.. _!!!: https://musicbrainz.org/artist/f26c72d3-e52c-467b-b651-679c73d8e1a7.html 1.0b4 (August 9, 2010) ---------------------- @@ -4232,7 +4232,7 @@ Vorbis) and an option to log untaggable albums during import. removed dependency on the aging ``cmdln`` module in favor of `a hand-rolled solution`_. -.. _a hand-rolled solution: http://gist.github.com/462717 +.. _a hand-rolled solution: https://gist.github.com/462717 1.0b1 (June 17, 2010) --------------------- diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 45640254c..a47d6c8f2 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -7,7 +7,7 @@ in hacking beets itself or creating plugins for it. See also the documentation for `MediaFile`_, the library used by beets to read and write metadata tags in media files. -.. _MediaFile: http://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/ .. toctree:: diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 7ff397bc6..3328654e0 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -15,7 +15,7 @@ structure should look like this:: myawesomeplugin.py .. _Stack Overflow question about namespace packages: - http://stackoverflow.com/questions/1675734/how-do-i-create-a-namespace-package-in-python/1676069#1676069 + https://stackoverflow.com/questions/1675734/how-do-i-create-a-namespace-package-in-python/1676069#1676069 Then, you'll need to put this stuff in ``__init__.py`` to make ``beetsplug`` a namespace package:: @@ -42,7 +42,7 @@ Then, as described above, edit your ``config.yaml`` to include ``plugins: myawesomeplugin`` (substituting the name of the Python module containing your plugin). -.. _virtualenv: http://pypi.python.org/pypi/virtualenv +.. _virtualenv: https://pypi.org/project/virtualenv .. _add_subcommands: @@ -73,7 +73,7 @@ but it defaults to an empty parser (you can extend it later). ``help`` is a description of your command, and ``aliases`` is a list of shorthand versions of your command name. -.. _OptionParser instance: http://docs.python.org/library/optparse.html +.. _OptionParser instance: https://docs.python.org/library/optparse.html You'll need to add a function to your command by saying ``mycommand.func = myfunction``. This function should take the following parameters: ``lib`` (a @@ -81,7 +81,7 @@ beets ``Library`` object) and ``opts`` and ``args`` (command-line options and arguments as returned by `OptionParser.parse_args`_). .. _OptionParser.parse_args: - http://docs.python.org/library/optparse.html#parsing-arguments + https://docs.python.org/library/optparse.html#parsing-arguments The function should use any of the utility functions defined in ``beets.ui``. Try running ``pydoc beets.ui`` to see what's available. @@ -301,7 +301,7 @@ To access this value, say ``self.config['foo'].get()`` at any point in your plugin's code. The `self.config` object is a *view* as defined by the `Confuse`_ library. -.. _Confuse: http://confuse.readthedocs.org/ +.. _Confuse: https://confuse.readthedocs.org/ If you want to access configuration values *outside* of your plugin's section, import the `config` object from the `beets` module. That is, just put ``from @@ -379,7 +379,7 @@ access to file tags. If you have created a descriptor you can add it through your plugins ``add_media_field()`` method. .. automethod:: beets.plugins.BeetsPlugin.add_media_field -.. _MediaFile: http://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/ Here's an example plugin that provides a meaningless new field "foo":: diff --git a/docs/faq.rst b/docs/faq.rst index b7ec10df5..9732a4725 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -6,8 +6,8 @@ Got a question that isn't answered here? Try `IRC`_, the `discussion board`_, or :ref:`filing an issue ` in the bug tracker. .. _IRC: irc://irc.freenode.net/beets -.. _mailing list: http://groups.google.com/group/beets-users -.. _discussion board: http://discourse.beets.io +.. _mailing list: https://groups.google.com/group/beets-users +.. _discussion board: https://discourse.beets.io .. contents:: :local: @@ -94,14 +94,14 @@ the tracks into a single directory to force them to be tagged together. An MBID looks like one of these: -- ``http://musicbrainz.org/release/ded77dcf-7279-457e-955d-625bd3801b87`` +- ``https://musicbrainz.org/release/ded77dcf-7279-457e-955d-625bd3801b87`` - ``d569deba-8c6b-4d08-8c43-d0e5a1b8c7f3`` Beets can recognize either the hex-with-dashes UUID-style string or the full URL that contains it (as of 1.0b11). You can get these IDs by `searching on the MusicBrainz web -site `__ and going to a *release* page (when +site `__ and going to a *release* page (when tagging full albums) or a *recording* page (when tagging singletons). Then, copy the URL of the page and paste it into beets. @@ -119,7 +119,7 @@ Run a command like this:: pip install -U beets -The ``-U`` flag tells `pip `__ to upgrade +The ``-U`` flag tells `pip `__ to upgrade beets to the latest version. If you want a specific version, you can specify with using ``==`` like so:: @@ -163,10 +163,10 @@ on GitHub. `Enter a new issue `__ there to report a bug. Please follow these guidelines when reporting an issue: - Most importantly: if beets is crashing, please `include the - traceback `__. Tracebacks can be more + traceback `__. Tracebacks can be more readable if you put them in a pastebin (e.g., `Gist `__ or - `Hastebin `__), especially when communicating + `Hastebin `__), especially when communicating over IRC or email. - Turn on beets' debug output (using the -v option: for example, ``beet -v import ...``) and include that with your bug report. Look @@ -188,7 +188,7 @@ there to report a bug. Please follow these guidelines when reporting an issue: If you've never reported a bug before, Mozilla has some well-written `general guidelines for good bug -reports `__. +reports `__. .. _find-config: @@ -237,7 +237,7 @@ Why does beets… There are a number of possibilities: - First, make sure the album is in `the MusicBrainz - database `__. You + database `__. You can search on their site to make sure it's cataloged there. (If not, anyone can edit MusicBrainz---so consider adding the data yourself.) - If the album in question is a multi-disc release, see the relevant @@ -320,7 +320,7 @@ it encounters files that *look* like music files (according to their extension) but seem to be broken. Most of the time, this is because the file is corrupted. To check whether the file is intact, try opening it in another media player (e.g., -`VLC `__) to see whether it can +`VLC `__) to see whether it can read the file. You can also use specialized programs for checking file integrity---for example, type ``metaflac --list music.flac`` to check FLAC files. @@ -378,4 +378,4 @@ installed using pip, the command ``pip show -f beets`` can show you where ``beet`` was placed on your system. If you need help extending your ``$PATH``, try `this Super User answer`_. -.. _this Super User answer: http://superuser.com/a/284361/4569 +.. _this Super User answer: https://superuser.com/a/284361/4569 diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index 38cc31d0c..091875c54 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -93,7 +93,7 @@ everything by the Long Winters for listening on the go. The plugin has many more dials you can fiddle with to get your conversions how you like them. Check out :doc:`its documentation `. -.. _ffmpeg: http://www.ffmpeg.org +.. _ffmpeg: https://www.ffmpeg.org Store any data you like diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 563b7ef82..1f5cc4681 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -12,7 +12,7 @@ Installing You will need Python. Beets works on `Python 2.7`_ and Python 3.4 or later. -.. _Python 2.7: http://www.python.org/download/ +.. _Python 2.7: https://www.python.org/download/ * **macOS** v10.7 (Lion) and later include Python 2.7 out of the box. You can opt for Python 3 by installing it via `Homebrew`_: @@ -49,7 +49,7 @@ Beets works on `Python 2.7`_ and Python 3.4 or later. * On **NixOS**, there's a `package `_ you can install with ``nix-env -i beets``. .. _DNF package: https://apps.fedoraproject.org/packages/beets -.. _SlackBuild: http://slackbuilds.org/repository/14.2/multimedia/beets/ +.. _SlackBuild: https://slackbuilds.org/repository/14.2/multimedia/beets/ .. _FreeBSD: http://portsmon.freebsd.org/portoverview.py?category=audio&portname=beets .. _AUR: https://aur.archlinux.org/packages/beets-git/ .. _Debian details: https://tracker.debian.org/pkg/beets @@ -64,14 +64,14 @@ beets`` if you run into permissions problems). To install without pip, download beets from `its PyPI page`_ and run ``python setup.py install`` in the directory therein. -.. _its PyPI page: http://pypi.python.org/pypi/beets#downloads -.. _pip: http://www.pip-installer.org/ +.. _its PyPI page: https://pypi.org/project/beets#downloads +.. _pip: https://pip.pypa.io The best way to upgrade beets to a new version is by running ``pip install -U beets``. You may want to follow `@b33ts`_ on Twitter to hear about progress on new versions. -.. _@b33ts: http://twitter.com/b33ts +.. _@b33ts: https://twitter.com/b33ts Installing on macOS 10.11 and Higher ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -87,7 +87,7 @@ If this happens, you can install beets for the current user only by typing ``~/Library/Python/3.6/bin`` to your ``$PATH``. .. _System Integrity Protection: https://support.apple.com/en-us/HT204899 -.. _Homebrew: http://brew.sh +.. _Homebrew: https://brew.sh Installing on Windows ^^^^^^^^^^^^^^^^^^^^^ @@ -122,10 +122,10 @@ Because I don't use Windows myself, I may have missed something. If you have trouble or you have more detail to contribute here, please direct it to `the mailing list`_. -.. _install Python: http://python.org/download/ +.. _install Python: https://python.org/download/ .. _beets.reg: https://github.com/beetbox/beets/blob/master/extra/beets.reg -.. _install pip: http://www.pip-installer.org/en/latest/installing.html#install-pip -.. _get-pip.py: https://raw.github.com/pypa/pip/master/contrib/get-pip.py +.. _install pip: https://pip.pypa.io/en/stable/installing/ +.. _get-pip.py: https://bootstrap.pypa.io/get-pip.py Configuring @@ -179,7 +179,7 @@ There are approximately six million other configuration options you can set here, including the directory and file naming scheme. See :doc:`/reference/config` for a full reference. -.. _YAML: http://yaml.org/ +.. _YAML: https://yaml.org/ Importing Your Library ---------------------- @@ -300,6 +300,6 @@ import`` gives more specific help about the ``import`` command. Please let me know what you think of beets via `the discussion board`_ or `Twitter`_. -.. _the mailing list: http://groups.google.com/group/beets-users -.. _the discussion board: http://discourse.beets.io -.. _twitter: http://twitter.com/b33ts +.. _the mailing list: https://groups.google.com/group/beets-users +.. _the discussion board: https://discourse.beets.io +.. _twitter: https://twitter.com/b33ts diff --git a/docs/guides/tagger.rst b/docs/guides/tagger.rst index b70857ca5..467d605a4 100644 --- a/docs/guides/tagger.rst +++ b/docs/guides/tagger.rst @@ -272,7 +272,7 @@ Before you jump into acoustic fingerprinting with both feet, though, give beets a try without it. You may be surprised at how well metadata-based matching works. -.. _Chromaprint: http://acoustid.org/chromaprint +.. _Chromaprint: https://acoustid.org/chromaprint Album Art, Lyrics, Genres and Such ---------------------------------- @@ -292,7 +292,7 @@ sure the album is present in `the MusicBrainz database`_. You can search on their site to make sure it's cataloged there. If not, anyone can edit MusicBrainz---so consider adding the data yourself. -.. _the MusicBrainz database: http://musicbrainz.org/ +.. _the MusicBrainz database: https://musicbrainz.org/ If you think beets is ignoring an album that's listed in MusicBrainz, please `file a bug report`_. @@ -305,5 +305,5 @@ I Hope That Makes Sense If we haven't made the process clear, please post on `the discussion board`_ and we'll try to improve this guide. -.. _the mailing list: http://groups.google.com/group/beets-users -.. _the discussion board: http://discourse.beets.io +.. _the mailing list: https://groups.google.com/group/beets-users +.. _the discussion board: https://discourse.beets.io diff --git a/docs/index.rst b/docs/index.rst index 43ba0526a..27fa4740b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,9 +18,9 @@ or `file a bug`_ in the issue tracker. Please let us know where you think this documentation can be improved. .. _beets: http://beets.io/ -.. _the mailing list: http://groups.google.com/group/beets-users +.. _the mailing list: https://groups.google.com/group/beets-users .. _file a bug: https://github.com/beetbox/beets/issues -.. _the discussion board: http://discourse.beets.io +.. _the discussion board: https://discourse.beets.io Contents -------- diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index 30a77d4b0..3221a07b3 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -41,9 +41,9 @@ To configure the plugin, make a ``absubmit:`` section in your configuration file - **extractor**: The absolute path to the `streaming_extractor_music`_ binary. Default: search for the program in your ``$PATH`` -.. _streaming_extractor_music: http://acousticbrainz.org/download -.. _FAQ: http://acousticbrainz.org/faq -.. _pip: http://www.pip-installer.org/ -.. _requests: http://docs.python-requests.org/en/master/ +.. _streaming_extractor_music: https://acousticbrainz.org/download +.. _FAQ: https://acousticbrainz.org/faq +.. _pip: https://pip.pypa.io +.. _requests: https://docs.python-requests.org/en/master/ .. _github: https://github.com/MTG/essentia .. _AcousticBrainz: https://acousticbrainz.org diff --git a/docs/plugins/acousticbrainz.rst b/docs/plugins/acousticbrainz.rst index 5bd514c64..7c24ffe0d 100644 --- a/docs/plugins/acousticbrainz.rst +++ b/docs/plugins/acousticbrainz.rst @@ -4,7 +4,7 @@ AcousticBrainz Plugin The ``acousticbrainz`` plugin gets acoustic-analysis information from the `AcousticBrainz`_ project. -.. _AcousticBrainz: http://acousticbrainz.org/ +.. _AcousticBrainz: https://acousticbrainz.org/ Enable the ``acousticbrainz`` plugin in your configuration (see :ref:`using-plugins`) and run it by typing:: diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index da77cd4cd..709dbb0a8 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -31,6 +31,6 @@ from MusicBrainz and other sources. If you have a Beatport ID or a URL for a release or track you want to tag, you can just enter one of the two at the "enter Id" prompt in the importer. -.. _requests: http://docs.python-requests.org/en/latest/ +.. _requests: https://docs.python-requests.org/en/latest/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib -.. _Beatport: http://beatport.com +.. _Beatport: https://beetport.com diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index c1a94e972..49563a73a 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -6,7 +6,7 @@ implements the MPD protocol, so it's compatible with all the great MPD clients out there. I'm using `Theremin`_, `gmpc`_, `Sonata`_, and `Ario`_ successfully. .. _Theremin: https://theremin.sigterm.eu/ -.. _gmpc: http://gmpc.wikia.com/wiki/Gnome_Music_Player_Client +.. _gmpc: https://gmpc.wikia.com/wiki/Gnome_Music_Player_Client .. _Sonata: http://sonata.berlios.de/ .. _Ario: http://ario-player.sourceforge.net/ @@ -29,8 +29,8 @@ You will also need the various GStreamer plugin packages to make everything work. See the :doc:`/plugins/chroma` documentation for more information on installing GStreamer plugins. -.. _GStreamer WinBuilds: http://www.gstreamer-winbuild.ylatuya.es/ -.. _Homebrew: http://mxcl.github.com/homebrew/ +.. _GStreamer WinBuilds: https://www.gstreamer-winbuild.ylatuya.es/ +.. _Homebrew: https://brew.sh Usage ----- @@ -44,7 +44,7 @@ Then, you can run BPD by invoking:: Fire up your favorite MPD client to start playing music. The MPD site has `a long list of available clients`_. Here are my favorites: -.. _a long list of available clients: http://mpd.wikia.com/wiki/Clients +.. _a long list of available clients: https://mpd.wikia.com/wiki/Clients * Linux: `gmpc`_, `Sonata`_ @@ -52,9 +52,9 @@ long list of available clients`_. Here are my favorites: * Windows: I don't know. Get in touch if you have a recommendation. -* iPhone/iPod touch: `MPoD`_ +* iPhone/iPod touch: `Rigelian`_ -.. _MPoD: http://www.katoemba.net/makesnosenseatall/mpod/ +.. _Rigelian: https://www.rigelian.net/ One nice thing about MPD's (and thus BPD's) client-server architecture is that the client can just as easily on a different computer from the server as it can @@ -109,7 +109,7 @@ behaviour to their MPD equivalents. BPD aims to look enough like MPD that it can interact with the ecosystem of clients, but doesn't try to be a fully-fledged MPD replacement in terms of its playback capabilities. -.. _the MPD protocol: http://www.musicpd.org/doc/protocol/ +.. _the MPD protocol: https://www.musicpd.org/doc/protocol/ These are some of the known differences between BPD and MPD: diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index 617d8cf69..1b86073b8 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -8,8 +8,8 @@ information at all (or have completely incorrect data). This plugin uses an open-source fingerprinting technology called `Chromaprint`_ and its associated Web service, called `Acoustid`_. -.. _Chromaprint: http://acoustid.org/chromaprint -.. _acoustid: http://acoustid.org/ +.. _Chromaprint: https://acoustid.org/chromaprint +.. _acoustid: https://acoustid.org/ Turning on fingerprinting can increase the accuracy of the autotagger---especially on files with very poor metadata---but it comes at a @@ -31,7 +31,7 @@ First, install pyacoustid itself. You can do this using `pip`_, like so:: $ pip install pyacoustid -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io Then, you will need to install `Chromaprint`_, either as a dynamic library or in the form of a command-line tool (``fpcalc``). @@ -45,7 +45,7 @@ The simplest way to get up and running, especially on Windows, is to means something like ``C:\\Program Files``. On OS X or Linux, put the executable somewhere like ``/usr/local/bin``. -.. _download: http://acoustid.org/chromaprint +.. _download: https://acoustid.org/chromaprint Installing the Library '''''''''''''''''''''' @@ -56,7 +56,7 @@ site has links to packages for major Linux distributions. If you use `Homebrew`_ on Mac OS X, you can install the library with ``brew install chromaprint``. -.. _Homebrew: http://mxcl.github.com/homebrew/ +.. _Homebrew: https://brew.sh/ You will also need a mechanism for decoding audio files supported by the `audioread`_ library: @@ -78,12 +78,12 @@ You will also need a mechanism for decoding audio files supported by the * On Windows, builds are provided by `GStreamer`_ .. _audioread: https://github.com/beetbox/audioread -.. _pyacoustid: http://github.com/beetbox/pyacoustid -.. _FFmpeg: http://ffmpeg.org/ -.. _MAD: http://spacepants.org/src/pymad/ -.. _pymad: http://www.underbit.com/products/mad/ -.. _Core Audio: http://developer.apple.com/technologies/mac/audio-and-video.html -.. _Gstreamer: http://gstreamer.freedesktop.org/ +.. _pyacoustid: https://github.com/beetbox/pyacoustid +.. _FFmpeg: https://ffmpeg.org/ +.. _MAD: https://spacepants.org/src/pymad/ +.. _pymad: https://www.underbit.com/products/mad/ +.. _Core Audio: https://developer.apple.com/technologies/mac/audio-and-video.html +.. _Gstreamer: https://gstreamer.freedesktop.org/ .. _PyGObject: https://wiki.gnome.org/Projects/PyGObject To decode audio formats (MP3, FLAC, etc.) with GStreamer, you'll need the @@ -132,4 +132,4 @@ Then, run ``beet submit``. (You can also provide a query to submit a subset of your library.) The command will use stored fingerprints if they're available; otherwise it will fingerprint each file before submitting it. -.. _get an API key: http://acoustid.org/api-key +.. _get an API key: https://acoustid.org/api-key diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 92545af30..59670c269 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -14,7 +14,7 @@ To use the ``convert`` plugin, first enable it in your configuration (see :ref:`using-plugins`). By default, the plugin depends on `FFmpeg`_ to transcode the audio, so you might want to install it. -.. _FFmpeg: http://ffmpeg.org +.. _FFmpeg: https://ffmpeg.org Usage @@ -170,6 +170,6 @@ can use the :doc:`/plugins/replaygain` to do this analysis. See the LAME options and a thorough discussion of MP3 encoding. .. _documentation: http://lame.sourceforge.net/using.php -.. _HydrogenAudio wiki: http://wiki.hydrogenaud.io/index.php?title=LAME -.. _gapless: http://wiki.hydrogenaud.io/index.php?title=Gapless_playback -.. _LAME: http://lame.sourceforge.net/ +.. _HydrogenAudio wiki: https://wiki.hydrogenaud.io/index.php?title=LAME +.. _gapless: https://wiki.hydrogenaud.io/index.php?title=Gapless_playback +.. _LAME: https://lame.sourceforge.net/ diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index a02b34590..622a085b4 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -4,7 +4,7 @@ Discogs Plugin The ``discogs`` plugin extends the autotagger's search capabilities to include matches from the `Discogs`_ database. -.. _Discogs: http://discogs.com +.. _Discogs: https://discogs.com Installation ------------ diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index 2a34a59e8..cc2fe6fc8 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -69,7 +69,7 @@ Note: ``compare_threshold`` option requires `ImageMagick`_, and ``maxwidth`` requires either `ImageMagick`_ or `Pillow`_. .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ .. _PHASH: http://www.fmwconcepts.com/misc_tests/perceptual_hash_test_results_510/ Manually Embedding and Extracting Art diff --git a/docs/plugins/embyupdate.rst b/docs/plugins/embyupdate.rst index d820f5c6b..626fafa9d 100644 --- a/docs/plugins/embyupdate.rst +++ b/docs/plugins/embyupdate.rst @@ -17,8 +17,8 @@ To use the ``embyupdate`` plugin you need to install the `requests`_ library wit With that all in place, you'll see beets send the "update" command to your Emby server every time you change your beets library. -.. _Emby: http://emby.media/ -.. _requests: http://docs.python-requests.org/en/latest/ +.. _Emby: https://emby.media/ +.. _requests: https://docs.python-requests.org/en/latest/ Configuration ------------- diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 4326ccb16..d712dfc8b 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -4,7 +4,7 @@ Export Plugin The ``export`` plugin lets you get data from the items and export the content as `JSON`_. -.. _JSON: http://www.json.org +.. _JSON: https://www.json.org Enable the ``export`` plugin (see :ref:`using-plugins` for help). Then, type ``beet export`` followed by a :doc:`query ` to get the data from your library. For example, run this:: diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 8af7f686a..f23fec765 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -11,7 +11,7 @@ To use the ``fetchart`` plugin, first enable it in your configuration (see The plugin uses `requests`_ to fetch album art from the Web. -.. _requests: http://docs.python-requests.org/en/latest/ +.. _requests: https://docs.python-requests.org/en/latest/ Fetching Album Art During Import -------------------------------- @@ -81,7 +81,7 @@ or `Pillow`_. .. _beets custom search engine: https://cse.google.com.au:443/cse/publicurl?cx=001442825323518660753:hrh5ch1gjzm .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ Here's an example that makes plugin select only images that contain ``front`` or ``back`` keywords in their filenames and prioritizes the iTunes source over @@ -135,7 +135,7 @@ On some versions of Windows, the program can be shadowed by a system-provided environment variable so that ImageMagick comes first or use Pillow instead. .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ .. _album-art-sources: @@ -191,7 +191,7 @@ Optionally, you can `define a custom search engine`_. Get your search engine's token and use it for your ``google_engine`` configuration option. The default engine searches the entire web for cover art. -.. _define a custom search engine: http://www.google.com/cse/all +.. _define a custom search engine: https://www.google.com/cse/all Note that the Google custom search API is limited to 100 queries per day. After that, the fetchart plugin will fall back on other declared data sources. diff --git a/docs/plugins/ftintitle.rst b/docs/plugins/ftintitle.rst index 8a080b3e2..66c9ecd69 100644 --- a/docs/plugins/ftintitle.rst +++ b/docs/plugins/ftintitle.rst @@ -41,4 +41,4 @@ your entire collection. Use the ``-d`` flag to remove featured artists (equivalent of the ``drop`` config option). -.. _MusicBrainz style: http://musicbrainz.org/doc/Style +.. _MusicBrainz style: https://musicbrainz.org/doc/Style diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index f95a6285d..1c8a8d417 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -107,7 +107,7 @@ Autotagger Extensions * :doc:`fromfilename`: Guess metadata for untagged tracks from their filenames. -.. _Discogs: http://www.discogs.com/ +.. _Discogs: https://www.discogs.com/ Metadata -------- @@ -136,7 +136,7 @@ Metadata * :doc:`zero`: Nullify fields by pattern or unconditionally. .. _KeyFinder: http://www.ibrahimshaath.co.uk/keyfinder/ -.. _streaming_extractor_music: http://acousticbrainz.org/download +.. _streaming_extractor_music: https://acousticbrainz.org/download Path Formats ------------ @@ -169,10 +169,10 @@ Interoperability * :doc:`thumbnails`: Get thumbnails with the cover art on your album folders. -.. _Emby: http://emby.media -.. _Plex: http://plex.tv -.. _Kodi: http://kodi.tv -.. _Sonos: http://sonos.com +.. _Emby: https://emby.media +.. _Plex: https://plex.tv +.. _Kodi: https://kodi.tv +.. _Sonos: https://sonos.com Miscellaneous ------------- @@ -194,14 +194,14 @@ Miscellaneous * :doc:`mbcollection`: Maintain your MusicBrainz collection list. * :doc:`mbsubmit`: Print an album's tracks in a MusicBrainz-friendly format. * :doc:`missing`: List missing tracks. -* `mstream`_: A music streaming server + webapp that can be used alongside beets. +* `mstream`_: A music streaming server + webapp that can be used alongside beets. * :doc:`random`: Randomly choose albums and tracks from your library. * :doc:`spotify`: Create Spotify playlists from the Beets library. * :doc:`types`: Declare types for flexible attributes. * :doc:`web`: An experimental Web-based GUI for beets. -.. _MPD: http://www.musicpd.org/ -.. _MPD clients: http://mpd.wikia.com/wiki/Clients +.. _MPD: https://www.musicpd.org/ +.. _MPD clients: https://mpd.wikia.com/wiki/Clients .. _mstream: https://github.com/IrosTheBeggar/mStream .. _other-plugins: diff --git a/docs/plugins/info.rst b/docs/plugins/info.rst index 238a957ff..d628fb4ad 100644 --- a/docs/plugins/info.rst +++ b/docs/plugins/info.rst @@ -42,4 +42,4 @@ Additional command-line options include: * ``--keys-only`` or ``-k``: Show the name of the tags without the values. .. _id3v2: http://id3v2.sourceforge.net -.. _mp3info: http://www.ibiblio.org/mp3info/ +.. _mp3info: https://www.ibiblio.org/mp3info/ diff --git a/docs/plugins/ipfs.rst b/docs/plugins/ipfs.rst index 141143ae7..5bf8ca906 100644 --- a/docs/plugins/ipfs.rst +++ b/docs/plugins/ipfs.rst @@ -4,7 +4,7 @@ IPFS Plugin The ``ipfs`` plugin makes it easy to share your library and music with friends. The plugin uses `ipfs`_ for storing the library and file content. -.. _ipfs: http://ipfs.io/ +.. _ipfs: https://ipfs.io/ Installation ------------ diff --git a/docs/plugins/keyfinder.rst b/docs/plugins/keyfinder.rst index 856939ecc..878830f29 100644 --- a/docs/plugins/keyfinder.rst +++ b/docs/plugins/keyfinder.rst @@ -29,4 +29,4 @@ configuration file. The available options are: `initial_key` value. Default: ``no``. -.. _KeyFinder: http://www.ibrahimshaath.co.uk/keyfinder/ +.. _KeyFinder: https://www.ibrahimshaath.co.uk/keyfinder/ diff --git a/docs/plugins/kodiupdate.rst b/docs/plugins/kodiupdate.rst index a1ec04775..e60f503f2 100644 --- a/docs/plugins/kodiupdate.rst +++ b/docs/plugins/kodiupdate.rst @@ -26,8 +26,8 @@ In Kodi's interface, navigate to System/Settings/Network/Services and choose "Al With that all in place, you'll see beets send the "update" command to your Kodi host every time you change your beets library. -.. _Kodi: http://kodi.tv/ -.. _requests: http://docs.python-requests.org/en/latest/ +.. _Kodi: https://kodi.tv/ +.. _requests: https://docs.python-requests.org/en/latest/ Configuration ------------- diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index c3d5f97ec..5fcdd2254 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -7,8 +7,8 @@ importing and autotagging music, beets does not assign a genre. The to your albums and items. .. _does not contain genre information: - http://musicbrainz.org/doc/General_FAQ#Why_does_MusicBrainz_not_support_genre_information.3F -.. _Last.fm: http://last.fm/ + https://musicbrainz.org/doc/General_FAQ#Why_does_MusicBrainz_not_support_genre_information.3F +.. _Last.fm: https://last.fm/ Installation ------------ @@ -34,7 +34,7 @@ The genre list file should contain one genre per line. Blank lines are ignored. For the curious, the default genre list is generated by a `script that scrapes Wikipedia`_. -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io .. _pylast: https://github.com/pylast/pylast .. _script that scrapes Wikipedia: https://gist.github.com/1241307 .. _internal whitelist: https://raw.githubusercontent.com/beetbox/beets/master/beetsplug/lastgenre/genres.txt @@ -72,7 +72,7 @@ nothing would ever be matched to a more generic node since all the specific subgenres are in the whitelist to begin with. -.. _YAML: http://www.yaml.org/ +.. _YAML: https://www.yaml.org/ .. _tree of nested genre names: https://raw.githubusercontent.com/beetbox/beets/master/beetsplug/lastgenre/genres-tree.yaml diff --git a/docs/plugins/lastimport.rst b/docs/plugins/lastimport.rst index 8006d6bbb..1c12b8616 100644 --- a/docs/plugins/lastimport.rst +++ b/docs/plugins/lastimport.rst @@ -6,7 +6,7 @@ library into beets' database. You can later create :doc:`smart playlists ` by querying ``play_count`` and do other fun stuff with this field. -.. _Last.fm: http://last.fm +.. _Last.fm: https://last.fm Installation ------------ @@ -23,7 +23,7 @@ Next, add your Last.fm username to your beets configuration file:: lastfm: user: beetsfanatic -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io .. _pylast: https://github.com/pylast/pylast Importing Play Counts diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 799bd0325..fac07ad87 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -5,9 +5,9 @@ The ``lyrics`` plugin fetches and stores song lyrics from databases on the Web. Namely, the current version of the plugin uses `Lyric Wiki`_, `Musixmatch`_, `Genius.com`_, and, optionally, the Google custom search API. -.. _Lyric Wiki: http://lyrics.wikia.com/ +.. _Lyric Wiki: https://lyrics.wikia.com/ .. _Musixmatch: https://www.musixmatch.com/ -.. _Genius.com: http://genius.com/ +.. _Genius.com: https://genius.com/ Fetch Lyrics During Import @@ -26,7 +26,7 @@ already have them. The lyrics will be stored in the beets database. If the ``import.write`` config option is on, then the lyrics will also be written to the files' tags. -.. _requests: http://docs.python-requests.org/en/latest/ +.. _requests: https://docs.python-requests.org/en/latest/ Configuration @@ -105,11 +105,11 @@ A minimal ``conf.py`` and ``index.rst`` files are created the first time the command is run. They are not overwritten on subsequent runs, so you can safely modify these files to customize the output. -.. _Sphinx: http://www.sphinx-doc.org/ +.. _Sphinx: https://www.sphinx-doc.org/ .. _reStructuredText: http://docutils.sourceforge.net/rst.html Sphinx supports various `builders -`_, but here are a +`_, but here are a few suggestions. * Build an HTML version:: @@ -148,13 +148,13 @@ Optionally, you can `define a custom search engine`_. Get your search engine's token and use it for your ``google_engine_ID`` configuration option. By default, beets use a list of sources known to be scrapeable. -.. _define a custom search engine: http://www.google.com/cse/all +.. _define a custom search engine: https://www.google.com/cse/all Note that the Google custom search API is limited to 100 queries per day. After that, the lyrics plugin will fall back on other declared data sources. -.. _pip: http://www.pip-installer.org/ -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ +.. _pip: https://pip.pypa.io +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ Activate Genius Lyrics ---------------------- diff --git a/docs/plugins/mbcollection.rst b/docs/plugins/mbcollection.rst index 803d34904..113855bce 100644 --- a/docs/plugins/mbcollection.rst +++ b/docs/plugins/mbcollection.rst @@ -4,7 +4,7 @@ MusicBrainz Collection Plugin The ``mbcollection`` plugin lets you submit your catalog to MusicBrainz to maintain your `music collection`_ list there. -.. _music collection: http://musicbrainz.org/doc/Collections +.. _music collection: https://musicbrainz.org/doc/Collections To begin, just enable the ``mbcollection`` plugin in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/mbsubmit.rst b/docs/plugins/mbsubmit.rst index 5c13375ba..70e14662d 100644 --- a/docs/plugins/mbsubmit.rst +++ b/docs/plugins/mbsubmit.rst @@ -5,7 +5,7 @@ The ``mbsubmit`` plugin provides an extra prompt choice during an import session that prints the tracks of the current album in a format that is parseable by MusicBrainz's `track parser`_. -.. _track parser: http://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings +.. _track parser: https://wiki.musicbrainz.org/History:How_To_Parse_Track_Listings Usage ----- @@ -15,7 +15,7 @@ and select the ``Print tracks`` choice which is by default displayed when no strong recommendations are found for the album:: No matching release found for 3 tracks. - For help, see: http://beets.readthedocs.org/en/latest/faq.html#nomatch + For help, see: https://beets.readthedocs.org/en/latest/faq.html#nomatch [U]se as-is, as Tracks, Group albums, Skip, Enter search, enter Id, aBort, Print tracks? p 01. An Obscure Track - An Obscure Artist (3:37) @@ -23,7 +23,7 @@ strong recommendations are found for the album:: 03. The Third Track - Another Obscure Artist (3:02) No matching release found for 3 tracks. - For help, see: http://beets.readthedocs.org/en/latest/faq.html#nomatch + For help, see: https://beets.readthedocs.org/en/latest/faq.html#nomatch [U]se as-is, as Tracks, Group albums, Skip, Enter search, enter Id, aBort, Print tracks? diff --git a/docs/plugins/metasync.rst b/docs/plugins/metasync.rst index 6703d3c19..691550595 100644 --- a/docs/plugins/metasync.rst +++ b/docs/plugins/metasync.rst @@ -22,7 +22,7 @@ Enable the ``metasync`` plugin in your configuration (see To synchronize with Amarok, you'll need the `dbus-python`_ library. There are packages for most major Linux distributions. -.. _dbus-python: http://dbus.freedesktop.org/releases/dbus-python/ +.. _dbus-python: https://dbus.freedesktop.org/releases/dbus-python/ Configuration diff --git a/docs/plugins/mpdstats.rst b/docs/plugins/mpdstats.rst index 2e5e78c36..b769e7468 100644 --- a/docs/plugins/mpdstats.rst +++ b/docs/plugins/mpdstats.rst @@ -9,7 +9,7 @@ habits from `MPD`_. It collects the following information about tracks: * last_played: UNIX timestamp when you last played this track. * rating: A rating based on *play_count* and *skip_count*. -.. _MPD: http://www.musicpd.org/ +.. _MPD: https://www.musicpd.org/ Installing Dependencies ----------------------- @@ -23,7 +23,7 @@ Install the library from `pip`_, like so:: Add the ``mpdstats`` plugin to your configuration (see :ref:`using-plugins`). -.. _pip: http://www.pip-installer.org/ +.. _pip: https://pip.pypa.io Usage ----- diff --git a/docs/plugins/mpdupdate.rst b/docs/plugins/mpdupdate.rst index 7ac647536..01a6a9fe7 100644 --- a/docs/plugins/mpdupdate.rst +++ b/docs/plugins/mpdupdate.rst @@ -4,7 +4,7 @@ MPDUpdate Plugin ``mpdupdate`` is a very simple plugin for beets that lets you automatically update `MPD`_'s index whenever you change your beets library. -.. _MPD: http://www.musicpd.org/ +.. _MPD: https://www.musicpd.org/ To use ``mpdupdate`` plugin, enable it in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index 4ac047660..f9312280a 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -21,11 +21,11 @@ To use the ``plexupdate`` plugin you need to install the `requests`_ library wit pip install requests -With that all in place, you'll see beets send the "update" command to your Plex +With that all in place, you'll see beets send the "update" command to your Plex server every time you change your beets library. -.. _Plex: http://plex.tv/ -.. _requests: http://docs.python-requests.org/en/latest/ +.. _Plex: https://plex.tv/ +.. _requests: https://docs.python-requests.org/en/latest/ .. _documentation about tokens: https://support.plex.tv/hc/en-us/articles/204059436-Finding-your-account-token-X-Plex-Token Configuration diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 825f279e2..728f1846e 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -4,7 +4,7 @@ ReplayGain Plugin This plugin adds support for `ReplayGain`_, a technique for normalizing audio playback levels. -.. _ReplayGain: http://wiki.hydrogenaudio.org/index.php?title=ReplayGain +.. _ReplayGain: https://wiki.hydrogenaudio.org/index.php?title=ReplayGain Installation @@ -27,7 +27,7 @@ install GStreamer and plugins for compatibility with your audio files. You will need at least GStreamer 1.0 and `PyGObject 3.x`_ (a.k.a. ``python-gi``). .. _PyGObject 3.x: https://pygobject.readthedocs.io/en/latest/ -.. _GStreamer: http://gstreamer.freedesktop.org/ +.. _GStreamer: https://gstreamer.freedesktop.org/ Then, enable the ``replaygain`` plugin (see :ref:`using-plugins`) and specify the GStreamer backend by adding this to your configuration file:: @@ -47,8 +47,8 @@ command-line tool or the `aacgain`_ fork thereof. Here are some hints: * On Windows, download and install the original `mp3gain`_. .. _mp3gain: http://mp3gain.sourceforge.net/download.php -.. _aacgain: http://aacgain.altosdesign.com -.. _Homebrew: http://mxcl.github.com/homebrew/ +.. _aacgain: https://aacgain.altosdesign.com +.. _Homebrew: https://brew.sh Then, enable the plugin (see :ref:`using-plugins`) and specify the "command" backend in your configuration file:: diff --git a/docs/plugins/sonosupdate.rst b/docs/plugins/sonosupdate.rst index 97a13bd07..cae69d554 100644 --- a/docs/plugins/sonosupdate.rst +++ b/docs/plugins/sonosupdate.rst @@ -14,5 +14,5 @@ To use the ``sonosupdate`` plugin you need to install the `soco`_ library with:: With that all in place, you'll see beets send the "update" command to your Sonos controller every time you change your beets library. -.. _Sonos: http://sonos.com/ +.. _Sonos: https://sonos.com/ .. _soco: http://python-soco.com diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index 3f4c6c43d..5d6ae8f47 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -31,8 +31,8 @@ Here's an example:: $ beet spotify "In The Lonely Hour" Processing 14 tracks... - http://open.spotify.com/track/19w0OHr8SiZzRhjpnjctJ4 - http://open.spotify.com/track/3PRLM4FzhplXfySa4B7bxS + https://open.spotify.com/track/19w0OHr8SiZzRhjpnjctJ4 + https://open.spotify.com/track/3PRLM4FzhplXfySa4B7bxS [...] Command-line options include: diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index daf4a0cfb..2d9331b7c 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -4,7 +4,7 @@ SubsonicUpdate Plugin ``subsonicupdate`` is a very simple plugin for beets that lets you automatically update `Subsonic`_'s index whenever you change your beets library. -.. _Subsonic: http://www.subsonic.org +.. _Subsonic: https://www.subsonic.org To use ``subsonicupdate`` plugin, enable it in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/thumbnails.rst b/docs/plugins/thumbnails.rst index c2a28d091..0f46e04e8 100644 --- a/docs/plugins/thumbnails.rst +++ b/docs/plugins/thumbnails.rst @@ -13,7 +13,7 @@ as the :doc:`/plugins/fetchart`. You'll need 2 additional python packages: `ImageMagick`_ or `Pillow`_. .. _Pillow: https://github.com/python-pillow/Pillow -.. _ImageMagick: http://www.imagemagick.org/ +.. _ImageMagick: https://www.imagemagick.org/ Configuration ------------- diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 35287acc8..d3ae668ce 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -19,13 +19,13 @@ The Web interface depends on `Flask`_. To get it, just run ``pip install flask``. Then enable the ``web`` plugin in your configuration (see :ref:`using-plugins`). -.. _Flask: http://flask.pocoo.org/ +.. _Flask: https://flask.pocoo.org/ If you need CORS (it's disabled by default---see :ref:`web-cors`, below), then you also need `flask-cors`_. Just type ``pip install flask-cors``. .. _flask-cors: https://github.com/CoryDolphin/flask-cors -.. _CORS: http://en.wikipedia.org/wiki/Cross-origin_resource_sharing +.. _CORS: https://en.wikipedia.org/wiki/Cross-origin_resource_sharing Run the Server @@ -78,8 +78,8 @@ The Web backend is built using a simple REST+JSON API with the excellent `Flask`_ library. The frontend is a single-page application written with `Backbone.js`_. This allows future non-Web clients to use the same backend API. -.. _Flask: http://flask.pocoo.org/ -.. _Backbone.js: http://backbonejs.org +.. _Flask: https://flask.pocoo.org/ +.. _Backbone.js: https://backbonejs.org Eventually, to make the Web player really viable, we should use a Flash fallback for unsupported formats/browsers. There are a number of options for this: @@ -88,8 +88,8 @@ for unsupported formats/browsers. There are a number of options for this: * `html5media`_ * `MediaElement.js`_ -.. _audio.js: http://kolber.github.com/audiojs/ -.. _html5media: http://html5media.info/ +.. _audio.js: https://kolber.github.io/audiojs/ +.. _html5media: https://html5media.info/ .. _MediaElement.js: http://mediaelementjs.com/ .. _web-cors: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 7b9e9eb72..2fc7c7b31 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -217,7 +217,7 @@ beatles`` prints out the number of tracks on each Beatles album. In Unix shells, remember to enclose the template argument in single quotes to avoid environment variable expansion. -.. _xargs: http://en.wikipedia.org/wiki/Xargs +.. _xargs: https://en.wikipedia.org/wiki/Xargs .. _remove-cmd: @@ -498,6 +498,6 @@ defines some bash-specific functions to make this work without errors:: See Also -------- - ``http://beets.readthedocs.org/`` + ``https://beets.readthedocs.org/`` :manpage:`beetsconfig(5)` diff --git a/docs/reference/config.rst b/docs/reference/config.rst index a96e3dfb3..687f6c3f9 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -31,7 +31,7 @@ In YAML, you will need to use spaces (not tabs!) to indent some lines. If you have questions about more sophisticated syntax, take a look at the `YAML`_ documentation. -.. _YAML: http://yaml.org/ +.. _YAML: https://yaml.org/ The rest of this page enumerates the dizzying litany of configuration options available in beets. You might also want to see an @@ -167,7 +167,7 @@ equivalent to wrapping all your path templates in the ``%asciify{}`` Default: ``no``. -.. _unidecode module: http://pypi.python.org/pypi/Unidecode +.. _unidecode module: https://pypi.org/project/Unidecode .. _art-filename: @@ -314,7 +314,7 @@ standard output. It's also used to read messages from the standard input. By default, this is determined automatically from the locale environment variables. -.. _known to python: http://docs.python.org/2/library/codecs.html#standard-encodings +.. _known to python: https://docs.python.org/2/library/codecs.html#standard-encodings .. _clutter: @@ -688,7 +688,7 @@ to one request per second. .. _your own MusicBrainz database: https://musicbrainz.org/doc/MusicBrainz_Server/Setup .. _main server: https://musicbrainz.org/ -.. _limited: http://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting +.. _limited: https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting .. _Building search indexes: https://musicbrainz.org/doc/MusicBrainz_Server/Setup#Building_search_indexes .. _searchlimit: @@ -981,6 +981,6 @@ Here's an example file:: See Also -------- - ``http://beets.readthedocs.org/`` + ``https://beets.readthedocs.org/`` :manpage:`beet(1)` diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index 79998a9e1..9213cae4b 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -23,7 +23,7 @@ a dollars sign. As with `Python template strings`_, ``${title}`` is equivalent to ``$title``; you can use this if you need to separate a field name from the text that follows it. -.. _Python template strings: http://docs.python.org/library/string.html#template-strings +.. _Python template strings: https://docs.python.org/library/string.html#template-strings A Note About Artists @@ -38,7 +38,7 @@ tracks in a "Talking Heads" directory and one in a "Tom Tom Club" directory. You probably don't want that! So use ``$albumartist``. .. _Stop Making Sense: - http://musicbrainz.org/release/798dcaab-0f1a-4f02-a9cb-61d5b0ddfd36.html + https://musicbrainz.org/release/798dcaab-0f1a-4f02-a9cb-61d5b0ddfd36.html As a convenience, however, beets allows ``$albumartist`` to fall back to the value for ``$artist`` and vice-versa if one tag is present but the other is not. @@ -89,8 +89,8 @@ These functions are built in to beets: without ``$``. Note that this doesn't work with built-in :ref:`itemfields`, as they are always defined. -.. _unidecode module: http://pypi.python.org/pypi/Unidecode -.. _strftime: http://docs.python.org/2/library/time.html#time.strftime +.. _unidecode module: https://pypi.org/project/Unidecode +.. _strftime: https://docs.python.org/3/library/time.html#time.strftime Plugins can extend beets with more template functions (see :ref:`templ_plugins`). @@ -228,8 +228,8 @@ Ordinary metadata: * disctitle * encoder -.. _artist credit: http://wiki.musicbrainz.org/Artist_Credit -.. _list of type names: http://musicbrainz.org/doc/Release_Group/Type +.. _artist credit: https://wiki.musicbrainz.org/Artist_Credit +.. _list of type names: https://musicbrainz.org/doc/Release_Group/Type Audio information: diff --git a/docs/reference/query.rst b/docs/reference/query.rst index d103d9aec..5c16f610b 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -122,7 +122,7 @@ expressions, such as ``()[]|``. To type those characters, you'll need to escape them (e.g., with backslashes or quotation marks, depending on your shell). -.. _Python's built-in implementation: http://docs.python.org/library/re.html +.. _Python's built-in implementation: https://docs.python.org/library/re.html .. _numericquery: diff --git a/extra/_beet b/extra/_beet index 56c86d036..a8c9083be 100644 --- a/extra/_beet +++ b/extra/_beet @@ -1,6 +1,6 @@ #compdef beet -# zsh completion for beets music library manager and MusicBrainz tagger: http://beets.radbox.org/ +# zsh completion for beets music library manager and MusicBrainz tagger: http://beets.io/ # Default values for BEETS_LIBRARY & BEETS_CONFIG needed for the cache checking function. # They will be updated under the assumption that the config file is in the same directory as the library. diff --git a/setup.py b/setup.py index 7c209f019..30db5272d 100755 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ setup( # badfiles: mp3val and flac # bpd: python-gi and GStreamer 1.0+ # embedart: ImageMagick - # absubmit: extractor binary from http://acousticbrainz.org/download + # absubmit: extractor binary from https://acousticbrainz.org/download # keyfinder: KeyFinder # replaygain: python-gi and GStreamer 1.0+ or mp3gain/aacgain # or Python Audio Tools diff --git a/test/test_art.py b/test/test_art.py index 556222f48..f4b3a6e62 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -159,9 +159,9 @@ class FSArtTest(UseThePlugin): class CombinedTest(FetchImageHelper, UseThePlugin): ASIN = 'xxxx' MBID = 'releaseid' - AMAZON_URL = 'http://images.amazon.com/images/P/{0}.01.LZZZZZZZ.jpg' \ + AMAZON_URL = 'https://images.amazon.com/images/P/{0}.01.LZZZZZZZ.jpg' \ .format(ASIN) - AAO_URL = 'http://www.albumart.org/index_detail.php?asin={0}' \ + AAO_URL = 'https://www.albumart.org/index_detail.php?asin={0}' \ .format(ASIN) CAA_URL = 'coverartarchive.org/release/{0}/front' \ .format(MBID) @@ -240,7 +240,7 @@ class CombinedTest(FetchImageHelper, UseThePlugin): class AAOTest(UseThePlugin): ASIN = 'xxxx' - AAO_URL = 'http://www.albumart.org/index_detail.php?asin={0}'.format(ASIN) + AAO_URL = 'https://www.albumart.org/index_detail.php?asin={0}'.format(ASIN) def setUp(self): super(AAOTest, self).setUp() From da8f4a294e5f72a0a1dd25a1b601e181b3a34af9 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 4 Jun 2019 12:01:40 +1000 Subject: [PATCH 027/613] Add repology badge This is just to advertise that beets is available in distros. The badge links to a list of distro packages for beets and the current versions they have available, which is useful for users and contributors. --- README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6b4ebb4fa..8f8705abd 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,9 @@ .. image:: https://travis-ci.org/beetbox/beets.svg?branch=master :target: https://travis-ci.org/beetbox/beets +.. image:: https://repology.org/badge/tiny-repos/beets.svg + :target: https://repology.org/project/beets/versions + beets ===== @@ -78,10 +81,12 @@ shockingly simple if you know a little Python. Install ------- -You can install beets by typing ``pip install beets``. Then check out the -`Getting Started`_ guide. +You can install beets by typing ``pip install beets``. +Beets has also been packaged in the `software repositories`_ of several distributions. +Check out the `Getting Started`_ guide for more information. .. _Getting Started: https://beets.readthedocs.org/page/guides/main.html +.. _software repositories: https://repology.org/project/beets/versions Contribute ---------- From 0e65800fbc1485e84ea74091b9a4f03f2b8040ba Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 4 Jun 2019 13:18:36 +1000 Subject: [PATCH 028/613] Expand library API docs --- docs/dev/api.rst | 221 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 177 insertions(+), 44 deletions(-) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index d9e68481d..09b9b2f85 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -1,12 +1,11 @@ -API Documentation -================= +Library Database API +==================== .. currentmodule:: beets.library -This page describes the internal API of beets' core. It's a work in -progress---since beets is an application first and a library second, its API -has been mainly undocumented until recently. Please file bugs if you run -across incomplete or incorrect docs here. +This page describes the internal API of beets' core database features. It +doesn't exhaustively document the API, but is aimed at giving an overview of +the architecture to orient anyone who wants to dive into the code. The :class:`Library` object is the central repository for data in beets. It represents a database containing songs, which are :class:`Item` instances, and @@ -15,8 +14,24 @@ groups of items, which are :class:`Album` instances. The Library Class ----------------- +The :class:`Library` is typically instantiated as a singleton. A single +invocation of beets usually has only one :class:`Library`. It's powered by +:class:`dbcore.Database` under the hood, which handles the `SQLite`_ +abstraction, something like a very minimal `ORM`_. The library is also +responsible for handling queries to retrieve stored objects. + .. autoclass:: Library(path, directory[, path_formats[, replacements]]) + .. automethod:: __init__ + + You can add new items or albums to the library: + + .. automethod:: add + + .. automethod:: add_album + + And there are methods for querying the database: + .. automethod:: items .. automethod:: albums @@ -25,60 +40,178 @@ The Library Class .. automethod:: get_album - .. automethod:: add - - .. automethod:: add_album + Any modifications must go through a :class:`Transaction` which you get can + using this method: .. automethod:: transaction +.. _SQLite: http://sqlite.org/ +.. _ORM: http://en.wikipedia.org/wiki/Object-relational_mapping + + +Model Classes +------------- + +The two model entities in beets libraries, :class:`Item` and :class:`Album`, +share a base class, :class:`LibModel`, that provides common functionality and +ORM-like abstraction. + +Model base +'''''''''' + +Models use dirty-flags to track when the object's metadata goes out of +sync with the database. The dirty dictionary maps field names to booleans +indicating whether the field has been written since the object was last +synchronized (via load or store) with the database. + +.. autoclass:: LibModel + + .. automethod:: all_keys + + .. automethod:: __init__ + + .. autoattribute:: _types + + .. autoattribute:: _fields + + There are CRUD-like methods for interacting with the database: + + .. automethod:: store + + .. automethod:: load + + .. automethod:: remove + + .. automethod:: add + + The fields model classes can be accessed using attributes (dots, as in + ``item.artist``) or items (brackets, as in ``item['artist']``). + The base class :class:`dbcore.Model` has a ``dict``-like interface, so + normal the normal mapping API is supported: + + .. automethod:: keys + + .. automethod:: update + + .. automethod:: items + + .. automethod:: get + +Item +'''' + +Each :class:`Item` object represents a song or track. (We use the more generic +term item because, one day, beets might support non-music media.) An item can +either be purely abstract, in which case it's just a bag of metadata fields, +or it can have an associated file (indicated by ``item.path``). + +In terms of the underlying SQLite database, items are backed by a single table +called items with one column per metadata fields. The metadata fields currently +in use are listed in ``library.py`` in ``Item._fields``. + +To read and write a file's tags, we use the `MediaFile`_ library. +To make changes to either the database or the tags on a file, you +update an item's fields (e.g., ``item.title = "Let It Be"``) and then call +``item.write()``. + +.. _MediaFile: http://mediafile.readthedocs.io/ + +.. autoclass:: Item + + .. automethod:: __init__ + + .. automethod:: from_path + + .. automethod:: get_album + + .. automethod:: destination + + The methods ``read()`` and ``write()`` are complementary: one reads a + file's tags and updates the item's metadata fields accordingly while the + other takes the item's fields and writes them to the file's tags. + + .. automethod:: read + + .. automethod:: write + + .. automethod:: try_write + + .. automethod:: try_sync + + The :class:`Item` class supplements the normal model interface so that they + interacting with the filesystem as well: + + .. automethod:: move + + .. automethod:: remove + + Items also track their modification times (mtimes) to help detect when they + become out of sync with on-disk metadata. + + .. automethod:: current_mtime + +Album +''''' + +An :class:`Album` is a collection of Items in the database. Every item in the +database has either zero or one associated albums (accessible via +``item.album_id``). An item that has no associated album is called a +singleton. + +An :class:`Album` object keeps track of album-level metadata, which is (mostly) +a subset of the track-level metadata. The album-level metadata fields are +listed in ``Album._fields``. +For those fields that are both item-level and album-level (e.g., ``year`` or +``albumartist``), every item in an album should share the same value. Albums +use an SQLite table called ``albums``, in which each column is an album +metadata field. + +.. autoclass:: Album + + .. automethod:: __init__ + + .. automethod:: item_dir + + To get or change an album's metadata, use its fields (e.g., + ``print(album.year)`` or ``album.year = 2012``). Changing fields in this + way updates the album itself and also changes the same field in all + associated items: + + .. autoattribute:: item_keys + + .. automethod:: store + + .. automethod:: try_sync + + .. automethod:: move + + .. automethod:: remove + + Albums also manage album art, image files that are associated with each + album: + + .. automethod:: set_art + + .. automethod:: move_art + + .. automethod:: art_destination + Transactions '''''''''''' The :class:`Library` class provides the basic methods necessary to access and manipulate its contents. To perform more complicated operations atomically, or to interact directly with the underlying SQLite database, you must use a -*transaction*. For example:: +*transaction* (see this `blog post`_ for motivation). For example:: lib = Library() with lib.transaction() as tx: items = lib.items(query) lib.add_album(list(items)) +.. _blog post: http://beets.io/blog/sqlite-nightmare.html + .. currentmodule:: beets.dbcore.db .. autoclass:: Transaction :members: - -Model Classes -------------- - -The two model entities in beets libraries, :class:`Item` and :class:`Album`, -share a base class, :class:`Model`, that provides common functionality and -ORM-like abstraction. - -The fields model classes can be accessed using attributes (dots, as in -``item.artist``) or items (brackets, as in ``item['artist']``). The -:class:`Model` base class provides some methods that resemble `dict` -objects. - -Model base -'''''''''' - -.. currentmodule:: beets.dbcore - -.. autoclass:: Model - :members: - -Item -'''' - -.. currentmodule:: beets.library - -.. autoclass:: Item - :members: - -Album -''''' - -.. autoclass:: Album - :members: From 984aa223c69b8c0831231919647d3bba96cb4839 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:03:36 +1000 Subject: [PATCH 029/613] docs: highlight model field API --- docs/dev/api.rst | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index 09b9b2f85..62dfd392c 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -45,16 +45,22 @@ responsible for handling queries to retrieve stored objects. .. automethod:: transaction -.. _SQLite: http://sqlite.org/ -.. _ORM: http://en.wikipedia.org/wiki/Object-relational_mapping +.. _SQLite: https://sqlite.org/ +.. _ORM: https://en.wikipedia.org/wiki/Object-relational_mapping Model Classes ------------- The two model entities in beets libraries, :class:`Item` and :class:`Album`, -share a base class, :class:`LibModel`, that provides common functionality and -ORM-like abstraction. +share a base class, :class:`LibModel`, that provides common functionality. That +class itself specialises :class:`dbcore.Model` which provides an ORM-like +abstraction. + +To get or change the metadata of a model (an item or album), either access its +attributes (e.g., ``print(album.year)`` or ``album.year = 2012``) or use the +``dict``-like interface (e.g. ``item['artist']``). + Model base '''''''''' @@ -84,8 +90,6 @@ synchronized (via load or store) with the database. .. automethod:: add - The fields model classes can be accessed using attributes (dots, as in - ``item.artist``) or items (brackets, as in ``item['artist']``). The base class :class:`dbcore.Model` has a ``dict``-like interface, so normal the normal mapping API is supported: @@ -114,7 +118,7 @@ To make changes to either the database or the tags on a file, you update an item's fields (e.g., ``item.title = "Let It Be"``) and then call ``item.write()``. -.. _MediaFile: http://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/ .. autoclass:: Item @@ -157,6 +161,8 @@ An :class:`Album` is a collection of Items in the database. Every item in the database has either zero or one associated albums (accessible via ``item.album_id``). An item that has no associated album is called a singleton. +Changing fields on an album (e.g. ``album.year = 2012``) updates the album +itself and also changes the same field in all associated items. An :class:`Album` object keeps track of album-level metadata, which is (mostly) a subset of the track-level metadata. The album-level metadata fields are @@ -172,10 +178,8 @@ metadata field. .. automethod:: item_dir - To get or change an album's metadata, use its fields (e.g., - ``print(album.year)`` or ``album.year = 2012``). Changing fields in this - way updates the album itself and also changes the same field in all - associated items: + Albums extend the normal model interface to also forward changes to their + items: .. autoattribute:: item_keys From e27c6e480b4920ce4a7aa8c966e94fa3659a57e0 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:10:10 +1000 Subject: [PATCH 030/613] docs: add query API reference --- docs/dev/api.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index 62dfd392c..6ccdfd57e 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -219,3 +219,32 @@ to interact directly with the underlying SQLite database, you must use a .. autoclass:: Transaction :members: + + +Queries +------- + +To access albums and items in a library, we use :doc:`/reference/query`. +In beets, the :class:`Query` abstract base class represents a criterion that +matches items or albums in the database. +Every subclass of :class:`Query` must implement two methods, which implement +two different ways of identifying matching items/albums. + +The ``clause()`` method should return an SQLite ``WHERE`` clause that matches +appropriate albums/items. This allows for efficient batch queries. +Correspondingly, the ``match(item)`` method should take an :class:`Item` object +and return a boolean, indicating whether or not a specific item matches the +criterion. This alternate implementation allows clients to determine whether +items that have already been fetched from the database match the query. + +There are many different types of queries. Just as an example, +:class:`FieldQuery` determines whether a certain field matches a certain value +(an equality query). +:class:`AndQuery` (like its abstract superclass, :class:`CollectionQuery`) +takes a set of other query objects and bundles them together, matching only +albums/items that match all constituent queries. + +Beets has a human-writable plain-text query syntax that can be parsed into +:class:`Query` objects. Calling ``AndQuery.from_strings`` parses a list of +query parts into a query object that can then be used with :class:`Library` +objects. From 918024a4658a82828b1e83a7e6a093d6c0f31fc9 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:16:12 +1000 Subject: [PATCH 031/613] docs: document mtime management --- docs/dev/api.rst | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/dev/api.rst b/docs/dev/api.rst index 6ccdfd57e..ec81c169b 100644 --- a/docs/dev/api.rst +++ b/docs/dev/api.rst @@ -120,6 +120,38 @@ update an item's fields (e.g., ``item.title = "Let It Be"``) and then call .. _MediaFile: https://mediafile.readthedocs.io/ +Items also track their modification times (mtimes) to help detect when they +become out of sync with on-disk metadata, mainly to speed up the +:ref:`update-cmd` (which needs to check whether the database is in sync with +the filesystem). This feature turns out to be sort of complicated. + +For any :class:`Item`, there are two mtimes: the on-disk mtime (maintained by +the OS) and the database mtime (maintained by beets). Correspondingly, there is +on-disk metadata (ID3 tags, for example) and DB metadata. The goal with the +mtime is to ensure that the on-disk and DB mtimes match when the on-disk and DB +metadata are in sync; this lets beets do a quick mtime check and avoid +rereading files in some circumstances. + +Specifically, beets attempts to maintain the following invariant: + + If the on-disk metadata differs from the DB metadata, then the on-disk + mtime must be greater than the DB mtime. + +As a result, it is always valid for the DB mtime to be zero (assuming that real +disk mtimes are always positive). However, whenever possible, beets tries to +set ``db_mtime = disk_mtime`` at points where it knows the metadata is +synchronized. When it is possible that the metadata is out of sync, beets can +then just set ``db_mtime = 0`` to return to a consistent state. + +This leads to the following implementation policy: + + * On every write of disk metadata (``Item.write()``), the DB mtime is updated + to match the post-write disk mtime. + * Same for metadata reads (``Item.read()``). + * On every modification to DB metadata (``item.field = ...``), the DB mtime + is reset to zero. + + .. autoclass:: Item .. automethod:: __init__ @@ -130,6 +162,8 @@ update an item's fields (e.g., ``item.title = "Let It Be"``) and then call .. automethod:: destination + .. automethod:: current_mtime + The methods ``read()`` and ``write()`` are complementary: one reads a file's tags and updates the item's metadata fields accordingly while the other takes the item's fields and writes them to the file's tags. @@ -149,11 +183,6 @@ update an item's fields (e.g., ``item.title = "Let It Be"``) and then call .. automethod:: remove - Items also track their modification times (mtimes) to help detect when they - become out of sync with on-disk metadata. - - .. automethod:: current_mtime - Album ''''' From de78151eea5928aab1bd434efb5792b4df9501dc Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:18:46 +1000 Subject: [PATCH 032/613] docs: rename api -> library --- docs/dev/index.rst | 2 +- docs/dev/{api.rst => library.rst} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/dev/{api.rst => library.rst} (100%) diff --git a/docs/dev/index.rst b/docs/dev/index.rst index a47d6c8f2..ebeccc535 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -12,4 +12,4 @@ and write metadata tags in media files. .. toctree:: plugins - api + library diff --git a/docs/dev/api.rst b/docs/dev/library.rst similarity index 100% rename from docs/dev/api.rst rename to docs/dev/library.rst From 6769da29ae105d5d12fd71cb7b8eaba629742d6a Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Wed, 5 Jun 2019 13:28:06 +1000 Subject: [PATCH 033/613] docs: add dev importer and cli text from wiki --- docs/dev/cli.rst | 9 +++++++++ docs/dev/importer.rst | 19 +++++++++++++++++++ docs/dev/index.rst | 2 ++ 3 files changed, 30 insertions(+) create mode 100644 docs/dev/cli.rst create mode 100644 docs/dev/importer.rst diff --git a/docs/dev/cli.rst b/docs/dev/cli.rst new file mode 100644 index 000000000..77d3af5a5 --- /dev/null +++ b/docs/dev/cli.rst @@ -0,0 +1,9 @@ +Providing a CLI +=============== + +The ``beets.ui`` module houses interactions with the user via a terminal, the +:doc:`/reference/cli`. +The main function is called when the user types beet on the command line. +The CLI functionality is organized into commands, some of which are built-in +and some of which are provided by plugins. The built-in commands are all +implemented in the ``beets.ui.commands`` submodule. diff --git a/docs/dev/importer.rst b/docs/dev/importer.rst new file mode 100644 index 000000000..5182c7134 --- /dev/null +++ b/docs/dev/importer.rst @@ -0,0 +1,19 @@ +Music Importer +============== + +The importer component is responsible for the user-centric workflow that adds +music to a library. This is one of the first aspects that a user experiences +when using beets: it finds music in the filesystem, groups it into albums, +finds corresponding metadata in MusicBrainz, asks the user for intervention, +applies changes, and moves/copies files. A description of its user interface is +given in :doc:`/guides/tagger`. + +The workflow is implemented in the ``beets.importer`` module and is +distinct from the core logic for matching MusicBrainz metadata (in the +``beets.autotag`` module). The workflow is also decoupled from the command-line +interface with the hope that, eventually, other (graphical) interfaces can be +bolted onto the same importer implementation. + +The importer is multithreaded and follows the pipeline pattern. Each pipeline +stage is a Python coroutine. The ``beets.util.pipeline`` module houses +a generic, reusable implementation of a multithreaded pipeline. diff --git a/docs/dev/index.rst b/docs/dev/index.rst index ebeccc535..f1465494d 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -13,3 +13,5 @@ and write metadata tags in media files. plugins library + importer + cli From 8363dedaebae23797abc5d2cddb0b41f8faf6158 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Wed, 5 Jun 2019 11:10:11 +0200 Subject: [PATCH 034/613] logging and minor comments --- beetsplug/parentwork.py | 60 +++++++++++++++++-------------------- docs/plugins/parentwork.rst | 34 ++++++++++----------- test/test_parentwork.py | 2 +- 3 files changed, 45 insertions(+), 51 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 753f1fa5f..2ee6739d9 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -26,8 +26,9 @@ import musicbrainzngs def direct_parent_id(mb_workid, work_date=None): - """ Given a mb_workid, find the id one of the works the work is part of - and the first composition date it encounters.""" + """Given a mb_workid, find the id one of the works the work is part of + and the first composition date it encounters. + """ work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", "artist-rels"]) @@ -38,16 +39,17 @@ def direct_parent_id(mb_workid, work_date=None): work_date = artist['end'] if 'work-relation-list' in work_info['work']: - for work_father in work_info['work']['work-relation-list']: - if work_father['type'] == 'parts' \ - and work_father.get('direction') == 'backward': - father_id = work_father['work']['id'] - return father_id, work_date + for direct_parent in work_info['work']['work-relation-list']: + if direct_parent['type'] == 'parts' \ + and direct_parent.get('direction') == 'backward': + direct_id = direct_parent['work']['id'] + return direct_id, work_date return None, work_date def work_parent_id(mb_workid): - """Find the parentwork id and composition date of a work given its id. """ + """Find the parent work id and composition date of a work given its id. + """ work_date = None while True: new_mb_workid, work_date = direct_parent_id(mb_workid, work_date) @@ -59,7 +61,8 @@ def work_parent_id(mb_workid): def find_parentwork_info(mb_workid): """Return the work relationships (dict) and composition date of a - parentwork given the id of the work""" + parent work given the id of the work + """ parent_id, work_date = work_parent_id(mb_workid) work_info = musicbrainzngs.get_work_by_id(parent_id, includes=["artist-rels"]) @@ -112,9 +115,10 @@ class ParentWorkPlugin(BeetsPlugin): item.store() def get_info(self, item, work_info): - """Given the parentwork info dict, fetch parent_composer, + """Given the parent work info dict, fetch parent_composer, parent_composer_sort, parentwork, parentwork_disambig, mb_workid and - composer_ids. """ + composer_ids. + """ parent_composer = [] parent_composer_sort = [] @@ -132,10 +136,8 @@ class ParentWorkPlugin(BeetsPlugin): parent_composer_sort) if not composer_exists: - self._log.info(item.artist + ' - ' + item.title) - self._log.debug( - "no composer, add one at https://musicbrainz.org/work/" + - work_info['work']['id']) + self._log.debug('no composer for {}; add one at \ +https://musicbrainz.org/work/{}', item, work_info['work']['id']) parentwork_info['parentwork'] = work_info['work']['title'] parentwork_info['mb_parentworkid'] = work_info['work']['id'] @@ -150,43 +152,35 @@ class ParentWorkPlugin(BeetsPlugin): return parentwork_info def find_work(self, item, force): - """ Finds the parentwork of a recording and populates the tags + """Finds the parent work of a recording and populates the tags accordingly. Namely, the tags parentwork, parentwork_disambig, mb_parentworkid, - parent_composer, parent_composer_sort and work_date are populated. """ + parent_composer, parent_composer_sort and work_date are populated. + """ if hasattr(item, 'parentwork'): hasparent = True else: hasparent = False if not item.mb_workid: - self._log.info("No work attached, recording id: " + - item.mb_trackid) - self._log.info(item.artist + ' - ' + item.title) - self._log.info("add one at https://musicbrainz.org" + - "/recording/" + item.mb_trackid) + self._log.info('No work for {}, \ +add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return if force or (not hasparent): try: work_info, work_date = find_parentwork_info(item.mb_workid) - except musicbrainzngs.musicbrainz.WebServiceError: - self._log.debug("Work unreachable") + except musicbrainzngs.musicbrainz.WebServiceError as e: + self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) + self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], + parent_info['parent_composer']) elif hasparent: - self._log.debug("Work already in library, not necessary fetching") + self._log.debug("{} : Work present, skipping", item) return - self._log.debug("Finished searching work for: " + - item.artist + ' - ' + item.title) - if hasattr(item, 'parentwork'): - self._log.debug("Work fetched: " + parent_info['parentwork'] + - ' - ' + parent_info['parent_composer']) - else: - self._log.debug("Work fetched: " + parent_info['parentwork']) - for key, value in parent_info.items(): if value: item[key] = value diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index d76070b33..cb586cce1 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -1,29 +1,29 @@ Parentwork Plugin ================= -The ``parentwork`` plugin fetches the work title, parentwork title and -parentwork composer. +The ``parentwork`` plugin fetches the work title, parent work title and +parent work composer. In the MusicBrainz database, a recording can be associated with a work. A work can itself be associated with another work, for example one being part -of the other (what I call the father work). This plugin looks the work id -from the library and then looks up the father, then the father of the father -and so on until it reaches the top. The work at the top is what I call the -parentwork. This plugin is especially designed for classical music. For -classical music, just fetching the work title as in MusicBrainz is not -satisfying, because MusicBrainz has separate works for, for example, all the -movements of a symphony. This plugin aims to solve this problem by not only -fetching the work itself from MusicBrainz but also its parentwork which would -be, in this case, the whole symphony. +of the other (what I call the direct parent). This plugin looks the work id +from the library and then looks up the direct parent, then the direct parent +of the direct parent and so on until it reaches the top. The work at the top +is what I call the parent work. This plugin is especially designed for +classical music. For classical music, just fetching the work title as in +MusicBrainz is not satisfying, because MusicBrainz has separate works for, for +example, all the movements of a symphony. This plugin aims to solve this +problem by not only fetching the work itself from MusicBrainz but also its +parent work which would be, in this case, the whole symphony. This plugin adds five tags: -- **parentwork**: The title of the parentwork. -- **mb_parentworkid**: The musicbrainz id of the parentwork. -- **parentwork_disambig**: The disambiguation of the parentwork title. -- **parent_composer**: The composer of the parentwork. -- **parent_composer_sort**: The sort name of the parentwork composer. -- **work_date**: THe composition date of the work, or the first parent work +- **parentwork**: The title of the parent work. +- **mb_parentworkid**: The musicbrainz id of the parent work. +- **parentwork_disambig**: The disambiguation of the parent work title. +- **parent_composer**: The composer of the parent work. +- **parent_composer_sort**: The sort name of the parent work composer. +- **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. To use the ``parentwork`` plugin, enable it in your configuration (see diff --git a/test/test_parentwork.py b/test/test_parentwork.py index 8447fcdd1..dfebc6602 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -77,7 +77,7 @@ class ParentWorkTest(unittest.TestCase, TestHelper): # test different cases, still with Matthew Passion Ouverture or Mozart # requiem - def test_father_work(self, command_output): + def test_direct_parent_work(self, command_output): mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', parentwork.direct_parent_id(mb_workid)[0]) From fae065693576593c2f86cdcd349db98a51339333 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Wed, 5 Jun 2019 13:39:13 +0200 Subject: [PATCH 035/613] still dealing with cases where no parent composer --- beetsplug/parentwork.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 2ee6739d9..76a98fc08 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -174,8 +174,13 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) - self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], - parent_info['parent_composer']) + if 'parent_composer' in parent_info: + self._log.debug("Work fetched: {} - {}", + parent_info['parentwork'], + parent_info['parent_composer']) + else: + self._log.debug("Work fetched: {} - no parent composer", + parent_info['parentwork']) elif hasparent: self._log.debug("{} : Work present, skipping", item) From 1d9e42567b9dfc4238a861ae030911080a1a2e9d Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Wed, 5 Jun 2019 13:40:04 +0200 Subject: [PATCH 036/613] flake8 --- beetsplug/parentwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 76a98fc08..2c5af03d1 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -178,7 +178,7 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], parent_info['parent_composer']) - else: + else: self._log.debug("Work fetched: {} - no parent composer", parent_info['parentwork']) From 670046dd9ab2b8fce6df3f22822be4d7ec2a0971 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 22:55:12 +0200 Subject: [PATCH 037/613] Build https URLS for beatport releases I'm not sure where these are used, but the website supports https and the API url already uses https, so this should be a safe call and not require a util.SNI_SUPPORTED check. --- beetsplug/beatport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index da59bef87..ab50f46f2 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -224,7 +224,7 @@ class BeatportRelease(BeatportObject): if 'category' in data: self.category = data['category'] if 'slug' in data: - self.url = "http://beatport.com/release/{0}/{1}".format( + self.url = "https://beatport.com/release/{0}/{1}".format( data['slug'], data['id']) @@ -252,8 +252,8 @@ class BeatportTrack(BeatportObject): except ValueError: pass if 'slug' in data: - self.url = "http://beatport.com/track/{0}/{1}".format(data['slug'], - data['id']) + self.url = "https://beatport.com/track/{0}/{1}" \ + .format(data['slug'], data['id']) self.track_number = data.get('trackNumber') From 1a23eab8b6a5c1f721300d5298e1caddd6cf533f Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 23:00:52 +0200 Subject: [PATCH 038/613] Use https for lyrics.wikia.com, when supported --- beetsplug/lyrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 9e44eeef6..16699d9d3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -55,6 +55,7 @@ except ImportError: from beets import plugins from beets import ui +from beets import util import beets DIV_RE = re.compile(r'<(/?)div>?', re.I) @@ -406,7 +407,10 @@ class Genius(Backend): class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" - URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' + if util.SNI_SUPPORTED: + URL_PATTERN = 'https://lyrics.wikia.com/%s:%s' + else: + URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' def fetch(self, artist, title): url = self.build_url(artist, title) From c144141e9a8d7dd3374d1f077f6f1787033b5223 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 23:06:58 +0200 Subject: [PATCH 039/613] Update a few more http URLs to https that I missed Should really be all now (pending the next commit). --- beets/ui/__init__.py | 4 ++-- beets/util/__init__.py | 4 ++-- beetsplug/spotify.py | 2 +- docs/plugins/smartplaylist.rst | 2 +- docs/plugins/web.rst | 2 +- test/test_mb.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index a88ed9aef..aae3a1769 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -204,7 +204,7 @@ def input_(prompt=None): """ # raw_input incorrectly sends prompts to stderr, not stdout, so we # use print_() explicitly to display prompts. - # http://bugs.python.org/issue1927 + # https://bugs.python.org/issue1927 if prompt: print_(prompt, end=u' ') @@ -929,7 +929,7 @@ class CommonOptionsParser(optparse.OptionParser, object): # # This is a fairly generic subcommand parser for optparse. It is # maintained externally here: -# http://gist.github.com/462717 +# https://gist.github.com/462717 # There you will also find a better description of the code and a more # succinct example program. diff --git a/beets/util/__init__.py b/beets/util/__init__.py index e2348cf6e..162502eb1 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -412,7 +412,7 @@ def syspath(path, prefix=True): path = path.decode(encoding, 'replace') # Add the magic prefix if it isn't already there. - # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx + # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX): if path.startswith(u'\\\\'): # UNC path. Final path should look like \\?\UNC\... @@ -563,7 +563,7 @@ def unique_path(path): # Note: The Windows "reserved characters" are, of course, allowed on # Unix. They are forbidden here because they cause problems on Samba # shares, which are sufficiently common as to cause frequent problems. -# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx +# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx CHAR_REPLACE = [ (re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere. (re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix). diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index f6df91bb3..d8d7637d6 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -22,7 +22,7 @@ class SpotifyPlugin(BeetsPlugin): # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' - open_track_url = 'http://open.spotify.com/track/' + open_track_url = 'https://open.spotify.com/track/' search_url = 'https://api.spotify.com/v1/search' album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index 8ccbd0091..e68217657 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -5,7 +5,7 @@ Smart Playlist Plugin beets queries every time your library changes. This plugin is specifically created to work well with `MPD's`_ playlist functionality. -.. _MPD's: http://www.musicpd.org/ +.. _MPD's: https://www.musicpd.org/ To use it, enable the ``smartplaylist`` plugin in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index d3ae668ce..d416b1b7d 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -90,7 +90,7 @@ for unsupported formats/browsers. There are a number of options for this: .. _audio.js: https://kolber.github.io/audiojs/ .. _html5media: https://html5media.info/ -.. _MediaElement.js: http://mediaelementjs.com/ +.. _MediaElement.js: https://mediaelementjs.com/ .. _web-cors: diff --git a/test/test_mb.py b/test/test_mb.py index d5cb7c468..de1ffd9a7 100644 --- a/test/test_mb.py +++ b/test/test_mb.py @@ -459,7 +459,7 @@ class ParseIDTest(_common.TestCase): def test_parse_id_url_finds_id(self): id_string = "28e32c71-1450-463e-92bf-e0a46446fc11" - id_url = "http://musicbrainz.org/entity/%s" % id_string + id_url = "https://musicbrainz.org/entity/%s" % id_string out = mb._parse_id(id_url) self.assertEqual(out, id_string) From 9631616b538b1ebe60efe032c87d736b2744bda6 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 23:08:18 +0200 Subject: [PATCH 040/613] Replace a couple URLs that don't point to anything I'm unsure regarding the pygst tutorial, so I just added another URL of the best resource I could find with a quick web search. --- beets/ui/__init__.py | 2 +- beetsplug/bpd/gstplayer.py | 3 ++- docs/reference/cli.rst | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index aae3a1769..aec0e80a9 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -475,7 +475,7 @@ def human_seconds_short(interval): # Colorization. # ANSI terminal colorization code heavily inspired by pygments: -# http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py +# https://bitbucket.org/birkenfeld/pygments-main/src/default/pygments/console.py # (pygments is by Tim Hatch, Armin Ronacher, et al.) COLOR_ESCAPE = "\x1b[" DARK_COLORS = { diff --git a/beetsplug/bpd/gstplayer.py b/beetsplug/bpd/gstplayer.py index 8d4e7c9ff..3ba293bf2 100644 --- a/beetsplug/bpd/gstplayer.py +++ b/beetsplug/bpd/gstplayer.py @@ -64,7 +64,8 @@ class GstPlayer(object): """ # Set up the Gstreamer player. From the pygst tutorial: - # http://pygstdocs.berlios.de/pygst-tutorial/playbin.html + # https://pygstdocs.berlios.de/pygst-tutorial/playbin.html (gone) + # https://brettviren.github.io/pygst-tutorial-org/pygst-tutorial.html #### # Updated to GStreamer 1.0 with: # https://wiki.ubuntu.com/Novacut/GStreamer1.0 diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 2fc7c7b31..e17d5b42f 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -453,7 +453,7 @@ available via your package manager. On OS X, you can install it via Homebrew with ``brew install bash-completion``; Homebrew will give you instructions for sourcing the script. -.. _bash-completion: http://bash-completion.alioth.debian.org/ +.. _bash-completion: https://github.com/scop/bash-completion .. _bash: https://www.gnu.org/software/bash/ The completion script suggests names of subcommands and (after typing From 728203e15af1e741ee6f0f118d57dbfc7d5e0487 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Thu, 6 Jun 2019 15:34:15 +0200 Subject: [PATCH 041/613] beets.io now supports HTTPS See https://github.com/beetbox/beets/pull/3297. --- README.rst | 2 +- README_kr.rst | 4 ++-- beets/autotag/mb.py | 2 +- beetsplug/beatport.py | 4 ++-- beetsplug/discogs.py | 2 +- docs/changelog.rst | 6 +++--- docs/dev/library.rst | 2 +- docs/guides/advanced.rst | 2 +- docs/guides/main.rst | 4 ++-- docs/index.rst | 2 +- extra/_beet | 16 ++++++++-------- setup.py | 2 +- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index 8f8705abd..f9be39c52 100644 --- a/README.rst +++ b/README.rst @@ -103,7 +103,7 @@ Read More Learn more about beets at `its Web site`_. Follow `@b33ts`_ on Twitter for news and updates. -.. _its Web site: http://beets.io/ +.. _its Web site: https://beets.io/ .. _@b33ts: https://twitter.com/b33ts/ Authors diff --git a/README_kr.rst b/README_kr.rst index 6bdcf56a6..25dd052d8 100644 --- a/README_kr.rst +++ b/README_kr.rst @@ -34,7 +34,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 - 필요하는 메타 데이터를 계산하거나 패치 할 때: `album art`_, `lyrics`_, `genres`_, `tempos`_, `ReplayGain`_ levels, or `acoustic fingerprints`_. -- `MusicBrainz`_, `Discogs`_,`Beatport`_로부터 메타데이터를 가져오거나, +- `MusicBrainz`_, `Discogs`_,`Beatport`_로부터 메타데이터를 가져오거나, 노래 제목이나 음향 특징으로 메타데이터를 추측한다 - `Transcode audio`_ 당신이 좋아하는 어떤 포맷으로든 변경한다. - 당신의 라이브러리에서 `duplicate tracks and albums`_ 이나 `albums that are missing tracks`_ 를 검사한다. @@ -45,7 +45,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 - 명령어로부터 음악 파일의 메타데이터를 분석할 수 있다. - `MPD`_ 프로토콜을 사용하여 음악 플레이어로 음악을 들으면, 엄청나게 다양한 인터페이스로 작동한다. -만약 Beets에 당신이 원하는게 아직 없다면, +만약 Beets에 당신이 원하는게 아직 없다면, 당신이 python을 안다면 `writing your own plugin`_ _은 놀라울정도로 간단하다. .. _plugins: https://beets.readthedocs.org/page/plugins/ diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index b8f91f440..1a6e0b1f1 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -39,7 +39,7 @@ else: SKIPPED_TRACKS = ['[data track]'] musicbrainzngs.set_useragent('beets', beets.__version__, - 'http://beets.io/') + 'https://beets.io/') class MusicBrainzAPIError(util.HumanReadableException): diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index ab50f46f2..3462f118a 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -34,7 +34,7 @@ import confuse AUTH_ERRORS = (TokenRequestDenied, TokenMissing, VerifierMissing) -USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) +USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) class BeatportAPIError(Exception): @@ -109,7 +109,7 @@ class BeatportClient(object): :rtype: (unicode, unicode) tuple """ self.api.parse_authorization_response( - "http://beets.io/auth?" + auth_data) + "https://beets.io/auth?" + auth_data) access_data = self.api.fetch_access_token( self._make_url('/identity/1/oauth/access-token')) return access_data['oauth_token'], access_data['oauth_token_secret'] diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 68b4b5a95..6a0a9c531 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -37,7 +37,7 @@ import traceback from string import ascii_lowercase -USER_AGENT = u'beets/{0} +http://beets.io/'.format(beets.__version__) +USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) # Exceptions that discogs_client should really handle but does not. CONNECTION_ERRORS = (ConnectionError, socket.error, http_client.HTTPException, diff --git a/docs/changelog.rst b/docs/changelog.rst index a667be780..264a82107 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1241,7 +1241,7 @@ Fixes: * :doc:`/plugins/replaygain`: Fix a crash using the Python Audio Tools backend. :bug:`1873` -.. _beets.io: http://beets.io/ +.. _beets.io: https://beets.io/ .. _Beetbox: https://github.com/beetbox @@ -2639,7 +2639,7 @@ previous versions would spit out a warning and then list your entire library. There's more detail than you could ever need `on the beets blog`_. -.. _on the beets blog: http://beets.io/blog/flexattr.html +.. _on the beets blog: https://beets.io/blog/flexattr.html 1.2.2 (August 27, 2013) @@ -3227,7 +3227,7 @@ begins today on features for version 1.1. unintentionally loading the plugins they contain. .. _The Echo Nest: http://the.echonest.com/ -.. _Tomahawk resolver: http://beets.io/blog/tomahawk-resolver.html +.. _Tomahawk resolver: https://beets.io/blog/tomahawk-resolver.html .. _mp3gain: http://mp3gain.sourceforge.net/download.php .. _aacgain: https://aacgain.altosdesign.com diff --git a/docs/dev/library.rst b/docs/dev/library.rst index ec81c169b..77e218b93 100644 --- a/docs/dev/library.rst +++ b/docs/dev/library.rst @@ -242,7 +242,7 @@ to interact directly with the underlying SQLite database, you must use a items = lib.items(query) lib.add_album(list(items)) -.. _blog post: http://beets.io/blog/sqlite-nightmare.html +.. _blog post: https://beets.io/blog/sqlite-nightmare.html .. currentmodule:: beets.dbcore.db diff --git a/docs/guides/advanced.rst b/docs/guides/advanced.rst index 091875c54..f4f8d3cd9 100644 --- a/docs/guides/advanced.rst +++ b/docs/guides/advanced.rst @@ -127,7 +127,7 @@ And, unlike :ref:`built-in fields `, such fields can be removed:: Read more than you ever wanted to know about the *flexible attributes* feature `on the beets blog`_. -.. _on the beets blog: http://beets.io/blog/flexattr.html +.. _on the beets blog: https://beets.io/blog/flexattr.html Choose a path style manually for some music diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 1f5cc4681..2f05634d9 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -4,7 +4,7 @@ Getting Started Welcome to `beets`_! This guide will help you begin using it to make your music collection better. -.. _beets: http://beets.io/ +.. _beets: https://beets.io/ Installing ---------- @@ -43,7 +43,7 @@ Beets works on `Python 2.7`_ and Python 3.4 or later. * On **Fedora** 22 or later, there is a `DNF package`_:: $ sudo dnf install beets beets-plugins beets-doc - + * On **Solus**, run ``eopkg install beets``. * On **NixOS**, there's a `package `_ you can install with ``nix-env -i beets``. diff --git a/docs/index.rst b/docs/index.rst index 27fa4740b..4919147ce 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ Freenode, drop by `the discussion board`_, send email to `the mailing list`_, or `file a bug`_ in the issue tracker. Please let us know where you think this documentation can be improved. -.. _beets: http://beets.io/ +.. _beets: https://beets.io/ .. _the mailing list: https://groups.google.com/group/beets-users .. _file a bug: https://github.com/beetbox/beets/issues .. _the discussion board: https://discourse.beets.io diff --git a/extra/_beet b/extra/_beet index a8c9083be..129c0485e 100644 --- a/extra/_beet +++ b/extra/_beet @@ -1,6 +1,6 @@ #compdef beet -# zsh completion for beets music library manager and MusicBrainz tagger: http://beets.io/ +# zsh completion for beets music library manager and MusicBrainz tagger: https://beets.io/ # Default values for BEETS_LIBRARY & BEETS_CONFIG needed for the cache checking function. # They will be updated under the assumption that the config file is in the same directory as the library. @@ -34,7 +34,7 @@ _beet_check_cache () { # useful: argument to _regex_arguments for matching any word local matchany=/$'[^\0]##\0'/ # arguments to _regex_arguments for completing files and directories -local -a files dirs +local -a files dirs files=("$matchany" ':file:file:_files') dirs=("$matchany" ':dir:directory:_dirs') @@ -73,7 +73,7 @@ if ! _retrieve_cache beetscmds || _cache_invalid beetscmds; then # create completion function for queries _regex_arguments _beet_query "$matchany" \# \( "$matchquery" ":query:query string:$queryelem" \) \# local "beets_query"="$(which _beet_query)" - # arguments for _regex_arguments for completing lists of queries and modifications + # arguments for _regex_arguments for completing lists of queries and modifications beets_query_args=( \( "$matchquery" ":query:query string:{_beet_query}" \) \# ) beets_modify_args=( \( "$matchmodify" ":modify:modify string:$modifyelem" \) \# ) # now build arguments for _beet and _beet_help completion functions @@ -82,7 +82,7 @@ if ! _retrieve_cache beetscmds || _cache_invalid beetscmds; then subcmd="${i[(w)1]}" # remove first word and parenthesised alias, replace : with -, [ with (, ] with ), and remove single quotes cmddesc="${${${${${i[(w)2,-1]##\(*\) #}//:/-}//\[/(}//\]/)}//\'/}" - # update arguments needed for creating _beet + # update arguments needed for creating _beet beets_regex_words_subcmds+=(/"${subcmd}"$'\0'/ ":subcmds:subcommands:((${subcmd}:${cmddesc// /\ }))") beets_regex_words_subcmds+=(\( "${matchany}" ":option:option:{_beet_subcmd ${subcmd}}" \) \# \|) # update arguments needed for creating _beet_help @@ -137,7 +137,7 @@ _beet_subcmd_options() { fi ;; (LOG) - local -a files + local -a files files=("$matchany" ':file:file:_files') regex_words+=("$opt:$optdesc:\$files") ;; @@ -180,7 +180,7 @@ _beet_subcmd() { if [[ ! $(type _beet_${subcmd} | grep function) =~ function ]]; then if ! _retrieve_cache "beets${subcmd}" || _cache_invalid "beets${subcmd}"; then local matchany=/$'[^\0]##\0'/ - local -a files + local -a files files=("$matchany" ':file:file:_files') # get arguments for completing subcommand options _beet_subcmd_options "$subcmd" @@ -197,7 +197,7 @@ _beet_subcmd() { (fields|migrate|version|config) _regex_arguments _beet_${subcmd} "${matchany}" /"${subcmd}"$'\0'/ "${options[@]}" ;; - (help) + (help) _regex_words subcmds "subcommands" "${beets_regex_words_help[@]}" _regex_arguments _beet_help "${matchany}" /$'help\0'/ "${options[@]}" "${reply[@]}" ;; @@ -232,6 +232,6 @@ zstyle ":completion:${curcontext}:" tag-order '! options' # Execute the completion function _beet "$@" -# Local Variables: +# Local Variables: # mode:shell-script # End: diff --git a/setup.py b/setup.py index 30db5272d..1078d6cc9 100755 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( description='music tagger and library organizer', author='Adrian Sampson', author_email='adrian@radbox.org', - url='http://beets.io/', + url='https://beets.io/', license='MIT', platforms='ALL', long_description=_read('README.rst'), From 2b00e1de2448b9a80545935354d06950be1ff65f Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 7 Jun 2019 14:17:39 +0200 Subject: [PATCH 042/613] beetsplug/importadded: Add missing path kwarg to update_after_write_time() This resolves #3301. --- beetsplug/importadded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/importadded.py b/beetsplug/importadded.py index 36407b14c..29aeeab0f 100644 --- a/beetsplug/importadded.py +++ b/beetsplug/importadded.py @@ -124,7 +124,7 @@ class ImportAddedPlugin(BeetsPlugin): util.displayable_path(item.path), item.added) item.store() - def update_after_write_time(self, item): + def update_after_write_time(self, item, path): """Update the mtime of the item's file with the item.added value after each write of the item if `preserve_write_mtimes` is enabled. """ From 765f7dc12d6732a9536df715cc3ac4f0d63aa57e Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 7 Jun 2019 14:57:38 +0200 Subject: [PATCH 043/613] first try to implement event handler --- beetsplug/parentwork.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 2c5af03d1..ee892e825 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -77,9 +77,14 @@ class ParentWorkPlugin(BeetsPlugin): 'auto': False, 'force': False, }) + if self.config['auto']: self.import_stages = [self.imported] + self.register_listener('database_change', self.find_work2) + + def find_work2(self, lib, model): + self.find_work(model, True) def commands(self): From 9c3c538dfb3940275148193f0011c107f632c523 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 7 Jun 2019 16:51:33 +0200 Subject: [PATCH 044/613] alternative way to refetch parent works --- beetsplug/parentwork.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index ee892e825..2c04fb5e9 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -77,14 +77,9 @@ class ParentWorkPlugin(BeetsPlugin): 'auto': False, 'force': False, }) - if self.config['auto']: self.import_stages = [self.imported] - self.register_listener('database_change', self.find_work2) - - def find_work2(self, lib, model): - self.find_work(model, True) def commands(self): @@ -161,7 +156,8 @@ https://musicbrainz.org/work/{}', item, work_info['work']['id']) accordingly. Namely, the tags parentwork, parentwork_disambig, mb_parentworkid, - parent_composer, parent_composer_sort and work_date are populated. + parent_composer, parent_composer_sort, mb_workid_current and work_date + are populated. """ if hasattr(item, 'parentwork'): @@ -172,13 +168,17 @@ https://musicbrainz.org/work/{}', item, work_info['work']['id']) self._log.info('No work for {}, \ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return - if force or (not hasparent): + workcorrect=True + if hasattr(item, 'mb_workid_current'): + workcorrect=item.mb_workid_current==item.mb_workid + if force or (not hasparent) or (not workcorrect): try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) + parent_info['mb_workid_current']=item.mb_workid if 'parent_composer' in parent_info: self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], From eacdb0d0e4334fe8a84f54b0779cbb2d3c4b232f Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 7 Jun 2019 17:15:04 +0200 Subject: [PATCH 045/613] refetching works moved to new PR --- beetsplug/parentwork.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 2c04fb5e9..681cff226 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -168,17 +168,13 @@ https://musicbrainz.org/work/{}', item, work_info['work']['id']) self._log.info('No work for {}, \ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return - workcorrect=True - if hasattr(item, 'mb_workid_current'): - workcorrect=item.mb_workid_current==item.mb_workid - if force or (not hasparent) or (not workcorrect): + if force or (not hasparent): try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) - parent_info['mb_workid_current']=item.mb_workid if 'parent_composer' in parent_info: self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], From 070f50e1e7b9299bc39e61a1993f725a7634008e Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Fri, 7 Jun 2019 17:16:03 +0200 Subject: [PATCH 046/613] docstring --- beetsplug/parentwork.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 681cff226..2c5af03d1 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -156,8 +156,7 @@ https://musicbrainz.org/work/{}', item, work_info['work']['id']) accordingly. Namely, the tags parentwork, parentwork_disambig, mb_parentworkid, - parent_composer, parent_composer_sort, mb_workid_current and work_date - are populated. + parent_composer, parent_composer_sort and work_date are populated. """ if hasattr(item, 'parentwork'): From 2c0d9b07dbf4f5581ed186032177d2d61720d781 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Sat, 8 Jun 2019 16:23:24 +0200 Subject: [PATCH 047/613] Fixed changelog for replaygain per_disc option --- docs/changelog.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 59def9fae..4265646bc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,11 @@ New features: to MPD commands we don't support yet. Let us know if you find an MPD client that doesn't get along with BPD! :bug:`3214` :bug:`800` +* :doc:`/plugins/replaygain`: The plugin now supports a ``per_disc`` option + which enables calculation of album ReplayGain on disc level instead of album + level. + Thanks to :user:`samuelnilsson` + :bug:`293` Fixes: @@ -140,10 +145,6 @@ The new core features are: :bug:`3123` * :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which passes that flag to ipfs. Thanks to :user:`wildthyme`. -* :doc:`/plugins/replaygain`: The plugin now supports a ``per_disc`` option - which enables calculation of album ReplayGain on disc level instead of album - level. - Thanks to :user:`samuelnilsson` ======= * A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks From f5c8650dc9f5a846f099af8779cecc18d40bc5ff Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Sat, 8 Jun 2019 16:44:03 +0200 Subject: [PATCH 048/613] Fixed merge issue regarding replaygain per_disc option --- docs/changelog.rst | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4265646bc..baef71508 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -126,27 +126,6 @@ The new core features are: example, ``beet modify -a artist:beatles artpath!`` resets ``artpath`` attribute from matching albums back to the default value. :bug:`2497` -* Modify selection can now be applied early without selecting every item. - :bug:`3083` -* :doc:`/plugins/chroma`: Fingerprint values are now properly stored as - strings, which prevents strange repeated output when running ``beet write``. - Thanks to :user:`Holzhaus`. - :bug:`3097` :bug:`2942` -* The ``move`` command now lists the number of items already in-place. - Thanks to :user:`RollingStar`. - :bug:`3117` -* :doc:`/plugins/spotify`: The plugin now uses OAuth for authentication to the - Spotify API. - Thanks to :user:`rhlahuja`. - :bug:`2694` :bug:`3123` -* :doc:`/plugins/spotify`: The plugin now works as an import metadata - provider: you can match tracks and albums using the Spotify database. - Thanks to :user:`rhlahuja`. - :bug:`3123` -* :doc:`/plugins/ipfs`: The plugin now supports a ``nocopy`` option which passes that flag to ipfs. - Thanks to :user:`wildthyme`. - -======= * A new importer option, :ref:`ignore_data_tracks`, lets you skip audio tracks contained in data files. :bug:`3021` From f865fc00cd7a1a8bff513f5cf7ad90ce0c4d34ac Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 8 Jun 2019 16:23:49 -0400 Subject: [PATCH 049/613] replaygain: Fix py3 crash in audiotools backend Fixes #3305. --- beetsplug/replaygain.py | 2 +- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 4cc5f435c..8e11ee370 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -713,7 +713,7 @@ class AudioToolsBackend(Backend): file format is not supported """ try: - audiofile = self._mod_audiotools.open(item.path) + audiofile = self._mod_audiotools.open(py3_path(syspath(item.path))) except IOError: raise ReplayGainError( u"File {} was not found".format(item.path) diff --git a/docs/changelog.rst b/docs/changelog.rst index baef71508..06c662be6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,9 @@ Fixes: fixing crashes in MPD clients like mpDris2 on seek. The ``playlistid`` command now works properly in its zero-argument form. :bug:`3214` +* :doc:`/plugins/replaygain`: Fix a Python 3 incompatibility in the Python + Audio Tools backend. + :bug:`3305` For plugin developers: From c96dcfffb65d31243491699838cf4bd51d1f0432 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 8 Jun 2019 22:44:33 +0200 Subject: [PATCH 050/613] docstrings and style --- beetsplug/parentwork.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 2c5af03d1..3b8e272d4 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -60,8 +60,8 @@ def work_parent_id(mb_workid): def find_parentwork_info(mb_workid): - """Return the work relationships (dict) and composition date of a - parent work given the id of the work + """Get the MusicBrainz information dict about a parent work, including + the artist relations, and the composition date for a work's parent work. """ parent_id, work_date = work_parent_id(mb_workid) work_info = musicbrainzngs.get_work_by_id(parent_id, @@ -95,12 +95,12 @@ class ParentWorkPlugin(BeetsPlugin): item.try_write() command = ui.Subcommand( 'parentwork', - help=u'Fetches parent works, composers and dates') + help=u'fetche parent works, composers and dates') command.parser.add_option( u'-f', u'--force', dest='force', action='store_true', default=None, - help=u'Re-fetches all parent works') + help=u're-fetch when parent work is already present') command.func = func return [command] @@ -155,19 +155,21 @@ https://musicbrainz.org/work/{}', item, work_info['work']['id']) """Finds the parent work of a recording and populates the tags accordingly. + The parent work is found recursively, by finding the direct parent + repeatedly until there are no more links in the chain. We return the + final, topmost work in the chain. + Namely, the tags parentwork, parentwork_disambig, mb_parentworkid, parent_composer, parent_composer_sort and work_date are populated. """ - if hasattr(item, 'parentwork'): - hasparent = True - else: - hasparent = False if not item.mb_workid: self._log.info('No work for {}, \ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return - if force or (not hasparent): + + hasparent = hasattr(item, 'parentwork') + if force or not hasparent: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: @@ -183,9 +185,10 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) parent_info['parentwork']) elif hasparent: - self._log.debug("{} : Work present, skipping", item) + self._log.debug("{}: Work present, skipping", item) return + # apply all non-null values to the item for key, value in parent_info.items(): if value: item[key] = value @@ -196,5 +199,3 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date']) - - item.store() From 022e3d44ead6ba688d5c08cf2a1092a84b837a54 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 8 Jun 2019 22:46:17 +0200 Subject: [PATCH 051/613] Update docs/plugins/parentwork.rst Co-Authored-By: Adrian Sampson --- docs/plugins/parentwork.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index cb586cce1..188826e7e 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -2,7 +2,7 @@ Parentwork Plugin ================= The ``parentwork`` plugin fetches the work title, parent work title and -parent work composer. +parent work composer from MusicBrainz. In the MusicBrainz database, a recording can be associated with a work. A work can itself be associated with another work, for example one being part From c8c206f19e3ec6ec15c264861b960864239d8803 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 8 Jun 2019 22:46:32 +0200 Subject: [PATCH 052/613] Update docs/plugins/parentwork.rst Co-Authored-By: Adrian Sampson --- docs/plugins/parentwork.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index 188826e7e..74d22320f 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -9,7 +9,7 @@ work can itself be associated with another work, for example one being part of the other (what I call the direct parent). This plugin looks the work id from the library and then looks up the direct parent, then the direct parent of the direct parent and so on until it reaches the top. The work at the top -is what I call the parent work. This plugin is especially designed for +is what we call the *parent work*. This plugin is especially designed for classical music. For classical music, just fetching the work title as in MusicBrainz is not satisfying, because MusicBrainz has separate works for, for example, all the movements of a symphony. This plugin aims to solve this From 2c3389beae9d1cf6b45ffe4567a7ad929a79747d Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 8 Jun 2019 22:46:47 +0200 Subject: [PATCH 053/613] Update docs/plugins/parentwork.rst Co-Authored-By: Adrian Sampson --- docs/plugins/parentwork.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index 74d22320f..e4469ae90 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -6,7 +6,7 @@ parent work composer from MusicBrainz. In the MusicBrainz database, a recording can be associated with a work. A work can itself be associated with another work, for example one being part -of the other (what I call the direct parent). This plugin looks the work id +of the other (what we call the *direct parent*). This plugin looks the work id from the library and then looks up the direct parent, then the direct parent of the direct parent and so on until it reaches the top. The work at the top is what we call the *parent work*. This plugin is especially designed for From fd14b5b64927c1bbe3a6a39ed000a41c01a5fd98 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 8 Jun 2019 22:55:05 +0200 Subject: [PATCH 054/613] docstrings and style --- beetsplug/parentwork.py | 2 +- docs/plugins/parentwork.rst | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 3b8e272d4..8e3dff32f 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -26,7 +26,7 @@ import musicbrainzngs def direct_parent_id(mb_workid, work_date=None): - """Given a mb_workid, find the id one of the works the work is part of + """Given a Musicbrainz id, find the id one of the works the work is part of and the first composition date it encounters. """ work_info = musicbrainzngs.get_work_by_id(mb_workid, diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index e4469ae90..a221e19b5 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -43,4 +43,3 @@ configuration file. The available options are: - **auto**: If enabled, automatically fetches works at import. It takes quite some time, because beets is restricted to one musicbrainz query per second. Default: ``no`` - From 9d184e3cade99e9a6529230f0f1ac095d05602ba Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 8 Jun 2019 22:58:05 +0200 Subject: [PATCH 055/613] docstrings and style --- beetsplug/parentwork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 8e3dff32f..63ef0102c 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -26,8 +26,8 @@ import musicbrainzngs def direct_parent_id(mb_workid, work_date=None): - """Given a Musicbrainz id, find the id one of the works the work is part of - and the first composition date it encounters. + """Given a Musicbrainz work id, find the id one of the works the work is + part of and the first composition date it encounters. """ work_info = musicbrainzngs.get_work_by_id(mb_workid, includes=["work-rels", From 9fcb66b3c82779576840ae219f56b34c82a3f923 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 8 Jun 2019 21:35:56 -0400 Subject: [PATCH 056/613] Nicer string wrap --- beetsplug/parentwork.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 63ef0102c..eaa8abb30 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -136,8 +136,11 @@ class ParentWorkPlugin(BeetsPlugin): parent_composer_sort) if not composer_exists: - self._log.debug('no composer for {}; add one at \ -https://musicbrainz.org/work/{}', item, work_info['work']['id']) + self._log.debug( + 'no composer for {}; add one at ' + 'https://musicbrainz.org/work/{}', + item, work_info['work']['id'], + ) parentwork_info['parentwork'] = work_info['work']['title'] parentwork_info['mb_parentworkid'] = work_info['work']['id'] From bfb94363c35ce65705bcd4a0c595c7825fab794e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 8 Jun 2019 21:37:24 -0400 Subject: [PATCH 057/613] Changelog & doc tweaks for #3279 --- docs/changelog.rst | 4 +++ docs/plugins/parentwork.rst | 54 ++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 06c662be6..e21168078 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,10 @@ New features: level. Thanks to :user:`samuelnilsson` :bug:`293` +* A new :doc:`/plugins/parentwork` gets information about the original work, + which is useful for classical music. + Thanks to :user:`dosoe`. + :bug:`2580` :bug:`3279` Fixes: diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index a221e19b5..cb9d7e6f6 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -1,30 +1,30 @@ -Parentwork Plugin +ParentWork Plugin ================= -The ``parentwork`` plugin fetches the work title, parent work title and +The ``parentwork`` plugin fetches the work title, parent work title and parent work composer from MusicBrainz. -In the MusicBrainz database, a recording can be associated with a work. A -work can itself be associated with another work, for example one being part -of the other (what we call the *direct parent*). This plugin looks the work id -from the library and then looks up the direct parent, then the direct parent -of the direct parent and so on until it reaches the top. The work at the top -is what we call the *parent work*. This plugin is especially designed for -classical music. For classical music, just fetching the work title as in -MusicBrainz is not satisfying, because MusicBrainz has separate works for, for -example, all the movements of a symphony. This plugin aims to solve this -problem by not only fetching the work itself from MusicBrainz but also its -parent work which would be, in this case, the whole symphony. +In the MusicBrainz database, a recording can be associated with a work. A +work can itself be associated with another work, for example one being part +of the other (what we call the *direct parent*). This plugin looks the work id +from the library and then looks up the direct parent, then the direct parent +of the direct parent and so on until it reaches the top. The work at the top +is what we call the *parent work*. This plugin is especially designed for +classical music. For classical music, just fetching the work title as in +MusicBrainz is not satisfying, because MusicBrainz has separate works for, for +example, all the movements of a symphony. This plugin aims to solve this +problem by not only fetching the work itself from MusicBrainz but also its +parent work which would be, in this case, the whole symphony. -This plugin adds five tags: +This plugin adds five tags: -- **parentwork**: The title of the parent work. -- **mb_parentworkid**: The musicbrainz id of the parent work. -- **parentwork_disambig**: The disambiguation of the parent work title. -- **parent_composer**: The composer of the parent work. -- **parent_composer_sort**: The sort name of the parent work composer. -- **work_date**: The composition date of the work, or the first parent work - that has a composition date. Format: yyyy-mm-dd. +- **parentwork**: The title of the parent work. +- **mb_parentworkid**: The musicbrainz id of the parent work. +- **parentwork_disambig**: The disambiguation of the parent work title. +- **parent_composer**: The composer of the parent work. +- **parent_composer_sort**: The sort name of the parent work composer. +- **work_date**: The composition date of the work, or the first parent work + that has a composition date. Format: yyyy-mm-dd. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). @@ -35,11 +35,11 @@ Configuration To configure the plugin, make a ``parentwork:`` section in your configuration file. The available options are: -- **force**: As a default, ``parentwork`` only fetches work info for - recordings that do not already have a ``parentwork`` tag. If ``force`` - is enabled, it fetches it for all recordings. +- **force**: As a default, ``parentwork`` only fetches work info for + recordings that do not already have a ``parentwork`` tag. If ``force`` + is enabled, it fetches it for all recordings. Default: ``no`` - -- **auto**: If enabled, automatically fetches works at import. It takes quite - some time, because beets is restricted to one musicbrainz query per second. + +- **auto**: If enabled, automatically fetches works at import. It takes quite + some time, because beets is restricted to one musicbrainz query per second. Default: ``no`` From 6a9616796cb5caf3a09abe458b320b4f259d8e2c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 8 Jun 2019 21:38:42 -0400 Subject: [PATCH 058/613] Break up a long paragraph --- docs/plugins/parentwork.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index cb9d7e6f6..9707650b4 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -9,12 +9,14 @@ work can itself be associated with another work, for example one being part of the other (what we call the *direct parent*). This plugin looks the work id from the library and then looks up the direct parent, then the direct parent of the direct parent and so on until it reaches the top. The work at the top -is what we call the *parent work*. This plugin is especially designed for +is what we call the *parent work*. + +This plugin is especially designed for classical music. For classical music, just fetching the work title as in MusicBrainz is not satisfying, because MusicBrainz has separate works for, for example, all the movements of a symphony. This plugin aims to solve this -problem by not only fetching the work itself from MusicBrainz but also its -parent work which would be, in this case, the whole symphony. +problem by also fetching the parent work, which would be the whole symphony in +this example. This plugin adds five tags: From 851c413976c83080e1cff3afe4a01364afc88461 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 10:37:33 +0200 Subject: [PATCH 059/613] adding config option for seperator and addressing review comments --- beets/config_default.yaml | 1 - beetsplug/discogs.py | 5 ++--- test/test_discogs.py | 3 +++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 538753bb7..cf9ae6bf9 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -131,7 +131,6 @@ match: track_index: 1.0 track_length: 2.0 track_id: 5.0 - style: 5.0 preferred: countries: [] media: [] diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 0865b691b..88e0704d9 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -55,6 +55,7 @@ class DiscogsPlugin(BeetsPlugin): 'tokenfile': 'discogs_token.json', 'source_weight': 0.5, 'user_token': '', + 'separator': u', ' }) self.config['apikey'].redact = True self.config['apisecret'].redact = True @@ -303,14 +304,12 @@ class DiscogsPlugin(BeetsPlugin): country = result.data.get('country') data_url = result.data.get('uri') style = result.data.get('styles') - print('style', style) if style is None: self._log.info('Style not Found') - return "Style not Defined" elif len(style) == 0: return style else: - style = ' - '.join(sorted(style)) + style = self.config['separator'].as_str().join(sorted(style)) # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. diff --git a/test/test_discogs.py b/test/test_discogs.py index 8b2eff9f1..4445014be 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -45,6 +45,9 @@ class DGAlbumInfoTest(_common.TestCase): 'name': 'FORMAT', 'qty': 1 }], + 'styles': [{ + 'STYLE' + }], 'labels': [{ 'name': 'LABEL NAME', 'catno': 'CATALOG NUMBER', From 6cdd1ab6c1fccd28f9a083e7a16284572921d1dc Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 12:08:07 +0200 Subject: [PATCH 060/613] fixing test --- test/test_discogs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/test_discogs.py b/test/test_discogs.py index 4445014be..cc97017fe 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -45,9 +45,9 @@ class DGAlbumInfoTest(_common.TestCase): 'name': 'FORMAT', 'qty': 1 }], - 'styles': [{ - 'STYLE' - }], + 'styles': [ + 'STYLE1', 'STYLE2' + ], 'labels': [{ 'name': 'LABEL NAME', 'catno': 'CATALOG NUMBER', @@ -59,6 +59,7 @@ class DGAlbumInfoTest(_common.TestCase): for recording in tracks: data['tracklist'].append(recording) + return Bag(data=data, # Make some fields available as properties, as they are # accessed by DiscogsPlugin methods. @@ -84,6 +85,8 @@ class DGAlbumInfoTest(_common.TestCase): tracklist where tracks have the specified `positions`.""" tracks = [self._make_track('TITLE%s' % i, position) for (i, position) in enumerate(positions, start=1)] + release = self._make_release(tracks) + print('release', release) return self._make_release(tracks) def test_parse_media_for_tracks(self): @@ -92,6 +95,7 @@ class DGAlbumInfoTest(_common.TestCase): release = self._make_release(tracks=tracks) d = DiscogsPlugin().get_album_info(release) + print('albumInfo', d.media) t = d.tracks self.assertEqual(d.media, 'FORMAT') self.assertEqual(t[0].media, d.media) From 319e1da727062605b86a4047aaa3d008489eedad Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 12:29:45 +0200 Subject: [PATCH 061/613] fixing tox.ini file to match master --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e3250bd6b..8736f0f3c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py38-test, py27-flake8, docs +envlist = py27-test, py37-test, py27-flake8, docs # The exhaustive list of environments is: # envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs From 371d978e134314cc22a87a9372cfa99aae06b3b7 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 12:39:21 +0200 Subject: [PATCH 062/613] removing blank line and making line shorter --- beets/autotag/hooks.py | 4 ++-- test/test_discogs.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index f822cdfde..b8e6108b4 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -79,8 +79,8 @@ class AlbumInfo(object): albumtype=None, va=False, year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, style=None, albumstatus=None, media=None, - albumdisambig=None, releasegroupdisambig=None, + language=None, country=None, style=None, albumstatus=None, + media=None, albumdisambig=None, releasegroupdisambig=None, artist_credit=None, original_year=None, original_month=None, original_day=None, data_source=None, data_url=None): self.album = album diff --git a/test/test_discogs.py b/test/test_discogs.py index cc97017fe..403f01da7 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -59,7 +59,6 @@ class DGAlbumInfoTest(_common.TestCase): for recording in tracks: data['tracklist'].append(recording) - return Bag(data=data, # Make some fields available as properties, as they are # accessed by DiscogsPlugin methods. From 9789c465aab4f061a49c5391f42204e46a64c9d3 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 12:49:04 +0200 Subject: [PATCH 063/613] removing blank line --- test/test_discogs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_discogs.py b/test/test_discogs.py index 403f01da7..36239d697 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -357,7 +357,6 @@ class DGAlbumInfoTest(_common.TestCase): self.assertEqual(d, None) self.assertIn('Release does not contain the required fields', logs[0]) - def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 77dcd63254b802af806ee07a1b10d2e2f8f78534 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 12:59:32 +0200 Subject: [PATCH 064/613] adding line --- test/test_discogs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_discogs.py b/test/test_discogs.py index 36239d697..403f01da7 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -357,6 +357,7 @@ class DGAlbumInfoTest(_common.TestCase): self.assertEqual(d, None) self.assertIn('Release does not contain the required fields', logs[0]) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 5fc21a1e211b8a83a76ebef33b27ddd7f39cd3ec Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 15:39:49 +0200 Subject: [PATCH 065/613] fixing per review comments --- beetsplug/discogs.py | 6 ++++-- test/test_discogs.py | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 88e0704d9..ed53d0012 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -304,9 +304,11 @@ class DiscogsPlugin(BeetsPlugin): country = result.data.get('country') data_url = result.data.get('uri') style = result.data.get('styles') + print('style', style) if style is None: - self._log.info('Style not Found') - elif len(style) == 0: + self._log.debug('Style not Found') + elif not style: + print('s', style) return style else: style = self.config['separator'].as_str().join(sorted(style)) diff --git a/test/test_discogs.py b/test/test_discogs.py index 403f01da7..8c9d7a249 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -85,7 +85,6 @@ class DGAlbumInfoTest(_common.TestCase): tracks = [self._make_track('TITLE%s' % i, position) for (i, position) in enumerate(positions, start=1)] release = self._make_release(tracks) - print('release', release) return self._make_release(tracks) def test_parse_media_for_tracks(self): @@ -94,7 +93,6 @@ class DGAlbumInfoTest(_common.TestCase): release = self._make_release(tracks=tracks) d = DiscogsPlugin().get_album_info(release) - print('albumInfo', d.media) t = d.tracks self.assertEqual(d.media, 'FORMAT') self.assertEqual(t[0].media, d.media) From 2c49c12166f49f0e3e2736577cc0eeb66863ea12 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 15:44:37 +0200 Subject: [PATCH 066/613] fixing per review comments --- beetsplug/discogs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index ed53d0012..ad170ca0b 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -304,11 +304,9 @@ class DiscogsPlugin(BeetsPlugin): country = result.data.get('country') data_url = result.data.get('uri') style = result.data.get('styles') - print('style', style) if style is None: self._log.debug('Style not Found') - elif not style: - print('s', style) + elif len(style) == 0: return style else: style = self.config['separator'].as_str().join(sorted(style)) From f0c91b8f45747bae4b6daf354c69bcecf8ff7e30 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 9 Jun 2019 20:01:55 +0200 Subject: [PATCH 067/613] fixing per review comments --- beetsplug/discogs.py | 2 +- test/test_discogs.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index ad170ca0b..e4c82cd14 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -306,7 +306,7 @@ class DiscogsPlugin(BeetsPlugin): style = result.data.get('styles') if style is None: self._log.debug('Style not Found') - elif len(style) == 0: + elif not style: return style else: style = self.config['separator'].as_str().join(sorted(style)) diff --git a/test/test_discogs.py b/test/test_discogs.py index 8c9d7a249..0acf54e8a 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -84,7 +84,6 @@ class DGAlbumInfoTest(_common.TestCase): tracklist where tracks have the specified `positions`.""" tracks = [self._make_track('TITLE%s' % i, position) for (i, position) in enumerate(positions, start=1)] - release = self._make_release(tracks) return self._make_release(tracks) def test_parse_media_for_tracks(self): From 55e003a3d4371acec2f04a34378494b8cb4eac38 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Jun 2019 09:11:38 +0200 Subject: [PATCH 068/613] fixing per review comments --- beetsplug/discogs.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index e4c82cd14..69b3746f7 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -303,13 +303,7 @@ class DiscogsPlugin(BeetsPlugin): mediums = [t.medium for t in tracks] country = result.data.get('country') data_url = result.data.get('uri') - style = result.data.get('styles') - if style is None: - self._log.debug('Style not Found') - elif not style: - return style - else: - style = self.config['separator'].as_str().join(sorted(style)) + style = self.format_style(result.data.get('styles')) # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. @@ -354,6 +348,15 @@ class DiscogsPlugin(BeetsPlugin): original_day=None, data_source='Discogs', data_url=data_url) + def format_style(self, style): + if style is None: + self._log.debug('Style not Found') + elif not style: + return style + else: + return self.config['separator'].as_str().join(sorted(style)) + + def get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main artist) for a list of discogs album or track artists. From 9bdadc5c73cfeebe072e5459f855e55ff6c9b070 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Jun 2019 09:33:10 +0200 Subject: [PATCH 069/613] removing extra line --- beetsplug/discogs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 69b3746f7..bdc2154de 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -356,7 +356,6 @@ class DiscogsPlugin(BeetsPlugin): else: return self.config['separator'].as_str().join(sorted(style)) - def get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main artist) for a list of discogs album or track artists. From 7ec1fc934be53d88e20150f9a9aa320a7a8d1cee Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Jun 2019 15:54:19 +0200 Subject: [PATCH 070/613] removing unneeded condition --- beetsplug/discogs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index bdc2154de..f6197ddba 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -351,8 +351,6 @@ class DiscogsPlugin(BeetsPlugin): def format_style(self, style): if style is None: self._log.debug('Style not Found') - elif not style: - return style else: return self.config['separator'].as_str().join(sorted(style)) From 34c28529a0da0ec8ab0100d13aeac00a1fb34cfe Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Jun 2019 16:41:41 +0200 Subject: [PATCH 071/613] Trigger From a94e4b0473fa600a9956da5da34b6fc4ce07736d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 10 Jun 2019 13:28:01 -0400 Subject: [PATCH 072/613] Changelog for #3251 --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e21168078..8f92f2140 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,9 @@ New features: which is useful for classical music. Thanks to :user:`dosoe`. :bug:`2580` :bug:`3279` +* :doc:`/plugins/discogs`: The field now collects the "style" field. + Thanks to :user:`thedevilisinthedetails`. + :bug:`2579` :bug:`3251` Fixes: From bf26e2cc7e6576e5dbc818514256ef213dce8855 Mon Sep 17 00:00:00 2001 From: Carl Suster Date: Tue, 11 Jun 2019 18:34:33 +1000 Subject: [PATCH 073/613] docs: mention mpdstats needs to be running Clarify that `mpdstats` is an MPD client and needs to be running all the time to collect statistics. See https://discourse.beets.io/t/mpdstats-requirements/796 --- docs/plugins/mpdstats.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/plugins/mpdstats.rst b/docs/plugins/mpdstats.rst index b769e7468..de9b2ca59 100644 --- a/docs/plugins/mpdstats.rst +++ b/docs/plugins/mpdstats.rst @@ -4,10 +4,14 @@ MPDStats Plugin ``mpdstats`` is a plugin for beets that collects statistics about your listening habits from `MPD`_. It collects the following information about tracks: -* play_count: The number of times you *fully* listened to this track. -* skip_count: The number of times you *skipped* this track. -* last_played: UNIX timestamp when you last played this track. -* rating: A rating based on *play_count* and *skip_count*. +* ``play_count``: The number of times you *fully* listened to this track. +* ``skip_count``: The number of times you *skipped* this track. +* ``last_played``: UNIX timestamp when you last played this track. +* ``rating``: A rating based on ``play_count`` and ``skip_count``. + +To gather these statistics it runs as an MPD client and watches the current state +of MPD. This means that ``mpdstats`` needs to be running continuously for it to +work. .. _MPD: https://www.musicpd.org/ From 7e4a80133e3ab52a4bbbbb67528f3789e8f93a52 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Tue, 11 Jun 2019 13:47:47 +0200 Subject: [PATCH 074/613] docs: Add changelog entry for #3301 fix --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 264a82107..940e13ba3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,9 @@ Fixes: fixing crashes in MPD clients like mpDris2 on seek. The ``playlistid`` command now works properly in its zero-argument form. :bug:`3214` +* :doc:`/plugins/importadded`: Fixed a crash that occurred when the + ``after_write`` signal was emitted. + :bug:`3301` For plugin developers: From 90cf579ee39bbdaad467baf83e8316c84bb30525 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 16 Jun 2019 21:55:35 +0200 Subject: [PATCH 075/613] adding genre, released_date and discogs_release_id --- beets/autotag/__init__.py | 3 +++ beets/autotag/hooks.py | 11 +++++++---- beets/library.py | 7 +++++++ beetsplug/beatport.py | 1 + beetsplug/discogs.py | 20 +++++++++++++------- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 48901f425..5d116c7f2 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -156,6 +156,9 @@ def apply_metadata(album_info, mapping): 'language', 'country', 'style', + 'genre', + 'discogs_release_id', + 'released_date', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index b8e6108b4..55ee033d4 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -79,10 +79,10 @@ class AlbumInfo(object): albumtype=None, va=False, year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, style=None, albumstatus=None, + language=None, country=None, style=None, genre=None, albumstatus=None, media=None, albumdisambig=None, releasegroupdisambig=None, artist_credit=None, original_year=None, original_month=None, - original_day=None, data_source=None, data_url=None): + original_day=None, data_source=None, data_url=None, discogs_release_id=None, released_date=None): self.album = album self.album_id = album_id self.artist = artist @@ -103,6 +103,7 @@ class AlbumInfo(object): self.language = language self.country = country self.style = style + self.genre = genre self.albumstatus = albumstatus self.media = media self.albumdisambig = albumdisambig @@ -113,6 +114,8 @@ class AlbumInfo(object): self.original_day = original_day self.data_source = data_source self.data_url = data_url + self.discogs_release_id = discogs_release_id + self.released_date = released_date # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -123,8 +126,8 @@ class AlbumInfo(object): """ for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', 'catalognum', 'script', 'language', 'country', 'style', - 'albumstatus', 'albumdisambig', 'releasegroupdisambig', - 'artist_credit', 'media']: + 'genre', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', + 'artist_credit', 'media', 'discogs_release_id', 'released_date']: value = getattr(self, fld) if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) diff --git a/beets/library.py b/beets/library.py index c7fbe48cf..713ad53cd 100644 --- a/beets/library.py +++ b/beets/library.py @@ -437,6 +437,8 @@ class Item(LibModel): 'albumartist_credit': types.STRING, 'genre': types.STRING, 'style': types.STRING, + 'discogs_release_id': types.INTEGER, + 'released_date': types.STRING, 'lyricist': types.STRING, 'composer': types.STRING, 'composer_sort': types.STRING, @@ -917,6 +919,8 @@ class Album(LibModel): 'album': types.STRING, 'genre': types.STRING, 'style': types.STRING, + 'discogs_release_id': types.INTEGER, + 'released_date': types.STRING, 'year': types.PaddedInt(4), 'month': types.PaddedInt(2), 'day': types.PaddedInt(2), @@ -962,6 +966,9 @@ class Album(LibModel): 'albumartist_credit', 'album', 'genre', + 'style', + 'discogs_release_id', + 'released_date', 'year', 'month', 'day', diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 0c25912b2..e85df87be 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -373,6 +373,7 @@ class BeatportPlugin(BeetsPlugin): return None release = self.client.get_release(match.group(2)) album = self._get_album_info(release) + print('ALBUM', album) return album def track_for_id(self, track_id): diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index f6197ddba..280bc659d 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -303,7 +303,10 @@ class DiscogsPlugin(BeetsPlugin): mediums = [t.medium for t in tracks] country = result.data.get('country') data_url = result.data.get('uri') - style = self.format_style(result.data.get('styles')) + style = self.format(result.data.get('styles')) + genre = self.format(result.data.get('genres')) + discogs_release_id = self.extract_release_id(result.data.get('uri')) + released_date = result.data.get('released') # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. @@ -341,18 +344,21 @@ class DiscogsPlugin(BeetsPlugin): day=None, label=label, mediums=len(set(mediums)), artist_sort=None, releasegroup_id=master_id, catalognum=catalogno, script=None, language=None, - country=country, style=style, + country=country, style=style, genre=genre, albumstatus=None, media=media, albumdisambig=None, artist_credit=None, original_year=original_year, original_month=None, original_day=None, data_source='Discogs', - data_url=data_url) + data_url=data_url, discogs_release_id=discogs_release_id, released_date=released_date) - def format_style(self, style): - if style is None: - self._log.debug('Style not Found') + def format(self, classification): + if classification is None: + self._log.debug('Classification not Found') else: - return self.config['separator'].as_str().join(sorted(style)) + return self.config['separator'].as_str().join(sorted(classification)) + + def extract_release_id(self, uri): + return uri.split("/")[-1] def get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main From f645400c5e9c834bddd4b0da56a8d59d19e85ceb Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Tue, 18 Jun 2019 23:17:38 +0200 Subject: [PATCH 076/613] replaygain: adapt to mediafile commit 95e569a Since commit 95e569a, mediafile takes care of the float -> Q7.8 conversion in R128 GAIN tags by itself. From `store_album_r128_gain` this conversion was already missing, remove it from `store_track_r128_gain`, too. fixes #3311 --- beetsplug/replaygain.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 8e11ee370..90d7ee236 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -901,27 +901,27 @@ class ReplayGainPlugin(BeetsPlugin): item.rg_track_gain = track_gain.gain item.rg_track_peak = track_gain.peak item.store() - - self._log.debug(u'applied track gain {0}, peak {1}', + self._log.debug(u'applied track gain {0} LU, peak {1} of FS', item.rg_track_gain, item.rg_track_peak) - def store_track_r128_gain(self, item, track_gain): - item.r128_track_gain = int(round(track_gain.gain * pow(2, 8))) - item.store() - - self._log.debug(u'applied r128 track gain {0}', item.r128_track_gain) - def store_album_gain(self, item, album_gain): item.rg_album_gain = album_gain.gain item.rg_album_peak = album_gain.peak item.store() - self._log.debug(u'applied album gain {0}, peak {1}', + self._log.debug(u'applied album gain {0} LU, peak {1} of FS', item.rg_album_gain, item.rg_album_peak) + def store_track_r128_gain(self, item, track_gain): + item.r128_track_gain = track_gain.gain + item.store() + + self._log.debug(u'applied r128 track gain {0} LU', + item.r128_track_gain) + def store_album_r128_gain(self, item, album_gain): item.r128_album_gain = album_gain.gain item.store() - self._log.debug(u'applied r128 album gain {0}', + self._log.debug(u'applied r128 album gain {0} LU', item.r128_album_gain) def handle_album(self, album, write, force=False): From a7e2de24998c8127dd1c0317dd7dce57c7cfc601 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Thu, 20 Jun 2019 12:49:39 +0200 Subject: [PATCH 077/613] require mediafile 0.2.0 mediafile 0.2.0 includes the changes used by commit f645400. Update setup.py to reflect that new version requirement. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1078d6cc9..cfcffdbf5 100755 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', - 'mediafile>=0.1.0', + 'mediafile>=0.2.0', 'confuse>=1.0.0', ] + [ # Avoid a version of munkres incompatible with Python 3. From 299cd01437797b4c49e1354b191680c09c0c4af5 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Thu, 20 Jun 2019 22:14:58 +0200 Subject: [PATCH 078/613] changelog: fix storage format in R128_ALBUM_GAIN --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d5918a2d..7f1d467c2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -48,6 +48,7 @@ Fixes: * :doc:`/plugins/importadded`: Fixed a crash that occurred when the ``after_write`` signal was emitted. :bug:`3301` +* doc:`plugins/replaygain`: Fix storage format in R128_ALBUM_GAIN tags. For plugin developers: From 55c2b2912c4317736b86b731d3e56b376be4b90d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 20 Jun 2019 17:23:51 -0400 Subject: [PATCH 079/613] Refine changelog for #3314 --- docs/changelog.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7f1d467c2..efb936403 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -48,7 +48,8 @@ Fixes: * :doc:`/plugins/importadded`: Fixed a crash that occurred when the ``after_write`` signal was emitted. :bug:`3301` -* doc:`plugins/replaygain`: Fix storage format in R128_ALBUM_GAIN tags. +* :doc:`plugins/replaygain`: Fix the storage format in R128 gain tags. + :bug:`3311` :bug:`3314` For plugin developers: From 2477443e58046ab402af33309f752e1ca7db2db1 Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sat, 22 Jun 2019 17:01:43 +0200 Subject: [PATCH 080/613] Add mosaic plugin --- docs/plugins/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 7417a56b3..19d2d6659 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -286,3 +286,4 @@ Here are a few of the plugins written by the beets community: .. _beets-popularity: https://github.com/abba23/beets-popularity .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl .. _beet-summarize: https://github.com/steven-murray/beet-summarize +.. _beet-mosaic: https://github.com/SusannaMaria/beets-mosaic From 43d7446df75764452a4438f17e58b117fde5ba3b Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sat, 22 Jun 2019 17:44:18 +0200 Subject: [PATCH 081/613] Right listing of mosaic plugin --- docs/plugins/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 19d2d6659..4f3e6fbff 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -265,6 +265,8 @@ Here are a few of the plugins written by the beets community: * `beet-summarize`_ can compute lots of counts and statistics about your music library. +* `beets-mosaic`_ generates a montage of a mosiac from cover art. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -286,4 +288,4 @@ Here are a few of the plugins written by the beets community: .. _beets-popularity: https://github.com/abba23/beets-popularity .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl .. _beet-summarize: https://github.com/steven-murray/beet-summarize -.. _beet-mosaic: https://github.com/SusannaMaria/beets-mosaic +.. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic From 1643eea3f569db0b4a4d28e361b80367b2628daa Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sun, 23 Jun 2019 13:04:17 +0200 Subject: [PATCH 082/613] Parameter handling --- beetsplug/absubmit.py | 27 ++++++++++++++++++++++++++- docs/plugins/absubmit.rst | 12 ++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index d9525e1d2..e1e327283 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -55,7 +55,11 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): def __init__(self): super(AcousticBrainzSubmitPlugin, self).__init__() - self.config.add({'extractor': u''}) + self.config.add({ + 'extractor': u'', + 'force': False, + 'dry': False + }) self.extractor = self.config['extractor'].as_str() if self.extractor: @@ -98,12 +102,23 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): 'absubmit', help=u'calculate and submit AcousticBrainz analysis' ) + cmd.parser.add_option( + u'-f', u'--force', dest='force_refetch', + action='store_true', default=False, + help=u're-download data when already present' + ) + cmd.parser.add_option( + u'-d', u'--dry', dest='dry_fetch', + action='store_true', default=False, + help=u'dry run, show files which would be processed' + ) cmd.func = self.command return [cmd] def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) + self.opts=opts util.par_map(self.analyze_submit, items) def analyze_submit(self, item): @@ -113,12 +128,22 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): def _get_analysis(self, item): mbid = item['mb_trackid'] + # If file has no mbid skip it. + if not self.opts.force_refetch and not self.config['force']: + mood_str = item.get('mood_acoustic', u'') + if mood_str: + return None + if not mbid: self._log.info(u'Not analysing {}, missing ' u'musicbrainz track id.', item) return None + if self.opts.dry_fetch or self.config['dry']: + self._log.info(u'dry run - extract item: {}', item) + return None + # Temporary file to save extractor output to, extractor only works # if an output file is given. Here we use a temporary file to copy # the data into a python object and then remove the file from the diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index 3221a07b3..6caffc27b 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -20,9 +20,12 @@ Submitting Data Type:: - beet absubmit [QUERY] + beet absubmit [-f] [-d] [QUERY] -to run the analysis program and upload its results. +to run the analysis program and upload its results. By default, the command will only look for AcousticBrainz data when the tracks +doesn't already have it; the ``-f`` or ``--force`` switch makes it refetch data even +when it already exists. You can use the ``-d`` or ``--dry``swtich to check which files will be +analyzed, before you start a longer period of processing. The plugin works on music with a MusicBrainz track ID attached. The plugin will also skip music that the analysis tool doesn't support. @@ -40,6 +43,11 @@ To configure the plugin, make a ``absubmit:`` section in your configuration file Default: ``no`` - **extractor**: The absolute path to the `streaming_extractor_music`_ binary. Default: search for the program in your ``$PATH`` +- **force**: Analyse AcousticBrainz data even for tracks that already have + it. + Default: ``no``. +- **dry**: No analyse AcousticBrainz data but print out the files which would be processed + Default: ``no``. .. _streaming_extractor_music: https://acousticbrainz.org/download .. _FAQ: https://acousticbrainz.org/faq From 0242176b408dc990b829088daca482400158c20e Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sun, 23 Jun 2019 16:59:33 +0200 Subject: [PATCH 083/613] Why binary import of json? --- beetsplug/absubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index e1e327283..fcdd02701 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -160,7 +160,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): item=item, error=e ) return None - with open(filename, 'rb') as tmp_file: + with open(filename, 'r') as tmp_file: analysis = json.load(tmp_file) # Add the hash to the output. analysis['metadata']['version']['essentia_build_sha'] = \ From f99b4841dfcd9d7b75abb685183669ce21d49b6f Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sun, 23 Jun 2019 17:04:43 +0200 Subject: [PATCH 084/613] Better documentation --- docs/plugins/absubmit.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index 6caffc27b..f6e561b35 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -43,10 +43,10 @@ To configure the plugin, make a ``absubmit:`` section in your configuration file Default: ``no`` - **extractor**: The absolute path to the `streaming_extractor_music`_ binary. Default: search for the program in your ``$PATH`` -- **force**: Analyse AcousticBrainz data even for tracks that already have +- **force**: Analyze items and submit of AcousticBrainz data even for tracks that already have it. Default: ``no``. -- **dry**: No analyse AcousticBrainz data but print out the files which would be processed +- **dry**: No analyze and submit of AcousticBrainz data but print out the items which would be processed Default: ``no``. .. _streaming_extractor_music: https://acousticbrainz.org/download From 6e179d86e6fe121f4191b6d3bcc6930fe83bd86c Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sun, 23 Jun 2019 17:43:40 +0200 Subject: [PATCH 085/613] Pep8 bugfix --- beetsplug/absubmit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index fcdd02701..b5f3009ca 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -111,14 +111,14 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): u'-d', u'--dry', dest='dry_fetch', action='store_true', default=False, help=u'dry run, show files which would be processed' - ) + ) cmd.func = self.command return [cmd] def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) - self.opts=opts + self.opts = opts util.par_map(self.analyze_submit, items) def analyze_submit(self, item): @@ -141,7 +141,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): return None if self.opts.dry_fetch or self.config['dry']: - self._log.info(u'dry run - extract item: {}', item) + self._log.info(u'dry run - extract item: {}', item) return None # Temporary file to save extractor output to, extractor only works From 2bfc7723fa3e8bf6fea57e68f24de5673b0dd633 Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Sun, 23 Jun 2019 18:01:21 +0200 Subject: [PATCH 086/613] Fix findings from travis-ci --- docs/plugins/absubmit.rst | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index f6e561b35..a9cf651ce 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -7,13 +7,18 @@ The ``absubmit`` plugin lets you submit acoustic analysis results to the Installation ------------ -The ``absubmit`` plugin requires the `streaming_extractor_music`_ program to run. Its source can be found on `GitHub`_, and while it is possible to compile the extractor from source, AcousticBrainz would prefer if you used their binary (see the AcousticBrainz `FAQ`_). +The ``absubmit`` plugin requires the `streaming_extractor_music`_ program +to run. Its source can be found on `GitHub`_, and while it is possible to +compile the extractor from source, AcousticBrainz would prefer if you used +their binary (see the AcousticBrainz `FAQ`_). -The ``absubmit`` plugin also requires `requests`_, which you can install using `pip`_ by typing:: +The ``absubmit`` plugin also requires `requests`_, which you can install +using `pip`_ by typing:: pip install requests -After installing both the extractor binary and requests you can enable the plugin ``absubmit`` in your configuration (see :ref:`using-plugins`). +After installing both the extractor binary and requests you can enable +the plugin ``absubmit`` in your configuration (see :ref:`using-plugins`). Submitting Data --------------- @@ -22,10 +27,12 @@ Type:: beet absubmit [-f] [-d] [QUERY] -to run the analysis program and upload its results. By default, the command will only look for AcousticBrainz data when the tracks -doesn't already have it; the ``-f`` or ``--force`` switch makes it refetch data even -when it already exists. You can use the ``-d`` or ``--dry``swtich to check which files will be -analyzed, before you start a longer period of processing. +To run the analysis program and upload its results. By default, the +command will only look for AcousticBrainz data when the tracks +doesn't already have it; the ``-f`` or ``--force`` switch makes it refetch +data even when it already exists. You can use the ``-d`` or ``--dry`` swtich +to check which files will be analyzed, before you start a longer period +of processing. The plugin works on music with a MusicBrainz track ID attached. The plugin will also skip music that the analysis tool doesn't support. @@ -37,17 +44,20 @@ will also skip music that the analysis tool doesn't support. Configuration ------------- -To configure the plugin, make a ``absubmit:`` section in your configuration file. The available options are: +To configure the plugin, make a ``absubmit:`` section in your configuration +file. The available options are: -- **auto**: Analyze every file on import. Otherwise, you need to use the ``beet absubmit`` command explicitly. +- **auto**: Analyze every file on import. Otherwise, you need to use the + ``beet absubmit`` command explicitly. Default: ``no`` - **extractor**: The absolute path to the `streaming_extractor_music`_ binary. Default: search for the program in your ``$PATH`` -- **force**: Analyze items and submit of AcousticBrainz data even for tracks that already have - it. - Default: ``no``. -- **dry**: No analyze and submit of AcousticBrainz data but print out the items which would be processed - Default: ``no``. +- **force**: Analyze items and submit of AcousticBrainz data even for tracks + that already have it. + Default: ``no``. +- **dry**: No analyze and submit of AcousticBrainz data but print out the + items which would be processed + Default: ``no``. .. _streaming_extractor_music: https://acousticbrainz.org/download .. _FAQ: https://acousticbrainz.org/faq From f254b33c6ede8170c05925ad0e30399814758fec Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Tue, 25 Jun 2019 20:22:26 +0200 Subject: [PATCH 087/613] Findings from PR --- beetsplug/absubmit.py | 14 ++++++++------ docs/plugins/absubmit.rst | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index b5f3009ca..ececda861 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -58,9 +58,11 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): self.config.add({ 'extractor': u'', 'force': False, - 'dry': False + 'pretend': False }) + self.PROBE_FIELD = 'mood_acoustic' + self.extractor = self.config['extractor'].as_str() if self.extractor: self.extractor = util.normpath(self.extractor) @@ -108,9 +110,9 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): help=u're-download data when already present' ) cmd.parser.add_option( - u'-d', u'--dry', dest='dry_fetch', + u'-p', u'--pretend', dest='pretend_fetch', action='store_true', default=False, - help=u'dry run, show files which would be processed' + help=u'pretend to perform action, but show only files which would be processed' ) cmd.func = self.command return [cmd] @@ -131,7 +133,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): # If file has no mbid skip it. if not self.opts.force_refetch and not self.config['force']: - mood_str = item.get('mood_acoustic', u'') + mood_str = item.get(self.PROBE_FIELD, u'') if mood_str: return None @@ -140,8 +142,8 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): u'musicbrainz track id.', item) return None - if self.opts.dry_fetch or self.config['dry']: - self._log.info(u'dry run - extract item: {}', item) + if self.opts.pretend_fetch or self.config['pretend']: + self._log.info(u'pretend action - extract item: {}', item) return None # Temporary file to save extractor output to, extractor only works diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index a9cf651ce..64c77e077 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -55,8 +55,8 @@ file. The available options are: - **force**: Analyze items and submit of AcousticBrainz data even for tracks that already have it. Default: ``no``. -- **dry**: No analyze and submit of AcousticBrainz data but print out the - items which would be processed +- **pretend**: Do not analyze and submit of AcousticBrainz data but print out + the items which would be processed. Default: ``no``. .. _streaming_extractor_music: https://acousticbrainz.org/download From 932d18a838925142f7dfa287e9da16e6e7c64fd6 Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Tue, 25 Jun 2019 20:37:12 +0200 Subject: [PATCH 088/613] Pep8 error ... --- beetsplug/absubmit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index ececda861..ac3094c9b 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -112,7 +112,8 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): cmd.parser.add_option( u'-p', u'--pretend', dest='pretend_fetch', action='store_true', default=False, - help=u'pretend to perform action, but show only files which would be processed' + help=u'pretend to perform action, but show ' \ + 'only files which would be processed' ) cmd.func = self.command return [cmd] From cab97c58d4dd420a79860d7f7000397545669fe7 Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Tue, 25 Jun 2019 20:57:43 +0200 Subject: [PATCH 089/613] Pep8 drives me sometimes crazy --- beetsplug/absubmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index ac3094c9b..29ed5c3fb 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -112,8 +112,8 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): cmd.parser.add_option( u'-p', u'--pretend', dest='pretend_fetch', action='store_true', default=False, - help=u'pretend to perform action, but show ' \ - 'only files which would be processed' + help=u'pretend to perform action, but show \ +only files which would be processed' ) cmd.func = self.command return [cmd] From b20516e552fce31ef61c0457ea6af08ab7bff083 Mon Sep 17 00:00:00 2001 From: Susanna Maria Date: Wed, 26 Jun 2019 20:29:17 +0200 Subject: [PATCH 090/613] Small improvement of code doc --- beetsplug/absubmit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 29ed5c3fb..594d0dc01 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -61,6 +61,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): 'pretend': False }) + # Define a field which shows that acousticbrainz info is present self.PROBE_FIELD = 'mood_acoustic' self.extractor = self.config['extractor'].as_str() From 0726123e41b4edc31ab1a71bdeb37d6ebe703972 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 27 Jun 2019 22:43:18 -0400 Subject: [PATCH 091/613] Move PROBE_FIELD to module scope (#3318) --- beetsplug/absubmit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 594d0dc01..baff922e4 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -32,6 +32,9 @@ from beets import plugins from beets import util from beets import ui +# We use this field to check whether AcousticBrainz info is present. +PROBE_FIELD = 'mood_acoustic' + class ABSubmitError(Exception): """Raised when failing to analyse file with extractor.""" @@ -61,9 +64,6 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): 'pretend': False }) - # Define a field which shows that acousticbrainz info is present - self.PROBE_FIELD = 'mood_acoustic' - self.extractor = self.config['extractor'].as_str() if self.extractor: self.extractor = util.normpath(self.extractor) @@ -135,7 +135,7 @@ only files which would be processed' # If file has no mbid skip it. if not self.opts.force_refetch and not self.config['force']: - mood_str = item.get(self.PROBE_FIELD, u'') + mood_str = item.get(PROBE_FIELD, u'') if mood_str: return None From 73f8edd116b558e581b5469dbce12371a45aa0dd Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 27 Jun 2019 22:44:57 -0400 Subject: [PATCH 092/613] Simplify force check (#3318) --- beetsplug/absubmit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index baff922e4..69c4d2a98 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -133,12 +133,12 @@ only files which would be processed' def _get_analysis(self, item): mbid = item['mb_trackid'] - # If file has no mbid skip it. + # Avoid re-analyzing files that already have AB data. if not self.opts.force_refetch and not self.config['force']: - mood_str = item.get(PROBE_FIELD, u'') - if mood_str: + if item.get(PROBE_FIELD): return None + # If file has no MBID, skip it. if not mbid: self._log.info(u'Not analysing {}, missing ' u'musicbrainz track id.', item) From 090711eeb2ce25a52070d04990cb6ce7354def7b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 27 Jun 2019 22:49:56 -0400 Subject: [PATCH 093/613] Changelog for #3318 --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index efb936403..f55e32994 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,12 @@ New features: * :doc:`/plugins/discogs`: The field now collects the "style" field. Thanks to :user:`thedevilisinthedetails`. :bug:`2579` :bug:`3251` +* :doc:`/plugins/absubmit`: By default, the plugin now avoids re-analyzing + files that already have AB data. + There are new ``force`` and ``pretend`` options to help control this new + behavior. + Thanks to :user:`SusannaMaria`. + :bug:`3318` Fixes: From 9e5c45d6ddda999fb425328b3db5974b61a9f285 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Fri, 28 Jun 2019 14:47:38 +0200 Subject: [PATCH 094/613] Convert plugin skips previously-converted files Clarifies documentation so it's clear that one can run `beet convert` without fearing that previously-converted files will be needlessly re-converted. --- docs/plugins/convert.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 59670c269..f80d10233 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -24,7 +24,9 @@ To convert a part of your collection, run ``beet convert QUERY``. The command will transcode all the files matching the query to the destination directory given by the ``-d`` (``--dest``) option or the ``dest`` configuration. The path layout mirrors that of your library, -but it may be customized through the ``paths`` configuration. +but it may be customized through the ``paths`` configuration. Files +that have been previously converted — and thus already exist in the +destination directory — will be skipped. The plugin uses a command-line program to transcode the audio. With the ``-f`` (``--format``) option you can choose the transcoding command From 23c9f87142125c82b5cd2c6f731a9317f61729d3 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 28 Jun 2019 09:24:15 -0400 Subject: [PATCH 095/613] Typography (#3319) --- docs/plugins/convert.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index f80d10233..72ea301f1 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -25,8 +25,8 @@ command will transcode all the files matching the query to the destination directory given by the ``-d`` (``--dest``) option or the ``dest`` configuration. The path layout mirrors that of your library, but it may be customized through the ``paths`` configuration. Files -that have been previously converted — and thus already exist in the -destination directory — will be skipped. +that have been previously converted---and thus already exist in the +destination directory---will be skipped. The plugin uses a command-line program to transcode the audio. With the ``-f`` (``--format``) option you can choose the transcoding command From e9dd226b93bc59afdacfc3fb89fa8f9f4d9c9cc2 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 12:06:38 +0200 Subject: [PATCH 096/613] fixing test --- test/test_discogs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_discogs.py b/test/test_discogs.py index 0acf54e8a..693768916 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -48,6 +48,9 @@ class DGAlbumInfoTest(_common.TestCase): 'styles': [ 'STYLE1', 'STYLE2' ], + 'genres': [ + 'GENRE1', 'GENRE2' + ], 'labels': [{ 'name': 'LABEL NAME', 'catno': 'CATALOG NUMBER', From 0e2e2dfec33138557490db257d5230af5d87023e Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 12:22:22 +0200 Subject: [PATCH 097/613] adding additional discogs attrributes --- test/test_discogs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_discogs.py b/test/test_discogs.py index 693768916..d97f172b9 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -35,6 +35,7 @@ class DGAlbumInfoTest(_common.TestCase): 'uri': 'ALBUM URI', 'title': 'ALBUM TITLE', 'year': '3001', + 'released': '2019-06-07', 'artists': [{ 'name': 'ARTIST NAME', 'id': 'ARTIST ID', From 8bf9d75f668dee13fed39fb08768656aace776fd Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 12:50:36 +0200 Subject: [PATCH 098/613] fixing test --- test/test_discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_discogs.py b/test/test_discogs.py index d97f172b9..6b8fdb8d8 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -32,7 +32,7 @@ class DGAlbumInfoTest(_common.TestCase): those required for the tests on this class.""" data = { 'id': 'ALBUM ID', - 'uri': 'ALBUM URI', + 'uri': 'https://www.discogs.com/release/release/13633721', 'title': 'ALBUM TITLE', 'year': '3001', 'released': '2019-06-07', From 510276f653e4dbbc1e05c2369b08632a88d21103 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 13:44:13 +0200 Subject: [PATCH 099/613] fixing test --- beets/autotag/hooks.py | 15 +++++++++------ beetsplug/discogs.py | 7 +++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 55ee033d4..fa4e0d73f 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -79,10 +79,12 @@ class AlbumInfo(object): albumtype=None, va=False, year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, style=None, genre=None, albumstatus=None, - media=None, albumdisambig=None, releasegroupdisambig=None, - artist_credit=None, original_year=None, original_month=None, - original_day=None, data_source=None, data_url=None, discogs_release_id=None, released_date=None): + language=None, country=None, style=None, genre=None, + albumstatus=None, media=None, albumdisambig=None, + releasegroupdisambig=None, artist_credit=None, + original_year=None, original_month=None, + original_day=None, data_source=None, data_url=None, + discogs_release_id=None, released_date=None): self.album = album self.album_id = album_id self.artist = artist @@ -126,8 +128,9 @@ class AlbumInfo(object): """ for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', 'catalognum', 'script', 'language', 'country', 'style', - 'genre', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', - 'artist_credit', 'media', 'discogs_release_id', 'released_date']: + 'genre', 'albumstatus', 'albumdisambig', + 'releasegroupdisambig', 'artist_credit', + 'media', 'discogs_release_id', 'released_date']: value = getattr(self, fld) if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 280bc659d..2469b85e5 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -349,7 +349,9 @@ class DiscogsPlugin(BeetsPlugin): albumdisambig=None, artist_credit=None, original_year=original_year, original_month=None, original_day=None, data_source='Discogs', - data_url=data_url, discogs_release_id=discogs_release_id, released_date=released_date) + data_url=data_url, + discogs_release_id=discogs_release_id, + released_date=released_date) def format(self, classification): if classification is None: @@ -358,7 +360,8 @@ class DiscogsPlugin(BeetsPlugin): return self.config['separator'].as_str().join(sorted(classification)) def extract_release_id(self, uri): - return uri.split("/")[-1] + if uri: + return uri.split("/")[-1] def get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main From e196c1dae6311dac44c8aece63ddaae4a8c5bdc8 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 13:54:38 +0200 Subject: [PATCH 100/613] fixing test --- beetsplug/discogs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 2469b85e5..4daa82fbf 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -357,7 +357,8 @@ class DiscogsPlugin(BeetsPlugin): if classification is None: self._log.debug('Classification not Found') else: - return self.config['separator'].as_str().join(sorted(classification)) + return self.config['separator'].as_str()\ + .join(sorted(classification)) def extract_release_id(self, uri): if uri: From 6ae73546e522fedf68c35d29f69f6ebfe16a3c90 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 14:09:31 +0200 Subject: [PATCH 101/613] Trigger From dd7e932a9a6b2eeaa4590c11d9c0e7ec381e5bc0 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 30 Jun 2019 14:34:13 +0200 Subject: [PATCH 102/613] removing print log --- beetsplug/beatport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index e85df87be..0c25912b2 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -373,7 +373,6 @@ class BeatportPlugin(BeetsPlugin): return None release = self.client.get_release(match.group(2)) album = self._get_album_info(release) - print('ALBUM', album) return album def track_for_id(self, track_id): From 0cd46dab770f8ae96e70760cec6393d175eaecbd Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 1 Jul 2019 21:04:35 +0200 Subject: [PATCH 103/613] fixing per review comments --- beets/autotag/__init__.py | 4 ++-- beets/autotag/hooks.py | 7 +++---- beets/library.py | 9 +++------ beetsplug/discogs.py | 16 ++++++++-------- test/test_discogs.py | 1 - 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 5d116c7f2..b0bbbbf04 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -157,8 +157,8 @@ def apply_metadata(album_info, mapping): 'country', 'style', 'genre', - 'discogs_release_id', - 'released_date', + 'discogs_albumid', + '', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index fa4e0d73f..b946ff7c9 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -84,7 +84,7 @@ class AlbumInfo(object): releasegroupdisambig=None, artist_credit=None, original_year=None, original_month=None, original_day=None, data_source=None, data_url=None, - discogs_release_id=None, released_date=None): + discogs_albumid=None): self.album = album self.album_id = album_id self.artist = artist @@ -116,8 +116,7 @@ class AlbumInfo(object): self.original_day = original_day self.data_source = data_source self.data_url = data_url - self.discogs_release_id = discogs_release_id - self.released_date = released_date + self.discogs_albumid = discogs_albumid # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -130,7 +129,7 @@ class AlbumInfo(object): 'catalognum', 'script', 'language', 'country', 'style', 'genre', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', 'artist_credit', - 'media', 'discogs_release_id', 'released_date']: + 'media', 'discogs_albumid']: value = getattr(self, fld) if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) diff --git a/beets/library.py b/beets/library.py index 713ad53cd..12850f933 100644 --- a/beets/library.py +++ b/beets/library.py @@ -437,8 +437,7 @@ class Item(LibModel): 'albumartist_credit': types.STRING, 'genre': types.STRING, 'style': types.STRING, - 'discogs_release_id': types.INTEGER, - 'released_date': types.STRING, + 'discogs_albumid': types.INTEGER, 'lyricist': types.STRING, 'composer': types.STRING, 'composer_sort': types.STRING, @@ -919,8 +918,7 @@ class Album(LibModel): 'album': types.STRING, 'genre': types.STRING, 'style': types.STRING, - 'discogs_release_id': types.INTEGER, - 'released_date': types.STRING, + 'discogs_albumid': types.INTEGER, 'year': types.PaddedInt(4), 'month': types.PaddedInt(2), 'day': types.PaddedInt(2), @@ -967,8 +965,7 @@ class Album(LibModel): 'album', 'genre', 'style', - 'discogs_release_id', - 'released_date', + 'discogs_albumid', 'year', 'month', 'day', diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 4daa82fbf..92a5773f5 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -305,8 +305,7 @@ class DiscogsPlugin(BeetsPlugin): data_url = result.data.get('uri') style = self.format(result.data.get('styles')) genre = self.format(result.data.get('genres')) - discogs_release_id = self.extract_release_id(result.data.get('uri')) - released_date = result.data.get('released') + discogs_albumid = self.extract_release_id(result.data.get('uri')) # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. @@ -350,19 +349,20 @@ class DiscogsPlugin(BeetsPlugin): original_year=original_year, original_month=None, original_day=None, data_source='Discogs', data_url=data_url, - discogs_release_id=discogs_release_id, - released_date=released_date) + discogs_albumid=discogs_albumid) def format(self, classification): - if classification is None: - self._log.debug('Classification not Found') - else: - return self.config['separator'].as_str()\ + if classification: + return self.config['separator'].as_str() \ .join(sorted(classification)) + else: + return None def extract_release_id(self, uri): if uri: return uri.split("/")[-1] + else: + return None def get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main diff --git a/test/test_discogs.py b/test/test_discogs.py index 6b8fdb8d8..61d9d5aa1 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -35,7 +35,6 @@ class DGAlbumInfoTest(_common.TestCase): 'uri': 'https://www.discogs.com/release/release/13633721', 'title': 'ALBUM TITLE', 'year': '3001', - 'released': '2019-06-07', 'artists': [{ 'name': 'ARTIST NAME', 'id': 'ARTIST ID', From ad67d708182a95f74c57372d2055135d8b5bcd38 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 1 Jul 2019 21:05:31 +0200 Subject: [PATCH 104/613] removing empty string --- beets/autotag/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index b0bbbbf04..2c8a2f984 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -158,7 +158,6 @@ def apply_metadata(album_info, mapping): 'style', 'genre', 'discogs_albumid', - '', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', From 1b4124931b72365b147a56bb97d07aee9ff3279a Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 1 Jul 2019 21:17:19 +0200 Subject: [PATCH 105/613] Trigger From ca4b101ef47c9b613865c0e6c31ea582f3e9339b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 1 Jul 2019 17:24:12 -0400 Subject: [PATCH 106/613] Changelog for #3322 (fix #465) --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f55e32994..c9fbc5fec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,10 @@ New features: behavior. Thanks to :user:`SusannaMaria`. :bug:`3318` +* :doc:`/plugins/discogs`: The plugin now also gets genre information and a + new ``discogs_albumid`` field from the Discogs API. + Thanks to :user:`thedevilisinthedetails`. + :bug:`465` :bug:`3322` Fixes: From 0d44d31913edef1cddaac5461bc33a2c975ec01d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 5 Jul 2019 09:09:33 -0400 Subject: [PATCH 107/613] Travis: re-enable Python 3.8 The Werkzeug issue on 3.8 seems to have been fixed: https://github.com/pallets/werkzeug/issues/1551 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 455ab4ca4..0ebd330a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ matrix: - python: 3.7 env: {TOX_ENV: py37-test} dist: xenial - # - python: 3.8-dev - # env: {TOX_ENV: py38-test} - # dist: xenial + - python: 3.8-dev + env: {TOX_ENV: py38-test} + dist: xenial # - python: pypy # - env: {TOX_ENV: pypy-test} - python: 3.6 From f39cff71d3144d4355acae2fa364f717d7214559 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 5 Jul 2019 09:49:53 -0400 Subject: [PATCH 108/613] Revert "Travis: re-enable Python 3.8" This reverts commit 0d44d31913edef1cddaac5461bc33a2c975ec01d. Apparently, 0.15.5 (which contains the 3.8 fix) is not actually released yet. Oops! --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ebd330a0..455ab4ca4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ matrix: - python: 3.7 env: {TOX_ENV: py37-test} dist: xenial - - python: 3.8-dev - env: {TOX_ENV: py38-test} - dist: xenial + # - python: 3.8-dev + # env: {TOX_ENV: py38-test} + # dist: xenial # - python: pypy # - env: {TOX_ENV: pypy-test} - python: 3.6 From 30395911e24e653146d731a045a7586d1ffc45bb Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Thu, 18 Oct 2018 20:34:58 +0200 Subject: [PATCH 109/613] util.command_output: return stderr, too Return a namedtuple CommandOutput(stdout, stderr) instead of just stdout from util.command_ouput, allowing separate access to stdout and stderr. This change is required by the ffmpeg replaygain backend (GitHub PullRequest #3056) as ffmpeg's ebur128 filter outputs only to stderr. --- beets/util/__init__.py | 16 +++++++++++++--- beets/util/artresizer.py | 4 ++-- beetsplug/absubmit.py | 2 +- beetsplug/duplicates.py | 2 +- beetsplug/ipfs.py | 4 ++-- beetsplug/keyfinder.py | 2 +- beetsplug/replaygain.py | 8 ++++---- test/test_keyfinder.py | 8 ++++---- test/test_replaygain.py | 6 +++--- 9 files changed, 31 insertions(+), 21 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 162502eb1..b23832c6d 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -24,7 +24,7 @@ import re import shutil import fnmatch import functools -from collections import Counter +from collections import Counter, namedtuple from multiprocessing.pool import ThreadPool import traceback import subprocess @@ -763,7 +763,11 @@ def cpu_count(): num = 0 elif sys.platform == 'darwin': try: - num = int(command_output(['/usr/sbin/sysctl', '-n', 'hw.ncpu'])) + num = int(command_output([ + '/usr/sbin/sysctl', + '-n', + 'hw.ncpu', + ]).stdout) except (ValueError, OSError, subprocess.CalledProcessError): num = 0 else: @@ -794,9 +798,15 @@ def convert_command_args(args): return [convert(a) for a in args] +# stdout and stderr as bytes +CommandOutput = namedtuple("CommandOutput", ("stdout", "stderr")) + + def command_output(cmd, shell=False): """Runs the command and returns its output after it has exited. + Returns a CommandOutput. + ``cmd`` is a list of arguments starting with the command names. The arguments are bytes on Unix and strings on Windows. If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a @@ -831,7 +841,7 @@ def command_output(cmd, shell=False): cmd=' '.join(cmd), output=stdout + stderr, ) - return stdout + return CommandOutput(stdout, stderr) def max_filename_length(path, limit=MAX_FILENAME_LENGTH): diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 1ee3e560d..99e28c0cc 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -129,7 +129,7 @@ def im_getsize(path_in): ['-format', '%w %h', util.syspath(path_in, prefix=False)] try: - out = util.command_output(cmd) + out = util.command_output(cmd).stdout except subprocess.CalledProcessError as exc: log.warning(u'ImageMagick size query failed') log.debug( @@ -265,7 +265,7 @@ def get_im_version(): cmd = cmd_name + ['--version'] try: - out = util.command_output(cmd) + out = util.command_output(cmd).stdout except (subprocess.CalledProcessError, OSError) as exc: log.debug(u'ImageMagick version check failed: {}', exc) else: diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 69c4d2a98..7419736a3 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -46,7 +46,7 @@ def call(args): Raise a AnalysisABSubmitError on failure. """ try: - return util.command_output(args) + return util.command_output(args).stdout except subprocess.CalledProcessError as e: raise ABSubmitError( u'{0} exited with status {1}'.format(args[0], e.returncode) diff --git a/beetsplug/duplicates.py b/beetsplug/duplicates.py index b316cfda6..4e6e540ea 100644 --- a/beetsplug/duplicates.py +++ b/beetsplug/duplicates.py @@ -205,7 +205,7 @@ class DuplicatesPlugin(BeetsPlugin): u'computing checksum', key, displayable_path(item.path)) try: - checksum = command_output(args) + checksum = command_output(args).stdout setattr(item, key, checksum) item.store() self._log.debug(u'computed checksum for {0} using {1}', diff --git a/beetsplug/ipfs.py b/beetsplug/ipfs.py index 90ba5fdd0..f2408c259 100644 --- a/beetsplug/ipfs.py +++ b/beetsplug/ipfs.py @@ -123,7 +123,7 @@ class IPFSPlugin(BeetsPlugin): cmd = "ipfs add -q -r".split() cmd.append(album_dir) try: - output = util.command_output(cmd).split() + output = util.command_output(cmd).stdout.split() except (OSError, subprocess.CalledProcessError) as exc: self._log.error(u'Failed to add {0}, error: {1}', album_dir, exc) return False @@ -183,7 +183,7 @@ class IPFSPlugin(BeetsPlugin): else: cmd = "ipfs add -q ".split() cmd.append(tmp.name) - output = util.command_output(cmd) + output = util.command_output(cmd).stdout except (OSError, subprocess.CalledProcessError) as err: msg = "Failed to publish library. Error: {0}".format(err) self._log.error(msg) diff --git a/beetsplug/keyfinder.py b/beetsplug/keyfinder.py index 3a738478e..ea928ef43 100644 --- a/beetsplug/keyfinder.py +++ b/beetsplug/keyfinder.py @@ -60,7 +60,7 @@ class KeyFinderPlugin(BeetsPlugin): try: output = util.command_output([bin, '-f', - util.syspath(item.path)]) + util.syspath(item.path)]).stdout except (subprocess.CalledProcessError, OSError) as exc: self._log.error(u'execution failed: {0}', exc) continue diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 90d7ee236..58a4df83c 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -47,12 +47,12 @@ class FatalGstreamerPluginReplayGainError(FatalReplayGainError): loading the required plugins.""" -def call(args): +def call(args, **kwargs): """Execute the command and return its output or raise a ReplayGainError on failure. """ try: - return command_output(args) + return command_output(args, **kwargs) except subprocess.CalledProcessError as e: raise ReplayGainError( u"{0} exited with status {1}".format(args[0], e.returncode) @@ -206,7 +206,7 @@ class Bs1770gainBackend(Backend): self._log.debug( u'executing {0}', u' '.join(map(displayable_path, args)) ) - output = call(args) + output = call(args).stdout self._log.debug(u'analysis finished: {0}', output) results = self.parse_tool_output(output, path_list, is_album) @@ -378,7 +378,7 @@ class CommandBackend(Backend): self._log.debug(u'analyzing {0} files', len(items)) self._log.debug(u"executing {0}", " ".join(map(displayable_path, cmd))) - output = call(cmd) + output = call(cmd).stdout self._log.debug(u'analysis finished') return self.parse_tool_output(output, len(items) + (1 if is_album else 0)) diff --git a/test/test_keyfinder.py b/test/test_keyfinder.py index c2b4227d7..a9ac43a27 100644 --- a/test/test_keyfinder.py +++ b/test/test_keyfinder.py @@ -38,7 +38,7 @@ class KeyFinderTest(unittest.TestCase, TestHelper): item = Item(path='/file') item.add(self.lib) - command_output.return_value = 'dbm' + command_output.return_value = util.CommandOutput(b"dbm", b"") self.run_command('keyfinder') item.load() @@ -47,7 +47,7 @@ class KeyFinderTest(unittest.TestCase, TestHelper): ['KeyFinder', '-f', util.syspath(item.path)]) def test_add_key_on_import(self, command_output): - command_output.return_value = 'dbm' + command_output.return_value = util.CommandOutput(b"dbm", b"") importer = self.create_importer() importer.run() @@ -60,7 +60,7 @@ class KeyFinderTest(unittest.TestCase, TestHelper): item = Item(path='/file', initial_key='F') item.add(self.lib) - command_output.return_value = 'C#m' + command_output.return_value = util.CommandOutput(b"C#m", b"") self.run_command('keyfinder') item.load() @@ -70,7 +70,7 @@ class KeyFinderTest(unittest.TestCase, TestHelper): item = Item(path='/file', initial_key='F') item.add(self.lib) - command_output.return_value = 'dbm' + command_output.return_value = util.CommandOutput(b"dbm", b"") self.run_command('keyfinder') item.load() diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 9f14374cc..b482da14e 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -23,6 +23,7 @@ from mock import patch from test.helper import TestHelper, capture_log, has_program from beets import config +from beets.util import CommandOutput from mediafile import MediaFile from beetsplug.replaygain import (FatalGstreamerPluginReplayGainError, GStreamerBackend) @@ -169,7 +170,6 @@ class ReplayGainLdnsCliTest(ReplayGainCliTestBase, unittest.TestCase): class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): - @patch('beetsplug.replaygain.call') def setUp(self, call_patch): self.setup_beets() @@ -186,14 +186,14 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): @patch('beetsplug.replaygain.call') def test_malformed_output(self, call_patch): # Return malformed XML (the ampersand should be &) - call_patch.return_value = """ + call_patch.return_value = CommandOutput(stdout=""" - """ + """, stderr="") with capture_log('beets.replaygain') as logs: self.run_command('replaygain') From 45aa75a4ef5426aa4961ee0d6a076c42d6ab0cf1 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Mon, 15 Jul 2019 11:39:30 +0200 Subject: [PATCH 110/613] document util.command_output return value change After commit `30395911 util.command_output: return stderr, too`, `command_output` returns a tuple of stdout and stderr. Document that change by adding a changelog entry and add a usage note to `command_output`'s docstring. --- beets/util/__init__.py | 3 ++- docs/changelog.rst | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index b23832c6d..29b2a73e7 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -805,7 +805,8 @@ CommandOutput = namedtuple("CommandOutput", ("stdout", "stderr")) def command_output(cmd, shell=False): """Runs the command and returns its output after it has exited. - Returns a CommandOutput. + Returns a CommandOutput. The attributes ``stdout`` and ``stderr`` contain + byte strings of the respective output streams. ``cmd`` is a list of arguments starting with the command names. The arguments are bytes on Unix and strings on Windows. diff --git a/docs/changelog.rst b/docs/changelog.rst index c9fbc5fec..e0e19101b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -74,6 +74,9 @@ For plugin developers: is almost identical apart from the name change. Again, we'll re-export at the old location (with a deprecation warning) for backwards compatibility, but might stop doing this in a future release. +* ``beets.util.command_output`` now returns a named tuple of stdout and stderr + instead of only returning stdout. stdout can be accessed as a simple + attribute. For packagers: From f835cca2357c1b76cf623c0ec7da52a9f87695ff Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 15 Jul 2019 09:51:18 -0400 Subject: [PATCH 111/613] Expand changelog for #3329 --- docs/changelog.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e0e19101b..aa544bcac 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -74,9 +74,12 @@ For plugin developers: is almost identical apart from the name change. Again, we'll re-export at the old location (with a deprecation warning) for backwards compatibility, but might stop doing this in a future release. -* ``beets.util.command_output`` now returns a named tuple of stdout and stderr - instead of only returning stdout. stdout can be accessed as a simple - attribute. +* ``beets.util.command_output`` now returns a named tuple containing both the + standard output and the standard error data instead of just stdout alone. + Client code will need to access the ``stdout`` attribute on the return + value. + Thanks to :user:`zsinskri`. + :bug:`3329` For packagers: From 0ebab5edaa9511a5cc5730d2650de34f64a758b0 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Mon, 15 Jul 2019 21:25:39 +0200 Subject: [PATCH 112/613] fix "Sporadic test failures in BPD tests #3309" The bpd test bind a socket in order to test the protocol implementation. When running concurrently this often resulted in an attempt to bind an already occupied port. By using the port number `0` we instead let the OS choose a free port. We then have to extract it from the socket (which is handled by `bluelet`) via `mock.patch`ing. --- test/test_player.py | 68 ++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 959d77eb3..036593bff 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -29,9 +29,8 @@ import time import yaml import tempfile from contextlib import contextmanager -import random -from beets.util import py3_path +from beets.util import py3_path, bluelet from beetsplug import bpd import confuse @@ -231,11 +230,6 @@ class MPCClient(object): return line -def start_beets(*args): - import beets.ui - beets.ui.main(list(args)) - - def implements(commands, expectedFailure=False): # noqa: N803 def _test(self): with self.run_bpd() as client: @@ -263,22 +257,18 @@ class BPDTestHelper(unittest.TestCase, TestHelper): self.unload_plugins() @contextmanager - def run_bpd(self, host='localhost', port=None, password=None, - do_hello=True, second_client=False): + def run_bpd(self, host='localhost', password=None, do_hello=True, + second_client=False): """ Runs BPD in another process, configured with the same library database as we created in the setUp method. Exposes a client that is connected to the server, and kills the server at the end. """ - # Choose a port (randomly) to avoid conflicts between parallel - # tests. - if not port: - port = 9876 + random.randint(0, 10000) - # Create a config file: config = { 'pluginpath': [py3_path(self.temp_dir)], 'plugins': 'bpd', - 'bpd': {'host': host, 'port': port, 'control_port': port + 1}, + # use port 0 to let the OS choose a free port + 'bpd': {'host': host, 'port': 0, 'control_port': 0}, } if password: config['bpd']['password'] = password @@ -289,28 +279,50 @@ class BPDTestHelper(unittest.TestCase, TestHelper): yaml.dump(config, Dumper=confuse.Dumper, encoding='utf-8')) config_file.close() + bluelet_listener = bluelet.Listener + @mock.patch("beets.util.bluelet.Listener") + def start_server(assigned_port, listener_patch): + """Start the bpd server, writing the port to `assigned_port`. + """ + def listener_wrap(host, port): + """Wrap `bluelet.Listener`, writing the port to `assigend_port`. + + `bluelet.Listener` has previously been saved to + `bluelet_listener` as this function will replace it at its + original location. + """ + listener = bluelet_listener(host, port) + assigned_port.value = listener.sock.getsockname()[1] + return listener + listener_patch.side_effect = listener_wrap + + import beets.ui + beets.ui.main([ + '--library', self.config['library'].as_filename(), + '--directory', py3_path(self.libdir), + '--config', py3_path(config_file.name), + 'bpd' + ]) + # Fork and launch BPD in the new process: - args = ( - '--library', self.config['library'].as_filename(), - '--directory', py3_path(self.libdir), - '--config', py3_path(config_file.name), - 'bpd' - ) - server = mp.Process(target=start_beets, args=args) + assigned_port = mp.Value("I", 0) + server = mp.Process(target=start_server, args=(assigned_port,)) server.start() # Wait until the socket is connected: - sock, sock2 = None, None for _ in range(20): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex((host, port)) == 0: + if assigned_port.value != 0: + # read which port has been assigned by the OS + port = assigned_port.value break - else: - sock.close() - time.sleep(0.01) + time.sleep(0.01) else: raise RuntimeError('Timed out waiting for the BPD server') + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + + sock2 = None try: if second_client: sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) From bbda292145d3c9758a4e7a849d822b3283e3b524 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Mon, 15 Jul 2019 22:25:48 +0200 Subject: [PATCH 113/613] bpd test: make `start_server` a freestanding function Under some circumstances (maybe under MS Windows?) local objects can't be pickled. When `start_server` is a local this causes a crash: https://ci.appveyor.com/project/beetbox/beets/builds/25996163/job/rbp3frnkwsvbuwx6#L541 Make `start_server` a freestanding function to mitigate this. --- test/test_player.py | 53 +++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 036593bff..37d0b6640 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -240,6 +240,27 @@ def implements(commands, expectedFailure=False): # noqa: N803 return unittest.expectedFailure(_test) if expectedFailure else _test +bluelet_listener = bluelet.Listener +@mock.patch("beets.util.bluelet.Listener") +def start_server(args, assigned_port, listener_patch): + """Start the bpd server, writing the port to `assigned_port`. + """ + def listener_wrap(host, port): + """Wrap `bluelet.Listener`, writing the port to `assigend_port`. + + `bluelet.Listener` has previously been saved to + `bluelet_listener` as this function will replace it at its + original location. + """ + listener = bluelet_listener(host, port) + assigned_port.value = listener.sock.getsockname()[1] + return listener + listener_patch.side_effect = listener_wrap + + import beets.ui + beets.ui.main(args) + + class BPDTestHelper(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets(disk=True) @@ -279,34 +300,14 @@ class BPDTestHelper(unittest.TestCase, TestHelper): yaml.dump(config, Dumper=confuse.Dumper, encoding='utf-8')) config_file.close() - bluelet_listener = bluelet.Listener - @mock.patch("beets.util.bluelet.Listener") - def start_server(assigned_port, listener_patch): - """Start the bpd server, writing the port to `assigned_port`. - """ - def listener_wrap(host, port): - """Wrap `bluelet.Listener`, writing the port to `assigend_port`. - - `bluelet.Listener` has previously been saved to - `bluelet_listener` as this function will replace it at its - original location. - """ - listener = bluelet_listener(host, port) - assigned_port.value = listener.sock.getsockname()[1] - return listener - listener_patch.side_effect = listener_wrap - - import beets.ui - beets.ui.main([ - '--library', self.config['library'].as_filename(), - '--directory', py3_path(self.libdir), - '--config', py3_path(config_file.name), - 'bpd' - ]) - # Fork and launch BPD in the new process: assigned_port = mp.Value("I", 0) - server = mp.Process(target=start_server, args=(assigned_port,)) + server = mp.Process(target=start_server, args=([ + '--library', self.config['library'].as_filename(), + '--directory', py3_path(self.libdir), + '--config', py3_path(config_file.name), + 'bpd' + ], assigned_port)) server.start() # Wait until the socket is connected: From fb07a5112a8284a533ffb8ec1242d5e23b3eafd4 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Jul 2019 13:16:01 +0200 Subject: [PATCH 114/613] bpd tests: terminate server upon connection failure --- test/test_player.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 37d0b6640..77ebef7d4 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -310,21 +310,23 @@ class BPDTestHelper(unittest.TestCase, TestHelper): ], assigned_port)) server.start() - # Wait until the socket is connected: - for _ in range(20): - if assigned_port.value != 0: - # read which port has been assigned by the OS - port = assigned_port.value - break - time.sleep(0.01) - else: - raise RuntimeError('Timed out waiting for the BPD server') - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - - sock2 = None try: + # Wait until the socket is connected: + for _ in range(20): + if assigned_port.value != 0: + # read which port has been assigned by the OS + port = assigned_port.value + break + time.sleep(0.01) + else: + raise RuntimeError( + 'Timed out waiting for the BPD server to start' + ) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + + sock2 = None if second_client: sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock2.connect((host, port)) From 871f79c8f2dee43b313142ed9ddc2a414252fe9a Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Jul 2019 13:38:57 +0200 Subject: [PATCH 115/613] bpd tests: close only existing sockets Close sockets in `finally`-clauses only after they have actually been created. --- test/test_player.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 77ebef7d4..b9d35281b 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -324,19 +324,25 @@ class BPDTestHelper(unittest.TestCase, TestHelper): ) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) + try: + sock.connect((host, port)) - sock2 = None - if second_client: - sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock2.connect((host, port)) - yield MPCClient(sock, do_hello), MPCClient(sock2, do_hello) - else: - yield MPCClient(sock, do_hello) + if second_client: + sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock2.connect((host, port)) + yield ( + MPCClient(sock, do_hello), + MPCClient(sock2, do_hello), + ) + finally: + sock2.close() + + else: + yield MPCClient(sock, do_hello) + finally: + sock.close() finally: - sock.close() - if sock2: - sock2.close() server.terminate() server.join(timeout=0.2) From 8ba79117d936870aff39c42a909348d7f254d7cd Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Jul 2019 16:08:46 +0200 Subject: [PATCH 116/613] bpd tests: use mp.Queue to communicate assigned port Use a `multiprocessing.Queue` instead of a `multiprocessing.Value` to avoid the manual polling/timeout handling. TODO: Strangely Listener seems to be constructed twice. Only the second one is used. Fix that and then remove the code working around it. --- test/test_player.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index b9d35281b..78e3948ca 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -247,13 +247,14 @@ def start_server(args, assigned_port, listener_patch): """ def listener_wrap(host, port): """Wrap `bluelet.Listener`, writing the port to `assigend_port`. - - `bluelet.Listener` has previously been saved to - `bluelet_listener` as this function will replace it at its - original location. """ + # `bluelet.Listener` has previously been saved to + # `bluelet_listener` as this function will replace it at its + # original location. listener = bluelet_listener(host, port) - assigned_port.value = listener.sock.getsockname()[1] + # read which port has been assigned by the OS + # TODO: change to put_nowait. There should always be a free slot. + assigned_port.put(listener.sock.getsockname()[1]) return listener listener_patch.side_effect = listener_wrap @@ -301,7 +302,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): config_file.close() # Fork and launch BPD in the new process: - assigned_port = mp.Value("I", 0) + assigned_port = mp.Queue(1) server = mp.Process(target=start_server, args=([ '--library', self.config['library'].as_filename(), '--directory', py3_path(self.libdir), @@ -311,17 +312,12 @@ class BPDTestHelper(unittest.TestCase, TestHelper): server.start() try: - # Wait until the socket is connected: - for _ in range(20): - if assigned_port.value != 0: - # read which port has been assigned by the OS - port = assigned_port.value - break - time.sleep(0.01) - else: - raise RuntimeError( - 'Timed out waiting for the BPD server to start' - ) + # TODO: ugly hack. remove + print("ignoring port in queue:", assigned_port.get(timeout=2)) + # Wait until the socket is connected + port = assigned_port.get(timeout=2) + print("test bpd server on port", port) + time.sleep(0.1) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: From 088af4d1714032e69956918048cf30f4be7ddc4b Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Jul 2019 19:03:29 +0200 Subject: [PATCH 117/613] bpd tests: skip `control_port` in queue When setting up bpd tests, two servers are startet: first a control server, then bpd. Both send their assigned ports down a queue. The recipient only needs bpd's port and thus skips the first queue entry. --- test/test_player.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/test_player.py b/test/test_player.py index 78e3948ca..021a04fc0 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -252,9 +252,8 @@ def start_server(args, assigned_port, listener_patch): # `bluelet_listener` as this function will replace it at its # original location. listener = bluelet_listener(host, port) - # read which port has been assigned by the OS - # TODO: change to put_nowait. There should always be a free slot. - assigned_port.put(listener.sock.getsockname()[1]) + # read port assigned by OS + assigned_port.put_nowait(listener.sock.getsockname()[1]) return listener listener_patch.side_effect = listener_wrap @@ -302,7 +301,7 @@ class BPDTestHelper(unittest.TestCase, TestHelper): config_file.close() # Fork and launch BPD in the new process: - assigned_port = mp.Queue(1) + assigned_port = mp.Queue(2) # 2 slots, `control_port` and `port` server = mp.Process(target=start_server, args=([ '--library', self.config['library'].as_filename(), '--directory', py3_path(self.libdir), @@ -312,12 +311,8 @@ class BPDTestHelper(unittest.TestCase, TestHelper): server.start() try: - # TODO: ugly hack. remove - print("ignoring port in queue:", assigned_port.get(timeout=2)) - # Wait until the socket is connected - port = assigned_port.get(timeout=2) - print("test bpd server on port", port) - time.sleep(0.1) + assigned_port.get(timeout=1) # skip control_port + port = assigned_port.get(timeout=0.5) # read port sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: From 5e5cb3cd4349d31ff5ab429ab723b517cf1844aa Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Jul 2019 19:26:45 +0200 Subject: [PATCH 118/613] changelog: fix "Sporadic test failures in BPD tests #3309" #3330 Add a changelog entry asking plugin developers to report any further occurrences of this failure. --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index aa544bcac..001509272 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -80,6 +80,9 @@ For plugin developers: value. Thanks to :user:`zsinskri`. :bug:`3329` +* There were sporadic failures in ``test.test_player``. Hopefully these are + fixed. If they resurface, please reopen the relevant issue. + :bug:`3309` :bug:`3330` For packagers: From 7a7314ee3f921805fac3d4bc55e59e29d81526a8 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Oct 2018 22:27:02 +0200 Subject: [PATCH 119/613] Allow other ReplayGain backends to support R128. Previously using EBU R128 forced the use of the bs1770gain backend. This change adds a whitelist of backends supporting R128. When the configured backend is in that list it will also be used for R128 calculations. Otherwise bs1770gain is still used as a default. This should not change the overall behaviour of the program at all, but allow for further R128-supporting backends to be added. --- beetsplug/replaygain.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 58a4df83c..084172336 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -88,6 +88,10 @@ class Backend(object): # individual tracks which can be used for any backend. raise NotImplementedError() + def use_ebu_r128(self): + """Set this Backend up to use EBU R128.""" + pass + # bsg1770gain backend class Bs1770gainBackend(Backend): @@ -277,6 +281,10 @@ class Bs1770gainBackend(Backend): out.append(album_gain["album"]) return out + def use_ebu_r128(self): + """Set this Backend up to use EBU R128.""" + self.method = '--ebu' + # mpgain/aacgain CLI tool backend. class CommandBackend(Backend): @@ -830,6 +838,8 @@ class ReplayGainPlugin(BeetsPlugin): "bs1770gain": Bs1770gainBackend, } + r128_backend_names = ["bs1770gain"] + def __init__(self): super(ReplayGainPlugin, self).__init__() @@ -1024,7 +1034,9 @@ class ReplayGainPlugin(BeetsPlugin): u"Fatal replay gain error: {0}".format(e)) def init_r128_backend(self): - backend_name = 'bs1770gain' + backend_name = self.config["backend"].as_str() + if backend_name not in self.r128_backend_names: + backend_name = "bs1770gain" try: self.r128_backend_instance = self.backends[backend_name]( @@ -1034,7 +1046,7 @@ class ReplayGainPlugin(BeetsPlugin): raise ui.UserError( u'replaygain initialization failed: {0}'.format(e)) - self.r128_backend_instance.method = '--ebu' + self.r128_backend_instance.use_ebu_r128() def imported(self, session, task): """Add replay gain info to items or albums of ``task``. From c3af5b3763b6bdafe66d53cb493660720eb93192 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Thu, 18 Oct 2018 20:50:19 +0200 Subject: [PATCH 120/613] replaygain: add ffmpeg backend Add replaygain backend using ffmpeg's ebur128 filter. The album gain is calculated as the mean of all BS.1770 gating block powers. Besides differences in gating block offset, this should be equivalent to a BS.1770 analysis of a proper concatenation of all tracks. Just calculating the mean of all track gains (as implemented by the bs1770gain backend) yields incorrect results as that would: - completely ignore track lengths - just using length in seconds won't work either (e.g. BS.1770 ignores passages below a threshold) - take the mean of track loudness, not power When using the ffmpeg replaygain backend to create R128_*_GAIN tags, the targetlevel will be set to -23 LUFS. GitHub PullRequest #3065 will make this configurable. It will also skip peak calculation, as there is no R128_*_PEAK tag. It is checked if the libavfilter library supports replaygain calculation. Before version 6.67.100 that did require the `--enable-libebur128` compile-time-option, after that the ebur128 library is included in libavfilter itself. Thus we require either a recent enough libavfilter version or the `--enable-libebur128` option. --- beetsplug/replaygain.py | 276 +++++++++++++++++++++++++++++++++++- docs/plugins/replaygain.rst | 22 ++- test/test_replaygain.py | 7 + 3 files changed, 298 insertions(+), 7 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 084172336..80934c1ec 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -18,6 +18,7 @@ from __future__ import division, absolute_import, print_function import subprocess import os import collections +import math import sys import warnings import xml.parsers.expat @@ -64,9 +65,22 @@ def call(args, **kwargs): raise ReplayGainError(u"argument encoding failed") +def db_to_lufs(db): + """Convert db to LUFS. + + According to https://wiki.hydrogenaud.io/index.php?title= + ReplayGain_2.0_specification#Reference_level + """ + return db - 107 + + # Backend base and plumbing classes. +# gain: in LU to reference level +# peak: part of full scale (FS is 1.0) Gain = collections.namedtuple("Gain", "gain peak") +# album_gain: Gain object +# track_gains: list of Gain objects AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") @@ -81,11 +95,15 @@ class Backend(object): self._log = log def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of Gain objects. + """ raise NotImplementedError() def compute_album_gain(self, items): - # TODO: implement album gain in terms of track gain of the - # individual tracks which can be used for any backend. + """Computes the album gain of the given album, returns an + AlbumGain object. + """ raise NotImplementedError() def use_ebu_r128(self): @@ -286,6 +304,257 @@ class Bs1770gainBackend(Backend): self.method = '--ebu' +# ffmpeg backend +class FfmpegBackend(Backend): + """A replaygain backend using ffmpegs ebur128 filter. + """ + def __init__(self, config, log): + super(FfmpegBackend, self).__init__(config, log) + config.add({ + "peak": "true" + }) + self._peak_method = config["peak"].as_str() + self._target_level = db_to_lufs(config['targetlevel'].as_number()) + self._ffmpeg_path = "ffmpeg" + + # check that ffmpeg is installed + try: + ffmpeg_version_out = call([self._ffmpeg_path, "-version"]) + except OSError: + raise FatalReplayGainError( + u"could not find ffmpeg at {0}".format(self._ffmpeg_path) + ) + incompatible_ffmpeg = True + for line in ffmpeg_version_out.stdout.splitlines(): + if line.startswith(b"configuration:"): + if b"--enable-libebur128" in line: + incompatible_ffmpeg = False + if line.startswith(b"libavfilter"): + version = line.split(b" ", 1)[1].split(b"/", 1)[0].split(b".") + version = tuple(map(int, version)) + if version >= (6, 67, 100): + incompatible_ffmpeg = False + if incompatible_ffmpeg: + raise FatalReplayGainError( + u"Installed FFmpeg version does not support ReplayGain." + u"calculation. Either libavfilter version 6.67.100 or above or" + u"the --enable-libebur128 configuration option is required." + ) + + # check that peak_method is valid + valid_peak_method = ("true", "sample") + if self._peak_method not in valid_peak_method: + raise ui.UserError( + u"Selected ReplayGain peak method {0} is not supported. " + u"Please select one of: {1}".format( + self._peak_method, + u', '.join(valid_peak_method) + ) + ) + + def compute_track_gain(self, items): + """Computes the track gain of the given tracks, returns a list + of Gain objects (the track gains). + """ + gains = [] + for item in items: + gains.append( + self._analyse_item( + item, + count_blocks=False, + )[0] # take only the gain, discarding number of gating blocks + ) + return gains + + def compute_album_gain(self, items): + """Computes the album gain of the given album, returns an + AlbumGain object. + """ + # analyse tracks + # list of track Gain objects + track_gains = [] + # maximum peak + album_peak = 0 + # sum of BS.1770 gating block powers + sum_powers = 0 + # total number of BS.1770 gating blocks + n_blocks = 0 + + for item in items: + track_gain, track_n_blocks = self._analyse_item(item) + + track_gains.append(track_gain) + + # album peak is maximum track peak + album_peak = max(album_peak, track_gain.peak) + + # prepare album_gain calculation + # total number of blocks is sum of track blocks + n_blocks += track_n_blocks + + # convert `LU to target_level` -> LUFS + track_loudness = self._target_level - track_gain.gain + # This reverses ITU-R BS.1770-4 p. 6 equation (5) to convert + # from loudness to power. The result is the average gating + # block power. + track_power = 10**((track_loudness + 0.691) / 10) + + # Weight that average power by the number of gating blocks to + # get the sum of all their powers. Add that to the sum of all + # block powers in this album. + sum_powers += track_power * track_n_blocks + + # calculate album gain + if n_blocks > 0: + # compare ITU-R BS.1770-4 p. 6 equation (5) + # Album gain is the replaygain of the concatenation of all tracks. + album_gain = -0.691 + 10 * math.log10(sum_powers / n_blocks) + else: + album_gain = -70 + # convert LUFS -> `LU to target_level` + album_gain = self._target_level - album_gain + + self._log.debug( + u"{0}: gain {1} LU, peak {2}" + .format(items, album_gain, album_peak) + ) + + return AlbumGain(Gain(album_gain, album_peak), track_gains) + + def _construct_cmd(self, item, peak_method): + """Construct the shell command to analyse items.""" + return [ + self._ffmpeg_path, + "-nostats", + "-hide_banner", + "-i", + item.path, + "-filter", + "ebur128=peak={0}".format(peak_method), + "-f", + "null", + "-", + ] + + def _analyse_item(self, item, count_blocks=True): + """Analyse item. Returns a Pair (Gain object, number of gating + blocks above threshold). + + If `count_blocks` is False, the number of gating blocks returned + will be 0. + """ + # call ffmpeg + self._log.debug(u"analyzing {0}".format(item)) + cmd = self._construct_cmd(item, self._peak_method) + self._log.debug( + u'executing {0}', u' '.join(map(displayable_path, cmd)) + ) + output = call(cmd).stderr.splitlines() + + # parse output + + if self._peak_method == "none": + peak = 0 + else: + line_peak = self._find_line( + output, + " {0} peak:".format(self._peak_method.capitalize()).encode(), + len(output) - 1, -1, + ) + peak = self._parse_float( + output[self._find_line( + output, b" Peak:", + line_peak, + )]) + # convert TPFS -> part of FS + peak = 10**(peak / 20) + + line_integrated_loudness = self._find_line( + output, b" Integrated loudness:", + len(output) - 1, -1, + ) + gain = self._parse_float( + output[self._find_line( + output, b" I:", + line_integrated_loudness, + )]) + # convert LUFS -> LU from target level + gain = self._target_level - gain + + # count BS.1770 gating blocks + n_blocks = 0 + if count_blocks: + gating_threshold = self._parse_float( + output[self._find_line( + output, b" Threshold:", + line_integrated_loudness, + )]) + for line in output: + if not line.startswith(b"[Parsed_ebur128"): + continue + if line.endswith(b"Summary:"): + continue + line = line.split(b"M:", 1) + if len(line) < 2: + continue + if self._parse_float(b"M: " + line[1]) >= gating_threshold: + n_blocks += 1 + self._log.debug( + u"{0}: {1} blocks over {2} LUFS" + .format(item, n_blocks, gating_threshold) + ) + + self._log.debug( + u"{0}: gain {1} LU, peak {2}" + .format(item, gain, peak) + ) + + return Gain(gain, peak), n_blocks + + def _find_line(self, output, search, start_index=0, step_size=1): + """Return index of line beginning with `search`. + + Begins searching at index `start_index` in `output`. + """ + end_index = len(output) if step_size > 0 else -1 + for i in range(start_index, end_index, step_size): + if output[i].startswith(search): + return i + raise ReplayGainError( + u"ffmpeg output: missing {0} after line {1}" + .format(repr(search), start_index) + ) + + def _parse_float(self, line): + """Extract a float. + + Extract a float from a key value pair in `line`. + """ + # extract value + value = line.split(b":", 1) + if len(value) < 2: + raise ReplayGainError( + u"ffmpeg ouput: expected key value pair, found {0}" + .format(line) + ) + value = value[1].lstrip() + # strip unit + value = value.split(b" ", 1)[0] + # cast value to as_type + try: + return float(value) + except ValueError: + raise ReplayGainError( + u"ffmpeg output: expected float value, found {1}" + .format(value) + ) + + def use_ebu_r128(self): + """Set this Backend up to use EBU R128.""" + self._target_level = -23 + self._peak_method = "none" # R128 tags do not need peak + + # mpgain/aacgain CLI tool backend. class CommandBackend(Backend): @@ -836,9 +1105,10 @@ class ReplayGainPlugin(BeetsPlugin): "gstreamer": GStreamerBackend, "audiotools": AudioToolsBackend, "bs1770gain": Bs1770gainBackend, + "ffmpeg": FfmpegBackend, } - r128_backend_names = ["bs1770gain"] + r128_backend_names = ["bs1770gain", "ffmpeg"] def __init__(self): super(ReplayGainPlugin, self).__init__() diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 57630f1d6..6b3dc153f 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -10,10 +10,10 @@ playback levels. Installation ------------ -This plugin can use one of three backends to compute the ReplayGain values: -GStreamer, mp3gain (and its cousin, aacgain), Python Audio Tools. mp3gain -can be easier to install but GStreamer and Audio Tools support more audio -formats. +This plugin can use one of many backends to compute the ReplayGain values: +GStreamer, mp3gain (and its cousin, aacgain), Python Audio Tools or ffmpeg. +mp3gain can be easier to install but GStreamer, Audio Tools and ffmpeg support +more audio formats. Once installed, this plugin analyzes all files during the import process. This can be a slow process; to instead analyze after the fact, disable automatic @@ -75,6 +75,15 @@ On OS X, most of the dependencies can be installed with `Homebrew`_:: .. _Python Audio Tools: http://audiotools.sourceforge.net +ffmpeg +`````` + +This backend uses ffmpeg to calculate EBU R128 gain values. +To use it, install the `ffmpeg`_ command-line tool and select the +``ffmpeg`` backend in your config file. + +.. _ffmpeg: https://ffmpeg.org + Configuration ------------- @@ -106,6 +115,11 @@ These options only work with the "command" backend: would keep clipping from occurring. Default: ``yes``. +This option only works with the "ffmpeg" backend: + +- **peak**: Either ``true`` (the default) or ``sample``. ``true`` is + more accurate but slower. + Manual Analysis --------------- diff --git a/test/test_replaygain.py b/test/test_replaygain.py index b482da14e..746a01355 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -45,6 +45,8 @@ if has_program('bs1770gain', ['--replaygain']): else: LOUDNESS_PROG_AVAILABLE = False +FFMPEG_AVAILABLE = has_program('ffmpeg', ['-version']) + def reset_replaygain(item): item['rg_track_peak'] = None @@ -205,6 +207,11 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): self.assertEqual(len(matching), 2) +@unittest.skipIf(not FFMPEG_AVAILABLE, u'ffmpeg cannot be found') +class ReplayGainFfmpegTest(ReplayGainCliTestBase, unittest.TestCase): + backend = u'ffmpeg' + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From b589521755daf5a670b65c4c5b1afe7c605c1903 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 7 Nov 2018 19:30:30 +0100 Subject: [PATCH 121/613] changelog entry: ffmpeg replaygain backend Add changelog entry for the new ffmpeg replaygain backend. --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 001509272..2b6740d48 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,9 @@ New features: level. Thanks to :user:`samuelnilsson` :bug:`293` +* :doc:`/plugins/replaygain`: The new ``ffmpeg`` ReplayGain backend supports + ``R128_`` tags, just like the ``bs1770gain`` backend. + :bug:`3056` * A new :doc:`/plugins/parentwork` gets information about the original work, which is useful for classical music. Thanks to :user:`dosoe`. From 271a3c980ca3f461028979b3ddf832d31c5e48e4 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 17 Jul 2019 19:50:43 +0200 Subject: [PATCH 122/613] replaygain: ffmpeg: increase parser readability Use keyword arguments to make the ffmpeg parser more readable. --- beetsplug/replaygain.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 80934c1ec..5817a8a75 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -459,25 +459,27 @@ class FfmpegBackend(Backend): line_peak = self._find_line( output, " {0} peak:".format(self._peak_method.capitalize()).encode(), - len(output) - 1, -1, - ) + start_line=(len(output) - 1), step_size=-1, + ) peak = self._parse_float( output[self._find_line( output, b" Peak:", line_peak, - )]) + )] + ) # convert TPFS -> part of FS peak = 10**(peak / 20) line_integrated_loudness = self._find_line( output, b" Integrated loudness:", - len(output) - 1, -1, - ) + start_line=(len(output) - 1), step_size=-1, + ) gain = self._parse_float( output[self._find_line( output, b" I:", line_integrated_loudness, - )]) + )] + ) # convert LUFS -> LU from target level gain = self._target_level - gain @@ -487,8 +489,9 @@ class FfmpegBackend(Backend): gating_threshold = self._parse_float( output[self._find_line( output, b" Threshold:", - line_integrated_loudness, - )]) + start_line=line_integrated_loudness, + )] + ) for line in output: if not line.startswith(b"[Parsed_ebur128"): continue @@ -502,27 +505,27 @@ class FfmpegBackend(Backend): self._log.debug( u"{0}: {1} blocks over {2} LUFS" .format(item, n_blocks, gating_threshold) - ) + ) self._log.debug( u"{0}: gain {1} LU, peak {2}" .format(item, gain, peak) - ) + ) return Gain(gain, peak), n_blocks - def _find_line(self, output, search, start_index=0, step_size=1): + def _find_line(self, output, search, start_line=0, step_size=1): """Return index of line beginning with `search`. - Begins searching at index `start_index` in `output`. + Begins searching at index `start_line` in `output`. """ end_index = len(output) if step_size > 0 else -1 - for i in range(start_index, end_index, step_size): + for i in range(start_line, end_index, step_size): if output[i].startswith(search): return i raise ReplayGainError( u"ffmpeg output: missing {0} after line {1}" - .format(repr(search), start_index) + .format(repr(search), start_line) ) def _parse_float(self, line): From 8b4d03095f5b50ec72031c48b15b009cbf300376 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Jul 2019 16:19:09 -0400 Subject: [PATCH 123/613] parentwork tests: Remove unnecessary mocking This mocking doesn't do anything because `command_output` is never invoked by the code under test. See also #3332. --- test/test_parentwork.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test/test_parentwork.py b/test/test_parentwork.py index dfebc6602..aa25c7f51 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -17,7 +17,6 @@ from __future__ import division, absolute_import, print_function -from mock import patch import unittest from test.helper import TestHelper @@ -25,7 +24,6 @@ from beets.library import Item from beetsplug import parentwork -@patch('beets.util.command_output') class ParentWorkTest(unittest.TestCase, TestHelper): def setUp(self): """Set up configuration""" @@ -36,39 +34,36 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.unload_plugins() self.teardown_beets() - def test_normal_case(self, command_output): + def test_normal_case(self): item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-b8ebc18e8c53') item.add(self.lib) - command_output.return_value = u'32c8943f-1b27-3a23-8660-4567f4847c94' self.run_command('parentwork') item.load() self.assertEqual(item['mb_parentworkid'], u'32c8943f-1b27-3a23-8660-4567f4847c94') - def test_force(self, command_output): + def test_force(self): self.config['parentwork']['force'] = True item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-b8ebc18e8c53', mb_parentworkid=u'XXX') item.add(self.lib) - command_output.return_value = u'32c8943f-1b27-3a23-8660-4567f4847c94' self.run_command('parentwork') item.load() self.assertEqual(item['mb_parentworkid'], u'32c8943f-1b27-3a23-8660-4567f4847c94') - def test_no_force(self, command_output): + def test_no_force(self): self.config['parentwork']['force'] = True item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-\ b8ebc18e8c53', mb_parentworkid=u'XXX') item.add(self.lib) - command_output.return_value = u'32c8943f-1b27-3a23-8660-4567f4847c94' self.run_command('parentwork') item.load() @@ -77,7 +72,7 @@ class ParentWorkTest(unittest.TestCase, TestHelper): # test different cases, still with Matthew Passion Ouverture or Mozart # requiem - def test_direct_parent_work(self, command_output): + def test_direct_parent_work(self): mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', parentwork.direct_parent_id(mb_workid)[0]) From f9ff56f4968ad951434aef97fcc6e4cac88d70b4 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Sun, 21 Jul 2019 01:18:49 +0200 Subject: [PATCH 124/613] improve wording in the ffmpeg replaygain backend This commit mostly addresses feedback: - remove some unused parenthesis - fix a typo - expand some docstrings - document that ffmpeg is usually easy to install --- beetsplug/replaygain.py | 19 ++++++++++--------- docs/plugins/replaygain.rst | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 5817a8a75..d9f8c02d9 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -306,7 +306,7 @@ class Bs1770gainBackend(Backend): # ffmpeg backend class FfmpegBackend(Backend): - """A replaygain backend using ffmpegs ebur128 filter. + """A replaygain backend using ffmpeg's ebur128 filter. """ def __init__(self, config, log): super(FfmpegBackend, self).__init__(config, log) @@ -342,7 +342,7 @@ class FfmpegBackend(Backend): ) # check that peak_method is valid - valid_peak_method = ("true", "sample") + valid_peak_method = "true", "sample" if self._peak_method not in valid_peak_method: raise ui.UserError( u"Selected ReplayGain peak method {0} is not supported. " @@ -437,8 +437,8 @@ class FfmpegBackend(Backend): ] def _analyse_item(self, item, count_blocks=True): - """Analyse item. Returns a Pair (Gain object, number of gating - blocks above threshold). + """Analyse item. Return a pair of a Gain object and the number + of gating blocks above the threshold. If `count_blocks` is False, the number of gating blocks returned will be 0. @@ -459,7 +459,7 @@ class FfmpegBackend(Backend): line_peak = self._find_line( output, " {0} peak:".format(self._peak_method.capitalize()).encode(), - start_line=(len(output) - 1), step_size=-1, + start_line=len(output) - 1, step_size=-1, ) peak = self._parse_float( output[self._find_line( @@ -472,7 +472,7 @@ class FfmpegBackend(Backend): line_integrated_loudness = self._find_line( output, b" Integrated loudness:", - start_line=(len(output) - 1), step_size=-1, + start_line=len(output) - 1, step_size=-1, ) gain = self._parse_float( output[self._find_line( @@ -529,9 +529,10 @@ class FfmpegBackend(Backend): ) def _parse_float(self, line): - """Extract a float. + """Extract a float from a key value pair in `line`. - Extract a float from a key value pair in `line`. + This format is expected: /[^:]:\s*value.*/, where `value` is + the float. """ # extract value value = line.split(b":", 1) @@ -543,7 +544,7 @@ class FfmpegBackend(Backend): value = value[1].lstrip() # strip unit value = value.split(b" ", 1)[0] - # cast value to as_type + # cast value to float try: return float(value) except ValueError: diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index 6b3dc153f..a68f3f7fc 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -12,8 +12,8 @@ Installation This plugin can use one of many backends to compute the ReplayGain values: GStreamer, mp3gain (and its cousin, aacgain), Python Audio Tools or ffmpeg. -mp3gain can be easier to install but GStreamer, Audio Tools and ffmpeg support -more audio formats. +ffmpeg and mp3gain can be easier to install. mp3gain supports less audio formats +then the other backend. Once installed, this plugin analyzes all files during the import process. This can be a slow process; to instead analyze after the fact, disable automatic From e5f2fe6fd322450c09f5ec175af91f3b4b9d4f90 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Sun, 21 Jul 2019 01:28:16 +0200 Subject: [PATCH 125/613] avoid test failure Use the POSIX character class instead of `\s` to match all whitespace in a regular expression describing the language of valid inputs, in order to avoid a test failure for the invalid escape sequence `\s` in Python strings. --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index d9f8c02d9..febacbc1b 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -531,7 +531,7 @@ class FfmpegBackend(Backend): def _parse_float(self, line): """Extract a float from a key value pair in `line`. - This format is expected: /[^:]:\s*value.*/, where `value` is + This format is expected: /[^:]:[[:space:]]*value.*/, where `value` is the float. """ # extract value From b9063a0240e1d9d53f5910b389767fd351b3a9be Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Mon, 22 Jul 2019 12:49:50 +0200 Subject: [PATCH 126/613] fix bs1770gain test This test caused other tests to fail due to missing cleanup. --- test/test_replaygain.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 746a01355..9937cc7d0 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -179,12 +179,25 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): # Patch call to return nothing, bypassing the bs1770gain installation # check. - call_patch.return_value = None - self.load_plugins('replaygain') + call_patch.return_value = CommandOutput(stdout=b"", stderr=b"") + try: + self.load_plugins('replaygain') + except Exception: + import sys + exc_info = sys.exc_info() + try: + self.tearDown() + except Exception: + pass + six.reraise(exc_info[1], None, exc_info[2]) for item in self.add_album_fixture(2).items(): reset_replaygain(item) + def tearDown(self): + self.teardown_beets() + self.unload_plugins() + @patch('beetsplug.replaygain.call') def test_malformed_output(self, call_patch): # Return malformed XML (the ampersand should be &) From 0c8eead45978addf6337b81ddd9efe65c0786f33 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Sat, 27 Oct 2018 21:04:51 +0200 Subject: [PATCH 127/613] replaygain: pass target_level and peak to backends Configure the replaygain analysis by passing arguments to the Backends. This avoids the difference between ReplayGain and EBU r128 backends; every Backend can now fulfil both tasks. Additionally it eases Backend development as the difference between the two tag formats is now completely handled in the main Plugin, not in the Backends. --- beetsplug/replaygain.py | 278 +++++++++++++++++++++++----------------- 1 file changed, 161 insertions(+), 117 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index febacbc1b..ea0d70aa5 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -21,6 +21,7 @@ import collections import math import sys import warnings +import enum import xml.parsers.expat from six.moves import zip @@ -74,6 +75,15 @@ def db_to_lufs(db): return db - 107 +def lufs_to_db(db): + """Convert LUFS to db. + + According to https://wiki.hydrogenaud.io/index.php?title= + ReplayGain_2.0_specification#Reference_level + """ + return db + 107 + + # Backend base and plumbing classes. # gain: in LU to reference level @@ -84,6 +94,12 @@ Gain = collections.namedtuple("Gain", "gain peak") AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") +class Peak(enum.Enum): + none = 0 + true = 1 + sample = 2 + + class Backend(object): """An abstract class representing engine for calculating RG values. """ @@ -94,22 +110,18 @@ class Backend(object): """ self._log = log - def compute_track_gain(self, items): + def compute_track_gain(self, items, target_level, peak): """Computes the track gain of the given tracks, returns a list of Gain objects. """ raise NotImplementedError() - def compute_album_gain(self, items): + def compute_album_gain(self, items, target_level, peak): """Computes the album gain of the given album, returns an AlbumGain object. """ raise NotImplementedError() - def use_ebu_r128(self): - """Set this Backend up to use EBU R128.""" - pass - # bsg1770gain backend class Bs1770gainBackend(Backend): @@ -117,44 +129,51 @@ class Bs1770gainBackend(Backend): its flavors EBU R128, ATSC A/85 and Replaygain 2.0. """ + methods = { + -24: "atsc", + -23: "ebu", + -18: "replaygain", + } + def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) config.add({ 'chunk_at': 5000, - 'method': 'replaygain', + 'method': '', }) self.chunk_at = config['chunk_at'].as_number() - self.method = '--' + config['method'].as_str() + # backward compatibility to `method` config option + self.__method = config['method'].as_str() cmd = 'bs1770gain' try: - call([cmd, self.method]) + call([cmd, "--help"]) self.command = cmd except OSError: raise FatalReplayGainError( - u'Is bs1770gain installed? Is your method in config correct?' + u'Is bs1770gain installed?' ) if not self.command: raise FatalReplayGainError( u'no replaygain command found: install bs1770gain' ) - def compute_track_gain(self, items): + def compute_track_gain(self, items, target_level, peak): """Computes the track gain of the given tracks, returns a list of TrackGain objects. """ - output = self.compute_gain(items, False) + output = self.compute_gain(items, target_level, False) return output - def compute_album_gain(self, items): + def compute_album_gain(self, items, target_level, peak): """Computes the album gain of the given album, returns an AlbumGain object. """ # TODO: What should be done when not all tracks in the album are # supported? - output = self.compute_gain(items, True) + output = self.compute_gain(items, target_level, True) if not output: raise ReplayGainError(u'no output from bs1770gain') @@ -179,7 +198,7 @@ class Bs1770gainBackend(Backend): else: break - def compute_gain(self, items, is_album): + def compute_gain(self, items, target_level, is_album): """Computes the track or album gain of a list of items, returns a list of TrackGain objects. When computing album gain, the last TrackGain object returned is @@ -200,22 +219,38 @@ class Bs1770gainBackend(Backend): i = 0 for chunk in self.isplitter(items, self.chunk_at): i += 1 - returnchunk = self.compute_chunk_gain(chunk, is_album) + returnchunk = self.compute_chunk_gain( + chunk, + is_album, + target_level + ) albumgaintot += returnchunk[-1].gain albumpeaktot = max(albumpeaktot, returnchunk[-1].peak) returnchunks = returnchunks + returnchunk[0:-1] returnchunks.append(Gain(albumgaintot / i, albumpeaktot)) return returnchunks else: - return self.compute_chunk_gain(items, is_album) + return self.compute_chunk_gain(items, is_album, target_level) - def compute_chunk_gain(self, items, is_album): + def compute_chunk_gain(self, items, is_album, target_level): """Compute ReplayGain values and return a list of results dictionaries as given by `parse_tool_output`. """ + # choose method + target_level = db_to_lufs(target_level) + if self.__method != "": + # backward compatibility to `method` option + method = self.__method + elif target_level in self.methods: + method = self.methods[target_level] + gain_adjustment = 0 + else: + method = self.methods[-23] + gain_adjustment = target_level - lufs_to_db(-23) + # Construct shell command. cmd = [self.command] - cmd += [self.method] + cmd += ["--" + method] cmd += ['--xml', '-p'] # Workaround for Windows: the underlying tool fails on paths @@ -232,6 +267,12 @@ class Bs1770gainBackend(Backend): self._log.debug(u'analysis finished: {0}', output) results = self.parse_tool_output(output, path_list, is_album) + + if gain_adjustment != 0: + for i in range(len(results)): + orig = results[i] + results[i] = Gain(orig.gain + gain_adjustment, orig.peak) + self._log.debug(u'{0} items, {1} results', len(items), len(results)) return results @@ -299,10 +340,6 @@ class Bs1770gainBackend(Backend): out.append(album_gain["album"]) return out - def use_ebu_r128(self): - """Set this Backend up to use EBU R128.""" - self.method = '--ebu' - # ffmpeg backend class FfmpegBackend(Backend): @@ -310,11 +347,6 @@ class FfmpegBackend(Backend): """ def __init__(self, config, log): super(FfmpegBackend, self).__init__(config, log) - config.add({ - "peak": "true" - }) - self._peak_method = config["peak"].as_str() - self._target_level = db_to_lufs(config['targetlevel'].as_number()) self._ffmpeg_path = "ffmpeg" # check that ffmpeg is installed @@ -341,18 +373,7 @@ class FfmpegBackend(Backend): u"the --enable-libebur128 configuration option is required." ) - # check that peak_method is valid - valid_peak_method = "true", "sample" - if self._peak_method not in valid_peak_method: - raise ui.UserError( - u"Selected ReplayGain peak method {0} is not supported. " - u"Please select one of: {1}".format( - self._peak_method, - u', '.join(valid_peak_method) - ) - ) - - def compute_track_gain(self, items): + def compute_track_gain(self, items, target_level, peak): """Computes the track gain of the given tracks, returns a list of Gain objects (the track gains). """ @@ -361,15 +382,19 @@ class FfmpegBackend(Backend): gains.append( self._analyse_item( item, + target_level, + peak, count_blocks=False, )[0] # take only the gain, discarding number of gating blocks ) return gains - def compute_album_gain(self, items): + def compute_album_gain(self, items, target_level, peak): """Computes the album gain of the given album, returns an AlbumGain object. """ + target_level_lufs = db_to_lufs(target_level) + # analyse tracks # list of track Gain objects track_gains = [] @@ -381,8 +406,9 @@ class FfmpegBackend(Backend): n_blocks = 0 for item in items: - track_gain, track_n_blocks = self._analyse_item(item) - + track_gain, track_n_blocks = self._analyse_item( + item, target_level, peak + ) track_gains.append(track_gain) # album peak is maximum track peak @@ -393,7 +419,7 @@ class FfmpegBackend(Backend): n_blocks += track_n_blocks # convert `LU to target_level` -> LUFS - track_loudness = self._target_level - track_gain.gain + track_loudness = target_level_lufs - track_gain.gain # This reverses ITU-R BS.1770-4 p. 6 equation (5) to convert # from loudness to power. The result is the average gating # block power. @@ -412,7 +438,7 @@ class FfmpegBackend(Backend): else: album_gain = -70 # convert LUFS -> `LU to target_level` - album_gain = self._target_level - album_gain + album_gain = target_level_lufs - album_gain self._log.debug( u"{0}: gain {1} LU, peak {2}" @@ -436,16 +462,23 @@ class FfmpegBackend(Backend): "-", ] - def _analyse_item(self, item, count_blocks=True): + def _analyse_item(self, item, target_level, peak, count_blocks=True): """Analyse item. Return a pair of a Gain object and the number of gating blocks above the threshold. If `count_blocks` is False, the number of gating blocks returned will be 0. """ + target_level_lufs = db_to_lufs(target_level) + peak_method = { + Peak.none: "none", + Peak.true: "true", + Peak.sample: "sample", + }[peak] + # call ffmpeg self._log.debug(u"analyzing {0}".format(item)) - cmd = self._construct_cmd(item, self._peak_method) + cmd = self._construct_cmd(item, peak_method) self._log.debug( u'executing {0}', u' '.join(map(displayable_path, cmd)) ) @@ -453,12 +486,12 @@ class FfmpegBackend(Backend): # parse output - if self._peak_method == "none": + if peak is Peak.none: peak = 0 else: line_peak = self._find_line( output, - " {0} peak:".format(self._peak_method.capitalize()).encode(), + " {0} peak:".format(peak_method.capitalize()).encode(), start_line=len(output) - 1, step_size=-1, ) peak = self._parse_float( @@ -481,7 +514,7 @@ class FfmpegBackend(Backend): )] ) # convert LUFS -> LU from target level - gain = self._target_level - gain + gain = target_level_lufs - gain # count BS.1770 gating blocks n_blocks = 0 @@ -553,11 +586,6 @@ class FfmpegBackend(Backend): .format(value) ) - def use_ebu_r128(self): - """Set this Backend up to use EBU R128.""" - self._target_level = -23 - self._peak_method = "none" # R128 tags do not need peak - # mpgain/aacgain CLI tool backend. class CommandBackend(Backend): @@ -592,18 +620,16 @@ class CommandBackend(Backend): ) self.noclip = config['noclip'].get(bool) - target_level = config['targetlevel'].as_number() - self.gain_offset = int(target_level - 89) - def compute_track_gain(self, items): + def compute_track_gain(self, items, target_level, peak): """Computes the track gain of the given tracks, returns a list of TrackGain objects. """ supported_items = list(filter(self.format_supported, items)) - output = self.compute_gain(supported_items, False) + output = self.compute_gain(supported_items, target_level, False) return output - def compute_album_gain(self, items): + def compute_album_gain(self, items, target_level, peak): """Computes the album gain of the given album, returns an AlbumGain object. """ @@ -615,7 +641,7 @@ class CommandBackend(Backend): self._log.debug(u'tracks are of unsupported format') return AlbumGain(None, []) - output = self.compute_gain(supported_items, True) + output = self.compute_gain(supported_items, target_level, True) return AlbumGain(output[-1], output[:-1]) def format_supported(self, item): @@ -627,7 +653,7 @@ class CommandBackend(Backend): return False return True - def compute_gain(self, items, is_album): + def compute_gain(self, items, target_level, is_album): """Computes the track or album gain of a list of items, returns a list of TrackGain objects. @@ -654,7 +680,7 @@ class CommandBackend(Backend): else: # Disable clipping warning. cmd = cmd + ['-c'] - cmd = cmd + ['-d', str(self.gain_offset)] + cmd = cmd + ['-d', str(int(target_level - 89))] cmd = cmd + [syspath(i.path) for i in items] self._log.debug(u'analyzing {0} files', len(items)) @@ -717,8 +743,6 @@ class GStreamerBackend(Backend): # to rganalsys should have their gain computed, even if it # already exists. self._rg.set_property("forced", True) - self._rg.set_property("reference-level", - config["targetlevel"].as_number()) self._sink = self.Gst.ElementFactory.make("fakesink", "sink") self._pipe = self.Gst.Pipeline() @@ -779,7 +803,7 @@ class GStreamerBackend(Backend): self.GLib = GLib self.Gst = Gst - def compute(self, files, album): + def compute(self, files, target_level, album): self._error = None self._files = list(files) @@ -788,6 +812,8 @@ class GStreamerBackend(Backend): self._file_tags = collections.defaultdict(dict) + self._rg.set_property("reference-level", target_level) + if album: self._rg.set_property("num-tracks", len(self._files)) @@ -796,8 +822,8 @@ class GStreamerBackend(Backend): if self._error is not None: raise self._error - def compute_track_gain(self, items): - self.compute(items, False) + def compute_track_gain(self, items, target_level, peak): + self.compute(items, target_level, False) if len(self._file_tags) != len(items): raise ReplayGainError(u"Some tracks did not receive tags") @@ -808,9 +834,9 @@ class GStreamerBackend(Backend): return ret - def compute_album_gain(self, items): + def compute_album_gain(self, items, target_level, peak): items = list(items) - self.compute(items, True) + self.compute(items, target_level, True) if len(self._file_tags) != len(items): raise ReplayGainError(u"Some items in album did not receive tags") @@ -1024,14 +1050,21 @@ class AudioToolsBackend(Backend): return return rg - def compute_track_gain(self, items): + def compute_track_gain(self, items, target_level, peak): """Compute ReplayGain values for the requested items. :return list: list of :class:`Gain` objects """ - return [self._compute_track_gain(item) for item in items] + return [self._compute_track_gain(item, target_level) for item in items] - def _title_gain(self, rg, audiofile): + def _with_target_level(self, gain, target_level): + """Return `gain` relative to `target_level`. + + Assumes `gain` is relative to 89 db. + """ + return gain + (target_level - 89) + + def _title_gain(self, rg, audiofile, target_level): """Get the gain result pair from PyAudioTools using the `ReplayGain` instance `rg` for the given `audiofile`. @@ -1041,14 +1074,15 @@ class AudioToolsBackend(Backend): try: # The method needs an audiotools.PCMReader instance that can # be obtained from an audiofile instance. - return rg.title_gain(audiofile.to_pcm()) + gain, peak = rg.title_gain(audiofile.to_pcm()) except ValueError as exc: # `audiotools.replaygain` can raise a `ValueError` if the sample # rate is incorrect. self._log.debug(u'error in rg.title_gain() call: {}', exc) raise ReplayGainError(u'audiotools audio data error') + return self._with_target_level(gain, target_level), peak - def _compute_track_gain(self, item): + def _compute_track_gain(self, item, target_level): """Compute ReplayGain value for the requested item. :rtype: :class:`Gain` @@ -1058,13 +1092,15 @@ class AudioToolsBackend(Backend): # Each call to title_gain on a ReplayGain object returns peak and gain # of the track. - rg_track_gain, rg_track_peak = self._title_gain(rg, audiofile) + rg_track_gain, rg_track_peak = self._title_gain( + rg, audiofile, target_level + ) self._log.debug(u'ReplayGain for track {0} - {1}: {2:.2f}, {3:.2f}', item.artist, item.title, rg_track_gain, rg_track_peak) return Gain(gain=rg_track_gain, peak=rg_track_peak) - def compute_album_gain(self, items): + def compute_album_gain(self, items, target_level, peak): """Compute ReplayGain values for the requested album and its items. :rtype: :class:`AlbumGain` @@ -1079,7 +1115,9 @@ class AudioToolsBackend(Backend): track_gains = [] for item in items: audiofile = self.open_audio_file(item) - rg_track_gain, rg_track_peak = self._title_gain(rg, audiofile) + rg_track_gain, rg_track_peak = self._title_gain( + rg, audiofile, target_level + ) track_gains.append( Gain(gain=rg_track_gain, peak=rg_track_peak) ) @@ -1089,6 +1127,7 @@ class AudioToolsBackend(Backend): # After getting the values for all tracks, it's possible to get the # album values. rg_album_gain, rg_album_peak = rg.album_gain() + rg_album_gain = self._with_target_level(rg_album_gain, target_level) self._log.debug(u'ReplayGain for album {0}: {1:.2f}, {2:.2f}', items[0].album, rg_album_gain, rg_album_peak) @@ -1112,7 +1151,10 @@ class ReplayGainPlugin(BeetsPlugin): "ffmpeg": FfmpegBackend, } - r128_backend_names = ["bs1770gain", "ffmpeg"] + peak_methods = { + "true": Peak.true, + "sample": Peak.sample, + } def __init__(self): super(ReplayGainPlugin, self).__init__() @@ -1124,7 +1166,8 @@ class ReplayGainPlugin(BeetsPlugin): 'backend': u'command', 'targetlevel': 89, 'r128': ['Opus'], - 'per_disc': False + 'per_disc': False, + "peak": "true", }) self.overwrite = self.config['overwrite'].get(bool) @@ -1138,6 +1181,16 @@ class ReplayGainPlugin(BeetsPlugin): u', '.join(self.backends.keys()) ) ) + peak_method = self.config["peak"].as_str() + if peak_method not in self.peak_methods: + raise ui.UserError( + u"Selected ReplayGain peak method {0} is not supported. " + u"Please select one of: {1}".format( + peak_method, + u', '.join(self.peak_methods.keys()) + ) + ) + self._peak_method = self.peak_methods[peak_method] # On-import analysis. if self.config['auto']: @@ -1154,8 +1207,6 @@ class ReplayGainPlugin(BeetsPlugin): raise ui.UserError( u'replaygain initialization failed: {0}'.format(e)) - self.r128_backend_instance = '' - def should_use_r128(self, item): """Checks the plugin setting to decide whether the calculation should be done using the EBU R128 standard and use R128_ tags instead. @@ -1208,6 +1259,20 @@ class ReplayGainPlugin(BeetsPlugin): self._log.debug(u'applied r128 album gain {0} LU', item.r128_album_gain) + def tag_specific_values(self, items): + """Return some tag specific values. + + Returns a tuple (store_track_gain, store_album_gain). + """ + if any([self.should_use_r128(item) for item in items]): + store_track_gain = self.store_track_r128_gain + store_album_gain = self.store_album_r128_gain + else: + store_track_gain = self.store_track_gain + store_album_gain = self.store_album_gain + + return store_track_gain, store_album_gain + def handle_album(self, album, write, force=False): """Compute album and track replay gain store it in all of the album's items. @@ -1229,16 +1294,8 @@ class ReplayGainPlugin(BeetsPlugin): u" for some tracks in album {0}".format(album) ) - if any([self.should_use_r128(item) for item in album.items()]): - if self.r128_backend_instance == '': - self.init_r128_backend() - backend_instance = self.r128_backend_instance - store_track_gain = self.store_track_r128_gain - store_album_gain = self.store_album_r128_gain - else: - backend_instance = self.backend_instance - store_track_gain = self.store_track_gain - store_album_gain = self.store_album_gain + tag_vals = self.tag_specific_values(album.items()) + store_track_gain, store_album_gain = tag_vals discs = dict() if self.per_disc: @@ -1251,7 +1308,11 @@ class ReplayGainPlugin(BeetsPlugin): for discnumber, items in discs.items(): try: - album_gain = backend_instance.compute_album_gain(items) + album_gain = self.backend_instance.compute_album_gain( + items, + self.config['targetlevel'].as_number(), + self._peak_method, + ) if len(album_gain.track_gains) != len(items): raise ReplayGainError( u"ReplayGain backend failed " @@ -1282,17 +1343,15 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info(u'analyzing {0}', item) - if self.should_use_r128(item): - if self.r128_backend_instance == '': - self.init_r128_backend() - backend_instance = self.r128_backend_instance - store_track_gain = self.store_track_r128_gain - else: - backend_instance = self.backend_instance - store_track_gain = self.store_track_gain + tag_vals = self.tag_specific_values([item]) + store_track_gain, store_album_gain = tag_vals try: - track_gains = backend_instance.compute_track_gain([item]) + track_gains = self.backend_instance.compute_track_gain( + [item], + self.config['targetlevel'].as_number(), + self._peak_method, + ) if len(track_gains) != 1: raise ReplayGainError( u"ReplayGain backend failed for track {0}".format(item) @@ -1307,21 +1366,6 @@ class ReplayGainPlugin(BeetsPlugin): raise ui.UserError( u"Fatal replay gain error: {0}".format(e)) - def init_r128_backend(self): - backend_name = self.config["backend"].as_str() - if backend_name not in self.r128_backend_names: - backend_name = "bs1770gain" - - try: - self.r128_backend_instance = self.backends[backend_name]( - self.config, self._log - ) - except (ReplayGainError, FatalReplayGainError) as e: - raise ui.UserError( - u'replaygain initialization failed: {0}'.format(e)) - - self.r128_backend_instance.use_ebu_r128() - def imported(self, session, task): """Add replay gain info to items or albums of ``task``. """ From e7e2c424e7dd0efb202c9550ce35438e7915d979 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Mon, 22 Jul 2019 13:20:28 +0200 Subject: [PATCH 128/613] replaygain: targetlevel and peak_method depends on tag format Allow to configure the target level for R128_* tags separately from REPLAYGAIN_* tags and skip peak calculation for R128_* tags if possible. --- beetsplug/replaygain.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ea0d70aa5..a0108b5a0 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1164,10 +1164,11 @@ class ReplayGainPlugin(BeetsPlugin): 'overwrite': False, 'auto': True, 'backend': u'command', + 'per_disc': False, + 'peak': 'true', 'targetlevel': 89, 'r128': ['Opus'], - 'per_disc': False, - "peak": "true", + 'r128_targetlevel': lufs_to_db(-23), }) self.overwrite = self.config['overwrite'].get(bool) @@ -1262,16 +1263,21 @@ class ReplayGainPlugin(BeetsPlugin): def tag_specific_values(self, items): """Return some tag specific values. - Returns a tuple (store_track_gain, store_album_gain). + Returns a tuple (store_track_gain, store_album_gain, target_level, + peak_method). """ if any([self.should_use_r128(item) for item in items]): store_track_gain = self.store_track_r128_gain store_album_gain = self.store_album_r128_gain + target_level = self.config['r128_targetlevel'].as_number() + peak = Peak.none # R128_* tags do not store the track/album peak else: store_track_gain = self.store_track_gain store_album_gain = self.store_album_gain + target_level = self.config['targetlevel'].as_number() + peak = self._peak_method - return store_track_gain, store_album_gain + return store_track_gain, store_album_gain, target_level, peak def handle_album(self, album, write, force=False): """Compute album and track replay gain store it in all of the @@ -1295,7 +1301,7 @@ class ReplayGainPlugin(BeetsPlugin): ) tag_vals = self.tag_specific_values(album.items()) - store_track_gain, store_album_gain = tag_vals + store_track_gain, store_album_gain, target_level, peak = tag_vals discs = dict() if self.per_disc: @@ -1309,9 +1315,7 @@ class ReplayGainPlugin(BeetsPlugin): for discnumber, items in discs.items(): try: album_gain = self.backend_instance.compute_album_gain( - items, - self.config['targetlevel'].as_number(), - self._peak_method, + items, target_level, peak ) if len(album_gain.track_gains) != len(items): raise ReplayGainError( @@ -1344,13 +1348,11 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info(u'analyzing {0}', item) tag_vals = self.tag_specific_values([item]) - store_track_gain, store_album_gain = tag_vals + store_track_gain, store_album_gain, target_level, peak = tag_vals try: track_gains = self.backend_instance.compute_track_gain( - [item], - self.config['targetlevel'].as_number(), - self._peak_method, + [item], target_level, peak ) if len(track_gains) != 1: raise ReplayGainError( From 5a8bdb67f7f2d24ac5b1af3fb69b3436a09e6d79 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Sat, 27 Oct 2018 22:39:45 +0200 Subject: [PATCH 129/613] replaygain: add target_level test Assert that analysing the same track with different target levels yields different gain adjustments. --- test/test_replaygain.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 9937cc7d0..8a0e3924b 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -143,6 +143,25 @@ class ReplayGainCliTestBase(TestHelper): self.assertNotEqual(max(gains), 0.0) self.assertNotEqual(max(peaks), 0.0) + def test_target_level_has_effect(self): + item = self.lib.items()[0] + + def analyse(target_level): + self.config['replaygain']['targetlevel'] = target_level + self._reset_replaygain(item) + self.run_command(u'replaygain', '-f') + mediafile = MediaFile(item.path) + return mediafile.rg_track_gain + + gain_relative_to_84 = analyse(84) + gain_relative_to_89 = analyse(89) + + # check that second calculation did work + if gain_relative_to_84 is not None: + self.assertIsNotNone(gain_relative_to_89) + + self.assertNotEqual(gain_relative_to_84, gain_relative_to_89) + @unittest.skipIf(not GST_AVAILABLE, u'gstreamer cannot be found') class ReplayGainGstCliTest(ReplayGainCliTestBase, unittest.TestCase): From 88ab5474c597d441e883d80dc47f6fe1fbe7c4e0 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Sat, 27 Oct 2018 22:49:21 +0200 Subject: [PATCH 130/613] replaygain: add R128_* tag test Assert that the replaygain plugin does not write REPLAYGAIN_* tags but R128_* tags, when instructed to do so. This test is skipped for the `command` backend as it does not support OPUS. --- test/test_replaygain.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 8a0e3924b..fe0515bee 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -87,6 +87,16 @@ class ReplayGainCliTestBase(TestHelper): self.teardown_beets() self.unload_plugins() + def _reset_replaygain(self, item): + item['rg_track_peak'] = None + item['rg_track_gain'] = None + item['rg_album_peak'] = None + item['rg_album_gain'] = None + item['r128_track_gain'] = None + item['r128_album_gain'] = None + item.write() + item.store() + def test_cli_saves_track_gain(self): for item in self.lib.items(): self.assertIsNone(item.rg_track_peak) @@ -143,6 +153,26 @@ class ReplayGainCliTestBase(TestHelper): self.assertNotEqual(max(gains), 0.0) self.assertNotEqual(max(peaks), 0.0) + def test_cli_writes_only_r128_tags(self): + if self.backend == "command": + # opus not supported by command backend + return + + album = self.add_album_fixture(2, ext="opus") + for item in album.items(): + self._reset_replaygain(item) + + self.run_command(u'replaygain', u'-a') + + for item in album.items(): + mediafile = MediaFile(item.path) + # does not write REPLAYGAIN_* tags + self.assertIsNone(mediafile.rg_track_gain) + self.assertIsNone(mediafile.rg_album_gain) + # writes R128_* tags + self.assertIsNotNone(mediafile.r128_track_gain) + self.assertIsNotNone(mediafile.r128_album_gain) + def test_target_level_has_effect(self): item = self.lib.items()[0] From fbc8cc484de67c497036a4f68302d8c0eb56b221 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Sat, 27 Oct 2018 21:30:18 +0200 Subject: [PATCH 131/613] update replaygain target level documentation - document `r128_targetlevel` - explain difference between `targetlevel` and `r128_targetlevel` - deprecate `method` option: use `targetlevel` instead. --- docs/plugins/replaygain.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index a68f3f7fc..fa120b35e 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -96,8 +96,13 @@ configuration file. The available options are: Default: ``command``. - **overwrite**: Re-analyze files that already have ReplayGain tags. Default: ``no``. -- **targetlevel**: A number of decibels for the target loudness level. - Default: 89. +- **targetlevel**: A number of decibels for the target loudness level for files + using ``REPLAYGAIN_`` tags. + Default: ``89``. +- **r128_targetlevel**: The target loudness level in decibels (i.e. + `` + 107``) for files using ``R128_`` tags. + Default: 84 (Use ``83`` for ATSC A/85, ``84`` for EBU R128 or ``89`` for + ReplayGain 2.0.) - **r128**: A space separated list of formats that will use ``R128_`` tags with integer values instead of the common ``REPLAYGAIN_`` tags with floating point values. Requires the "ffmpeg" backend. @@ -120,6 +125,13 @@ This option only works with the "ffmpeg" backend: - **peak**: Either ``true`` (the default) or ``sample``. ``true`` is more accurate but slower. +This option is deprecated: + +- **method**: The loudness scanning standard: either `replaygain` for + ReplayGain 2.0, `ebu` for EBU R128, or `atsc` for ATSC A/85. This dictates + the reference level: -18, -23, or -24 LUFS respectively. Only supported by + "bs1770gain" backend. + Manual Analysis --------------- From da602d779ce2b67101bfd72e2d9d266a6069e020 Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Wed, 7 Nov 2018 19:48:22 +0100 Subject: [PATCH 132/613] changelog: new `r128_targetlevel` option Add changelog entries for the introduction of the `r128_targetlevel` configuration option, superseding the `method` option. --- docs/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2b6740d48..3d55ab7e5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,13 @@ New features: new ``discogs_albumid`` field from the Discogs API. Thanks to :user:`thedevilisinthedetails`. :bug:`465` :bug:`3322` +* :doc:`plugins/replaygain`: ``r128_targetlevel`` is a new configuration option + for the ReplayGain plugin: It defines the reference volume for files using + ``R128_`` tags. ``targtelevel`` only configures the reference volume for + ``REPLAYGAIN_`` files. + This also deprecates the ``bs1770gain`` ReplayGain backend's ``method`` + option. Use ``targetlevel`` and ``r128_targetlevel`` instead. + :bug:`3065` Fixes: @@ -86,6 +93,10 @@ For plugin developers: * There were sporadic failures in ``test.test_player``. Hopefully these are fixed. If they resurface, please reopen the relevant issue. :bug:`3309` :bug:`3330` +* The internal structure of the replaygain plugin had some changes: There are no + longer separate R128 backend instances. Instead the targetlevel is passed to + ``compute_album_gain`` and ``compute_track_gain``. + :bug:`3065` For packagers: From d1ba309f36774fe71213c663a35dce49da8eae64 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Mon, 22 Jul 2019 16:55:46 +0200 Subject: [PATCH 133/613] Add a new method that copied pathlib.path.as_posix --- beets/util/__init__.py | 4 ++++ beetsplug/playlist.py | 11 +++++++++-- beetsplug/smartplaylist.py | 10 +++++++--- test/test_files.py | 4 ++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 29b2a73e7..9c87e7994 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -222,6 +222,10 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): for res in sorted_walk(cur, ignore, ignore_hidden, logger): yield res +def pathlib_as_posix(path): + """Return the string representation of the path with forward (/) + slashes.""" + return path.replace(b'\\', b'/') def mkdirall(path): """Make all the enclosing directories of path (like mkdir -p on the diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 4ab02c6b7..af98a250c 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -18,6 +18,8 @@ import os import fnmatch import tempfile import beets +from beets.util import (pathlib_as_posix) + class PlaylistQuery(beets.dbcore.Query): @@ -86,6 +88,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): 'auto': False, 'playlist_dir': '.', 'relative_to': 'library', + 'forward_slash': False, }) self.playlist_dir = self.config['playlist_dir'].as_filename() @@ -160,6 +163,8 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): try: new_path = self.changes[beets.util.normpath(lookup)] except KeyError: + if self.config['forward_slash'].get(): + line = pathlib_as_posix(line) tempfp.write(line) else: if new_path is None: @@ -170,8 +175,10 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): changes += 1 if is_relative: new_path = os.path.relpath(new_path, base_dir) - - tempfp.write(line.replace(original_path, new_path)) + line = line.replace(original_path, new_path) + if self.config['forward_slash'].get(): + line = pathlib_as_posix(line) + tempfp.write(line) if changes or deletions: self._log.info( diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index a83fc4d19..a440cb7fd 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -21,7 +21,7 @@ from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets import ui from beets.util import (mkdirall, normpath, sanitize_path, syspath, - bytestring_path) + bytestring_path, pathlib_as_posix) from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery from beets.dbcore.query import MultipleSort, ParsingError @@ -37,7 +37,8 @@ class SmartPlaylistPlugin(BeetsPlugin): 'relative_to': None, 'playlist_dir': u'.', 'auto': True, - 'playlists': [] + 'playlists': [], + 'forward_slash': False, }) self._matched_playlists = None @@ -206,6 +207,9 @@ class SmartPlaylistPlugin(BeetsPlugin): mkdirall(m3u_path) with open(syspath(m3u_path), 'wb') as f: for path in m3us[m3u]: - f.write(path + b'\n') + if self.config['forward_slash'].get(): + path = pathlib_as_posix(path) + f.write(path) + f.write(b'\n') self._log.info(u"{0} playlists updated", len(self._matched_playlists)) diff --git a/test/test_files.py b/test/test_files.py index ff055ac6f..87e4862f3 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -194,6 +194,10 @@ class HelperTest(_common.TestCase): p = 'a/b/c' a = ['a', 'b', 'c'] self.assertEqual(util.components(p), a) + def test_forward_slash(self): + p = r'C:\a\b\c' + a = r'C:/a/b/c' + self.assertEqual(util.pathlib_as_posix(p), a) class AlbumFileTest(_common.TestCase): From 1a4699d0ee080a02a1587c2fa612cd6fefeec685 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:46:31 +0200 Subject: [PATCH 134/613] Add documentation --- docs/plugins/playlist.rst | 5 +++++ docs/plugins/smartplaylist.rst | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 3622581db..4cc226265 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -11,6 +11,7 @@ Then configure your playlists like this:: auto: no relative_to: ~/Music playlist_dir: ~/.mpd/playlists + forward_slash: no It is possible to query the library based on a playlist by speicifying its absolute path:: @@ -45,3 +46,7 @@ other configuration options are: set it to ``playlist`` to use the playlist's parent directory or to ``library`` to use the library directory. Default: ``library`` +- **forward_slah**: Forces forward slashes in the generated playlist files. + If you intend to use this plugin to generate playlists for MPD on + Windows, set this to yes. + Default: Use system separator diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index e68217657..3d7e7a77b 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -14,6 +14,7 @@ Then configure your smart playlists like the following example:: smartplaylist: relative_to: ~/Music playlist_dir: ~/.mpd/playlists + forward_slah: no playlists: - name: all.m3u query: '' @@ -96,3 +97,7 @@ other configuration options are: directory. If you intend to use this plugin to generate playlists for MPD, point this to your MPD music directory. Default: Use absolute paths. +- **forward_slah**: Forces forward slashes in the generated playlist files. + If you intend to use this plugin to generate playlists for MPD on + Windows, set this to yes. + Default: Use system separator From da864402d5739733aad23736c14c9f65d8591812 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:49:17 +0200 Subject: [PATCH 135/613] Review: Remove unnecessary get --- beetsplug/playlist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index af98a250c..29639aba7 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -163,7 +163,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): try: new_path = self.changes[beets.util.normpath(lookup)] except KeyError: - if self.config['forward_slash'].get(): + if self.config['forward_slash']: line = pathlib_as_posix(line) tempfp.write(line) else: @@ -176,7 +176,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): if is_relative: new_path = os.path.relpath(new_path, base_dir) line = line.replace(original_path, new_path) - if self.config['forward_slash'].get(): + if self.config['forward_slash']: line = pathlib_as_posix(line) tempfp.write(line) From dd9de059688f401f7ca5269da53506af60f38b43 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:50:20 +0200 Subject: [PATCH 136/613] Review: Remove unnecessary split of concat --- beetsplug/smartplaylist.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index a440cb7fd..757879770 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -209,7 +209,6 @@ class SmartPlaylistPlugin(BeetsPlugin): for path in m3us[m3u]: if self.config['forward_slash'].get(): path = pathlib_as_posix(path) - f.write(path) - f.write(b'\n') + f.write(path + b'\n') self._log.info(u"{0} playlists updated", len(self._matched_playlists)) From 3a5ea58f7a9d9867d5e7e06e600be08bf6f13746 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:52:26 +0200 Subject: [PATCH 137/613] Review: format docstring on the following line --- beets/util/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 9c87e7994..2890576fd 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -224,7 +224,8 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): def pathlib_as_posix(path): """Return the string representation of the path with forward (/) - slashes.""" + slashes. + """ return path.replace(b'\\', b'/') def mkdirall(path): From aa1da3166fb80361bcf82c7c3cda10891865b6fe Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:53:17 +0200 Subject: [PATCH 138/613] Review: Remove unnecessary parens --- beetsplug/playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 29639aba7..464d69caf 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -18,7 +18,7 @@ import os import fnmatch import tempfile import beets -from beets.util import (pathlib_as_posix) +from beets.util import pathlib_as_posix From ee7f93933685f61192f3b2c0162310003e3190fc Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:53:50 +0200 Subject: [PATCH 139/613] Review: Remove stray blank line --- beetsplug/playlist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 464d69caf..3b8ca94c9 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -21,7 +21,6 @@ import beets from beets.util import pathlib_as_posix - class PlaylistQuery(beets.dbcore.Query): """Matches files listed by a playlist file. """ From 68ccfe0e6c62419010b09c5fbbcebf2dc36223dc Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:54:54 +0200 Subject: [PATCH 140/613] Review: Add missing blank line --- test/test_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_files.py b/test/test_files.py index 87e4862f3..969026b0f 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -194,6 +194,7 @@ class HelperTest(_common.TestCase): p = 'a/b/c' a = ['a', 'b', 'c'] self.assertEqual(util.components(p), a) + def test_forward_slash(self): p = r'C:\a\b\c' a = r'C:/a/b/c' From 076a82daa6f52dcc1176b3ae3b4b37dfabfaf8ef Mon Sep 17 00:00:00 2001 From: MartyLake Date: Tue, 23 Jul 2019 23:56:39 +0200 Subject: [PATCH 141/613] Review: Rename method --- beets/util/__init__.py | 2 +- beetsplug/playlist.py | 6 +++--- beetsplug/smartplaylist.py | 4 ++-- test/test_files.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 2890576fd..aae14ee63 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -222,7 +222,7 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): for res in sorted_walk(cur, ignore, ignore_hidden, logger): yield res -def pathlib_as_posix(path): +def path_as_posix(path): """Return the string representation of the path with forward (/) slashes. """ diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 3b8ca94c9..302ddb56d 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -18,7 +18,7 @@ import os import fnmatch import tempfile import beets -from beets.util import pathlib_as_posix +from beets.util import path_as_posix class PlaylistQuery(beets.dbcore.Query): @@ -163,7 +163,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): new_path = self.changes[beets.util.normpath(lookup)] except KeyError: if self.config['forward_slash']: - line = pathlib_as_posix(line) + line = path_as_posix(line) tempfp.write(line) else: if new_path is None: @@ -176,7 +176,7 @@ class PlaylistPlugin(beets.plugins.BeetsPlugin): new_path = os.path.relpath(new_path, base_dir) line = line.replace(original_path, new_path) if self.config['forward_slash']: - line = pathlib_as_posix(line) + line = path_as_posix(line) tempfp.write(line) if changes or deletions: diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 757879770..4ffdd21e7 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -21,7 +21,7 @@ from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets import ui from beets.util import (mkdirall, normpath, sanitize_path, syspath, - bytestring_path, pathlib_as_posix) + bytestring_path, path_as_posix) from beets.library import Item, Album, parse_query_string from beets.dbcore import OrQuery from beets.dbcore.query import MultipleSort, ParsingError @@ -208,7 +208,7 @@ class SmartPlaylistPlugin(BeetsPlugin): with open(syspath(m3u_path), 'wb') as f: for path in m3us[m3u]: if self.config['forward_slash'].get(): - path = pathlib_as_posix(path) + path = path_as_posix(path) f.write(path + b'\n') self._log.info(u"{0} playlists updated", len(self._matched_playlists)) diff --git a/test/test_files.py b/test/test_files.py index 969026b0f..6a6fa3531 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -198,7 +198,7 @@ class HelperTest(_common.TestCase): def test_forward_slash(self): p = r'C:\a\b\c' a = r'C:/a/b/c' - self.assertEqual(util.pathlib_as_posix(p), a) + self.assertEqual(util.path_as_posix(p), a) class AlbumFileTest(_common.TestCase): From 5f9a394ca9b4f04c612b054baf102dda3ceedfce Mon Sep 17 00:00:00 2001 From: Paul Malcolm Date: Tue, 23 Jul 2019 19:55:28 -0400 Subject: [PATCH 142/613] Issue #2860 Fetch more acousticbrainz fields --- beetsplug/acousticbrainz.py | 8 ++++++++ docs/plugins/acousticbrainz.rst | 2 ++ test/test_acousticbrainz.py | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 01f3ac6ac..725e0d634 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -74,6 +74,9 @@ ABSCHEME = { 'sad': 'mood_sad' } }, + 'moods_mirex': { + 'value': 'moods_mirex' + }, 'ismir04_rhythm': { 'value': 'rhythm' }, @@ -82,6 +85,9 @@ ABSCHEME = { 'tonal': 'tonal' } }, + 'timbre': { + 'value': 'timbre' + }, 'voice_instrumental': { 'value': 'voice_instrumental' }, @@ -124,7 +130,9 @@ class AcousticPlugin(plugins.BeetsPlugin): 'mood_party': types.Float(6), 'mood_relaxed': types.Float(6), 'mood_sad': types.Float(6), + 'moods_mirex': types.STRING, 'rhythm': types.Float(6), + 'timbre': types.STRING, 'tonal': types.Float(6), 'voice_instrumental': types.STRING, } diff --git a/docs/plugins/acousticbrainz.rst b/docs/plugins/acousticbrainz.rst index 7c24ffe0d..7d7aed237 100644 --- a/docs/plugins/acousticbrainz.rst +++ b/docs/plugins/acousticbrainz.rst @@ -38,7 +38,9 @@ these fields: * ``mood_party`` * ``mood_relaxed`` * ``mood_sad`` +* ``moods_mirex`` * ``rhythm`` +* ``timbre`` * ``tonal`` * ``voice_instrumental`` diff --git a/test/test_acousticbrainz.py b/test/test_acousticbrainz.py index 0b1407581..4c0b0137b 100644 --- a/test/test_acousticbrainz.py +++ b/test/test_acousticbrainz.py @@ -95,7 +95,9 @@ class MapDataToSchemeTest(unittest.TestCase): ('danceable', 0.143928021193), ('rhythm', 'VienneseWaltz'), ('mood_electronic', 0.339881360531), - ('mood_happy', 0.0894767045975) + ('mood_happy', 0.0894767045975), + ('moods_mirex', "Cluster3"), + ('timbre', "bright") } self.assertEqual(mapping, expected) From aa176255512be698bddb8c66c1c9924b155561d6 Mon Sep 17 00:00:00 2001 From: Paul Malcolm Date: Tue, 23 Jul 2019 19:56:02 -0400 Subject: [PATCH 143/613] changelog entry: additional acousticbrainz fields --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2b6740d48..cd6863781 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,10 @@ New features: new ``discogs_albumid`` field from the Discogs API. Thanks to :user:`thedevilisinthedetails`. :bug:`465` :bug:`3322` +* :doc:`/plugins/acousticbrainz`: The plugin now fetches two more additional + fields: ``moods_mirex`` and ``timbre``. + Thanks to :user:`malcops`. + :bug:`2860` Fixes: From c52973e1c06470ff9ecd948c51b85620c54a2712 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Wed, 24 Jul 2019 09:53:54 +0200 Subject: [PATCH 144/613] Review: Adds missing point to finish sentences --- docs/plugins/playlist.rst | 2 +- docs/plugins/smartplaylist.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 4cc226265..5abbe6dcf 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -49,4 +49,4 @@ other configuration options are: - **forward_slah**: Forces forward slashes in the generated playlist files. If you intend to use this plugin to generate playlists for MPD on Windows, set this to yes. - Default: Use system separator + Default: Use system separator. diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index 3d7e7a77b..2aef5e8f8 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -100,4 +100,4 @@ other configuration options are: - **forward_slah**: Forces forward slashes in the generated playlist files. If you intend to use this plugin to generate playlists for MPD on Windows, set this to yes. - Default: Use system separator + Default: Use system separator. From 7ee11b0f1a6ea6adf23ef82957ed1c2e092c30e4 Mon Sep 17 00:00:00 2001 From: MartyLake Date: Wed, 24 Jul 2019 10:08:40 +0200 Subject: [PATCH 145/613] Review: add missing lines --- beets/util/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index aae14ee63..bb84aedc7 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -222,12 +222,14 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): for res in sorted_walk(cur, ignore, ignore_hidden, logger): yield res + def path_as_posix(path): """Return the string representation of the path with forward (/) slashes. """ return path.replace(b'\\', b'/') + def mkdirall(path): """Make all the enclosing directories of path (like mkdir -p on the parent). From 819f03f0e046bbad14d911bafa054a204101b41a Mon Sep 17 00:00:00 2001 From: MartyLake Date: Wed, 24 Jul 2019 10:11:32 +0200 Subject: [PATCH 146/613] Add special case for string --- beets/util/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index bb84aedc7..cc23b6874 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -227,6 +227,8 @@ def path_as_posix(path): """Return the string representation of the path with forward (/) slashes. """ + if isinstance(path, str): + return path.replace('\\', '/') return path.replace(b'\\', b'/') From fb9666017133cd3951d8b993c9eef35a76d7753f Mon Sep 17 00:00:00 2001 From: MartyLake Date: Wed, 24 Jul 2019 18:09:54 +0200 Subject: [PATCH 147/613] Review: simpler implementation and test Because **all** the path are bytestrings --- beets/util/__init__.py | 2 -- test/test_files.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index cc23b6874..bb84aedc7 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -227,8 +227,6 @@ def path_as_posix(path): """Return the string representation of the path with forward (/) slashes. """ - if isinstance(path, str): - return path.replace('\\', '/') return path.replace(b'\\', b'/') diff --git a/test/test_files.py b/test/test_files.py index 6a6fa3531..f31779672 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -196,8 +196,8 @@ class HelperTest(_common.TestCase): self.assertEqual(util.components(p), a) def test_forward_slash(self): - p = r'C:\a\b\c' - a = r'C:/a/b/c' + p = br'C:\a\b\c' + a = br'C:/a/b/c' self.assertEqual(util.path_as_posix(p), a) From 9392256993b0cf6f4e2a545d9c504decb71542d9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 24 Jul 2019 22:13:53 -0400 Subject: [PATCH 148/613] Spelling & changelog for #3334 --- docs/changelog.rst | 5 +++++ docs/plugins/playlist.rst | 2 +- docs/plugins/smartplaylist.rst | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cd6863781..e6e846eef 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,6 +46,11 @@ New features: fields: ``moods_mirex`` and ``timbre``. Thanks to :user:`malcops`. :bug:`2860` +* :doc:`/plugins/playlist` and :doc:`/plugins/smartplaylist`: A new + ``forward_slash`` config option facilitates compatibility with MPD on + Windows. + Thanks to :user:`MartyLake`. + :bug:`3331` :bug:`3334` Fixes: diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 5abbe6dcf..31609cb3c 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -46,7 +46,7 @@ other configuration options are: set it to ``playlist`` to use the playlist's parent directory or to ``library`` to use the library directory. Default: ``library`` -- **forward_slah**: Forces forward slashes in the generated playlist files. +- **forward_slash**: Forces forward slashes in the generated playlist files. If you intend to use this plugin to generate playlists for MPD on Windows, set this to yes. Default: Use system separator. diff --git a/docs/plugins/smartplaylist.rst b/docs/plugins/smartplaylist.rst index 2aef5e8f8..dd3ee45ba 100644 --- a/docs/plugins/smartplaylist.rst +++ b/docs/plugins/smartplaylist.rst @@ -14,7 +14,7 @@ Then configure your smart playlists like the following example:: smartplaylist: relative_to: ~/Music playlist_dir: ~/.mpd/playlists - forward_slah: no + forward_slash: no playlists: - name: all.m3u query: '' @@ -97,7 +97,7 @@ other configuration options are: directory. If you intend to use this plugin to generate playlists for MPD, point this to your MPD music directory. Default: Use absolute paths. -- **forward_slah**: Forces forward slashes in the generated playlist files. +- **forward_slash**: Forces forward slashes in the generated playlist files. If you intend to use this plugin to generate playlists for MPD on Windows, set this to yes. Default: Use system separator. From a9f70f81518903b26968056cc81b03589a90e10c Mon Sep 17 00:00:00 2001 From: Zsin Skri Date: Thu, 25 Jul 2019 23:19:10 +0200 Subject: [PATCH 149/613] apply suggested improvements Apply improvements suggested in GitHub PullRequest #3065: - be idiomatic - 0 is falsy - check enum equality, not identity - mutate list by constructing a new one - improve documentation - fix a typo - do not mention deprecation of a config option --- beetsplug/replaygain.py | 17 +++++++---------- docs/changelog.rst | 2 +- docs/plugins/replaygain.rst | 7 ------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index a0108b5a0..a618939b8 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -268,10 +268,11 @@ class Bs1770gainBackend(Backend): self._log.debug(u'analysis finished: {0}', output) results = self.parse_tool_output(output, path_list, is_album) - if gain_adjustment != 0: - for i in range(len(results)): - orig = results[i] - results[i] = Gain(orig.gain + gain_adjustment, orig.peak) + if gain_adjustment: + results = [ + Gain(res.gain + gain_adjustment, res.peak) + for res in results + ] self._log.debug(u'{0} items, {1} results', len(items), len(results)) return results @@ -470,11 +471,7 @@ class FfmpegBackend(Backend): will be 0. """ target_level_lufs = db_to_lufs(target_level) - peak_method = { - Peak.none: "none", - Peak.true: "true", - Peak.sample: "sample", - }[peak] + peak_method = peak.name # call ffmpeg self._log.debug(u"analyzing {0}".format(item)) @@ -486,7 +483,7 @@ class FfmpegBackend(Backend): # parse output - if peak is Peak.none: + if peak == Peak.none: peak = 0 else: line_peak = self._find_line( diff --git a/docs/changelog.rst b/docs/changelog.rst index 3d55ab7e5..a13f46794 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,7 +44,7 @@ New features: :bug:`465` :bug:`3322` * :doc:`plugins/replaygain`: ``r128_targetlevel`` is a new configuration option for the ReplayGain plugin: It defines the reference volume for files using - ``R128_`` tags. ``targtelevel`` only configures the reference volume for + ``R128_`` tags. ``targetlevel`` only configures the reference volume for ``REPLAYGAIN_`` files. This also deprecates the ``bs1770gain`` ReplayGain backend's ``method`` option. Use ``targetlevel`` and ``r128_targetlevel`` instead. diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index fa120b35e..ea119167d 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -125,13 +125,6 @@ This option only works with the "ffmpeg" backend: - **peak**: Either ``true`` (the default) or ``sample``. ``true`` is more accurate but slower. -This option is deprecated: - -- **method**: The loudness scanning standard: either `replaygain` for - ReplayGain 2.0, `ebu` for EBU R128, or `atsc` for ATSC A/85. This dictates - the reference level: -18, -23, or -24 LUFS respectively. Only supported by - "bs1770gain" backend. - Manual Analysis --------------- From 60c174101fd50aaa892dd2555c5777468e8fa409 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Mon, 29 Jul 2019 10:32:19 +0200 Subject: [PATCH 150/613] ffmpeg replaygain backend: Only calculate replaygain for audio stream. Fixed documentation for backend option. --- beetsplug/replaygain.py | 1 + docs/plugins/replaygain.rst | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index a618939b8..f9373835f 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -456,6 +456,7 @@ class FfmpegBackend(Backend): "-hide_banner", "-i", item.path, + "-map a:0", "-filter", "ebur128=peak={0}".format(peak_method), "-f", diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index ea119167d..9602618da 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -92,7 +92,8 @@ configuration file. The available options are: - **auto**: Enable ReplayGain analysis during import. Default: ``yes``. -- **backend**: The analysis backend; either ``gstreamer``, ``command``, or ``audiotools``. +- **backend**: The analysis backend; either ``gstreamer``, ``command``, ``audiotools`` + or ``ffmpeg``. Default: ``command``. - **overwrite**: Re-analyze files that already have ReplayGain tags. Default: ``no``. From 49080289853f65cfbfb7d4aebfbf61aed864ede0 Mon Sep 17 00:00:00 2001 From: Guilherme Danno Date: Tue, 30 Jul 2019 16:46:59 -0300 Subject: [PATCH 151/613] Use the 'resource_url' from discogs_client --- beetsplug/discogs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 3ba68e2ce..37ff68fa4 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -208,7 +208,8 @@ class DiscogsPlugin(BeetsPlugin): getattr(result, 'title') except DiscogsAPIError as e: if e.status_code != 404: - self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) + self._log.debug(u'API Error: {0} (query: {1})', e, + result.data['resource_url']) if e.status_code == 401: self.reset_auth() return self.album_for_id(album_id) @@ -260,7 +261,8 @@ class DiscogsPlugin(BeetsPlugin): return year except DiscogsAPIError as e: if e.status_code != 404: - self._log.debug(u'API Error: {0} (query: {1})', e, result._uri) + self._log.debug(u'API Error: {0} (query: {1})', e, + result.data['resource_url']) if e.status_code == 401: self.reset_auth() return self.get_master_year(master_id) From 63594386c2ac35c7af1c70f1d1b8850e46b30f78 Mon Sep 17 00:00:00 2001 From: Guilherme Danno Date: Tue, 30 Jul 2019 17:01:55 -0300 Subject: [PATCH 152/613] Add the changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b5a500fda..0252104fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -79,6 +79,9 @@ Fixes: :bug:`3301` * :doc:`plugins/replaygain`: Fix the storage format in R128 gain tags. :bug:`3311` :bug:`3314` +* :doc:`/plugins/discogs`: Fixed a crash that occurred when the Master URI + isn't set + :bug:`2965` :bug:`3239` For plugin developers: From 5f6d1f0f964675dd101b234c991f5da579986736 Mon Sep 17 00:00:00 2001 From: octos Date: Wed, 31 Jul 2019 22:41:32 -0500 Subject: [PATCH 153/613] improve readability --- docs/plugins/zero.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/plugins/zero.rst b/docs/plugins/zero.rst index 3ef241c87..6d398f2d9 100644 --- a/docs/plugins/zero.rst +++ b/docs/plugins/zero.rst @@ -5,9 +5,10 @@ The ``zero`` plugin allows you to null fields in files' metadata tags. Fields can be nulled unconditionally or conditioned on a pattern match. For example, the plugin can strip useless comments like "ripped by MyGreatRipper." -The plugin can work in one of two modes. The first mode, the default, is a -blacklist, where you choose the tags you want to remove. The second mode is a -whitelist, where you instead specify the tags you want to keep. +The plugin can work in one of two modes: + +* ``fields`` - a blacklist, where you choose the tags you want to remove (used by default). +* ``keep_fields`` - a whitelist, where you instead specify the tags you want to keep. To use the ``zero`` plugin, enable the plugin in your configuration (see :ref:`using-plugins`). @@ -20,17 +21,17 @@ fields to nullify and the conditions for nullifying them: * Set ``auto`` to ``yes`` to null fields automatically on import. Default: ``yes``. -* Set ``fields`` to a whitespace-separated list of fields to change. You can +* Set ``fields`` to a whitespace-separated list of fields to remove. You can get the list of all available fields by running ``beet fields``. In addition, the ``images`` field allows you to remove any images embedded in the media file. * Set ``keep_fields`` to *invert* the logic of the plugin. Only these fields will be kept; other fields will be removed. Remember to set only - ``fields`` or ``keep_fields``---not both! + ``fields`` or ``keep_fields`` -- not both! * To conditionally filter a field, use ``field: [regexp, regexp]`` to specify regular expressions. -* By default this plugin only affects files' tags ; the beets database is left - unchanged. To update the tags in the database, set the ``update_database`` option. +* By default this plugin only affects files' tags; the beets database is left + unchanged. To update the tags in the database, set the ``update_database`` option to true. For example:: From c74c9a46a9026c3c550c0c75d12eaf5dee854c5a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 1 Aug 2019 09:32:33 -0400 Subject: [PATCH 154/613] Punctuation improvements --- docs/plugins/zero.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/zero.rst b/docs/plugins/zero.rst index 6d398f2d9..1ed915891 100644 --- a/docs/plugins/zero.rst +++ b/docs/plugins/zero.rst @@ -7,8 +7,8 @@ the plugin can strip useless comments like "ripped by MyGreatRipper." The plugin can work in one of two modes: -* ``fields`` - a blacklist, where you choose the tags you want to remove (used by default). -* ``keep_fields`` - a whitelist, where you instead specify the tags you want to keep. +* ``fields``: A blacklist, where you choose the tags you want to remove (used by default). +* ``keep_fields``: A whitelist, where you instead specify the tags you want to keep. To use the ``zero`` plugin, enable the plugin in your configuration (see :ref:`using-plugins`). @@ -27,7 +27,7 @@ fields to nullify and the conditions for nullifying them: embedded in the media file. * Set ``keep_fields`` to *invert* the logic of the plugin. Only these fields will be kept; other fields will be removed. Remember to set only - ``fields`` or ``keep_fields`` -- not both! + ``fields`` or ``keep_fields``---not both! * To conditionally filter a field, use ``field: [regexp, regexp]`` to specify regular expressions. * By default this plugin only affects files' tags; the beets database is left From 6e24669d61279c8c58022ba5a965ad738f07bdc8 Mon Sep 17 00:00:00 2001 From: Samuel Nilsson Date: Sat, 3 Aug 2019 22:51:40 +0200 Subject: [PATCH 155/613] Fix #3341 --- beetsplug/replaygain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index f9373835f..e9d5cc4af 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -456,7 +456,8 @@ class FfmpegBackend(Backend): "-hide_banner", "-i", item.path, - "-map a:0", + "-map", + "a:0", "-filter", "ebur128=peak={0}".format(peak_method), "-f", From 7ec363230903ebc7dfcbf0660efb3be78c2e63fd Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 3 Aug 2019 19:07:56 -0700 Subject: [PATCH 156/613] Fix `year` assignment with year-only release date --- beetsplug/spotify.py | 2 +- docs/changelog.rst | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d8d7637d6..0af0dc9aa 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -173,7 +173,7 @@ class SpotifyPlugin(BeetsPlugin): year, month = date_parts day = None elif release_date_precision == 'year': - year = date_parts + year = date_parts[0] month = None day = None else: diff --git a/docs/changelog.rst b/docs/changelog.rst index 0252104fe..9871d8305 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -80,8 +80,10 @@ Fixes: * :doc:`plugins/replaygain`: Fix the storage format in R128 gain tags. :bug:`3311` :bug:`3314` * :doc:`/plugins/discogs`: Fixed a crash that occurred when the Master URI - isn't set + isn't set. :bug:`2965` :bug:`3239` +* :doc:`/plugins/spotify`: Fix handling of year-only release dates + returned by Spotify Albums API. For plugin developers: From 75eb2f4621f753e3aaf683734b53d039d317b96f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 3 Aug 2019 22:40:50 -0400 Subject: [PATCH 157/613] Changelog thanks & bug link to #3343 --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9871d8305..e6ee7df3c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -82,8 +82,10 @@ Fixes: * :doc:`/plugins/discogs`: Fixed a crash that occurred when the Master URI isn't set. :bug:`2965` :bug:`3239` -* :doc:`/plugins/spotify`: Fix handling of year-only release dates +* :doc:`/plugins/spotify`: Fix handling of year-only release dates returned by Spotify Albums API. + Thanks to :user:`rhlahuja`. + :bug:`3343` For plugin developers: From a2ee8da8d62eaed5878c4694cf7a4f42b1d55ecf Mon Sep 17 00:00:00 2001 From: Sebastian Pucilowski Date: Sun, 11 Aug 2019 11:21:22 +1000 Subject: [PATCH 158/613] Refactor magic values in discogs_client --- beetsplug/discogs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 37ff68fa4..4996c5d7c 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -38,6 +38,8 @@ from string import ascii_lowercase USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) +API_KEY = 'rAzVUQYRaoFjeBjyWuWZ' +API_SECRET = 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy' # Exceptions that discogs_client should really handle but does not. CONNECTION_ERRORS = (ConnectionError, socket.error, http_client.HTTPException, @@ -50,8 +52,8 @@ class DiscogsPlugin(BeetsPlugin): def __init__(self): super(DiscogsPlugin, self).__init__() self.config.add({ - 'apikey': 'rAzVUQYRaoFjeBjyWuWZ', - 'apisecret': 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy', + 'apikey': API_KEY, + 'apisecret': API_SECRET, 'tokenfile': 'discogs_token.json', 'source_weight': 0.5, 'user_token': '', From 0bc3727fcf339c5f42204b8d229b0a83c25e7918 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 19 Aug 2019 23:23:00 +0200 Subject: [PATCH 159/613] Correctly display track number 0 in show_change fixes issue #3346: When the per_disc_numbering option was set, the UI would previously show a #0 -> #1 change when actually the index would be set to 0 (a valid index, such as for hidden tracks). Now, properly distinguish index 0 and None (i.e. not set) --- beets/ui/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 53253c1da..57b627c77 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -241,7 +241,8 @@ def show_change(cur_artist, cur_album, match): if mediums and mediums > 1: return u'{0}-{1}'.format(medium, medium_index) else: - return six.text_type(medium_index or index) + return six.text_type(medium_index if medium_index is not None + else index) else: return six.text_type(index) From 29d6967fa7d6d04a8df8a98c23c623ed6786586b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 20 Aug 2019 09:29:35 +0200 Subject: [PATCH 160/613] update changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e6ee7df3c..e394a4cd9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -86,6 +86,9 @@ Fixes: returned by Spotify Albums API. Thanks to :user:`rhlahuja`. :bug:`3343` +* Fixed a bug that caused the UI to display incorrect track numbers for tracks + with index 0 when the ``per_disc_numbering`` option was set. + :bug:`3346` For plugin developers: From 4820cee35cb9ea07e26b8a98a5445782f6ba27c4 Mon Sep 17 00:00:00 2001 From: Kier Davis Date: Tue, 20 Aug 2019 14:38:25 +0200 Subject: [PATCH 161/613] convert: add option to symlink instead of copying As proposed in #2324. Updated commit from #2326. Co-authored-by: Vexatos --- beetsplug/convert.py | 73 +++++++++++++++++++++++++++++----------- docs/changelog.rst | 3 ++ docs/plugins/convert.rst | 11 ++++++ 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 6ed139da0..47ae9fbba 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -114,6 +114,7 @@ class ConvertPlugin(BeetsPlugin): self.config.add({ u'dest': None, u'pretend': False, + u'link': False, u'threads': util.cpu_count(), u'format': u'mp3', u'id3v23': u'inherit', @@ -167,6 +168,9 @@ class ConvertPlugin(BeetsPlugin): help=u'set the target format of the tracks') cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes', help=u'do not ask for confirmation') + cmd.parser.add_option('-l', '--link', action='store_true', dest='link', + help=u'symlink files that do not need transcoding. \ + Don\'t use with \'embed\'.') cmd.parser.add_album_option() cmd.func = self.convert_func return [cmd] @@ -251,7 +255,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(source)) def convert_item(self, dest_dir, keep_new, path_formats, fmt, - pretend=False): + pretend=False, link=False): """A pipeline thread that converts `Item` objects from a library. """ @@ -304,15 +308,26 @@ class ConvertPlugin(BeetsPlugin): except subprocess.CalledProcessError: continue else: - if pretend: - self._log.info(u'cp {0} {1}', - util.displayable_path(original), - util.displayable_path(converted)) + if link: + if pretend: + self._log.info(u'ln -s {0} {1}', + util.displayable_path(original), + util.displayable_path(converted)) + else: + # No transcoding necessary. + self._log.info(u'Linking {0}', + util.displayable_path(item.path)) + util.link(original, converted) else: - # No transcoding necessary. - self._log.info(u'Copying {0}', - util.displayable_path(item.path)) - util.copy(original, converted) + if pretend: + self._log.info(u'cp {0} {1}', + util.displayable_path(original), + util.displayable_path(converted)) + else: + # No transcoding necessary. + self._log.info(u'Copying {0}', + util.displayable_path(item.path)) + util.copy(original, converted) if pretend: continue @@ -346,7 +361,8 @@ class ConvertPlugin(BeetsPlugin): plugins.send('after_convert', item=item, dest=converted, keepnew=False) - def copy_album_art(self, album, dest_dir, path_formats, pretend=False): + def copy_album_art(self, album, dest_dir, path_formats, pretend=False, + link=False): """Copies or converts the associated cover art of the album. Album must have at least one track. """ @@ -399,15 +415,26 @@ class ConvertPlugin(BeetsPlugin): if not pretend: ArtResizer.shared.resize(maxwidth, album.artpath, dest) else: - if pretend: - self._log.info(u'cp {0} {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) + if link: + if pretend: + self._log.info(u'ln -s {0} {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + else: + self._log.info(u'Linking cover art from {0} to {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + util.link(album.artpath, dest) else: - self._log.info(u'Copying cover art from {0} to {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) - util.copy(album.artpath, dest) + if pretend: + self._log.info(u'cp {0} {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + else: + self._log.info(u'Copying cover art from {0} to {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + util.copy(album.artpath, dest) def convert_func(self, lib, opts, args): dest = opts.dest or self.config['dest'].get() @@ -426,6 +453,11 @@ class ConvertPlugin(BeetsPlugin): else: pretend = self.config['pretend'].get(bool) + if opts.link is not None: + link = opts.link + else: + link = self.config['link'].get(bool) + if opts.album: albums = lib.albums(ui.decargs(args)) items = [i for a in albums for i in a.items()] @@ -446,13 +478,14 @@ class ConvertPlugin(BeetsPlugin): if opts.album and self.config['copy_album_art']: for album in albums: - self.copy_album_art(album, dest, path_formats, pretend) + self.copy_album_art(album, dest, path_formats, pretend, link) convert = [self.convert_item(dest, opts.keep_new, path_formats, fmt, - pretend) + pretend, + link) for _ in range(threads)] pipe = util.pipeline.Pipeline([iter(items), convert]) pipe.run_parallel() diff --git a/docs/changelog.rst b/docs/changelog.rst index e394a4cd9..310e5fb0d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,9 @@ New features: (the MBID), and ``work_disambig`` (the disambiguation string). Thanks to :user:`dosoe`. :bug:`2580` :bug:`3272` +* :doc:`/plugins/convert`: Added new ``-l`` (``--link``) flag and ``link`` option + which symlinks files that do not need to be converted instead of copying them. + :bug:`2324` * :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 of the MPD protocol. This is enough to get it talking to more complicated clients like ncmpcpp, but there are still some incompatibilities, largely due diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 72ea301f1..775434d66 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -48,6 +48,10 @@ To test your configuration without taking any actions, use the ``--pretend`` flag. The plugin will print out the commands it will run instead of executing them. +By default, files that do not need to be transcoded will be copied to their +destination. Passing the ``-l`` (``--link``) flag creates symbolic links +instead. Refer to the ``link`` option below for potential issues with this. + Configuration ------------- @@ -93,6 +97,13 @@ file. The available options are: - **threads**: The number of threads to use for parallel encoding. By default, the plugin will detect the number of processors available and use them all. +- **link**: By default, files that do not need to be transcoded will be copied + to their destination. This option creates symbolic links instead. Note that + options such as ``embed`` that modify the output files after the transcoding + step will cause the original files to be modified as well if ``link`` is + enabled. For this reason, it is highly recommended not use to ``link`` and + ``embed`` at the same time. + Default: ``false``. You can also configure the format to use for transcoding (see the next section): From a61aa7406122682c725a850353a9f937e74b9517 Mon Sep 17 00:00:00 2001 From: Vexatos Date: Tue, 20 Aug 2019 16:07:01 +0200 Subject: [PATCH 162/613] convert: add option to hardlink instead of copying. Overrides the --link option. As proposed in #2324. --- beetsplug/convert.py | 42 ++++++++++++++++++++++++++++++++++------ docs/changelog.rst | 4 ++++ docs/plugins/convert.rst | 13 ++++++++++++- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 47ae9fbba..cee1e080a 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -115,6 +115,7 @@ class ConvertPlugin(BeetsPlugin): u'dest': None, u'pretend': False, u'link': False, + u'hardlink': False, u'threads': util.cpu_count(), u'format': u'mp3', u'id3v23': u'inherit', @@ -171,6 +172,9 @@ class ConvertPlugin(BeetsPlugin): cmd.parser.add_option('-l', '--link', action='store_true', dest='link', help=u'symlink files that do not need transcoding. \ Don\'t use with \'embed\'.') + cmd.parser.add_option('-H', '--hardlink', action='store_true', dest='hardlink', + help=u'hardlink files that do not need transcoding. \ + Overrides --link. Don\'t use with \'embed\'.') cmd.parser.add_album_option() cmd.func = self.convert_func return [cmd] @@ -255,7 +259,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(source)) def convert_item(self, dest_dir, keep_new, path_formats, fmt, - pretend=False, link=False): + pretend=False, link=False, hardlink=False): """A pipeline thread that converts `Item` objects from a library. """ @@ -308,7 +312,17 @@ class ConvertPlugin(BeetsPlugin): except subprocess.CalledProcessError: continue else: - if link: + if hardlink: + if pretend: + self._log.info(u'ln {0} {1}', + util.displayable_path(original), + util.displayable_path(converted)) + else: + # No transcoding necessary. + self._log.info(u'Hardlinking {0}', + util.displayable_path(item.path)) + util.hardlink(original, converted) + elif link: if pretend: self._log.info(u'ln -s {0} {1}', util.displayable_path(original), @@ -362,7 +376,7 @@ class ConvertPlugin(BeetsPlugin): dest=converted, keepnew=False) def copy_album_art(self, album, dest_dir, path_formats, pretend=False, - link=False): + link=False, hardlink=False): """Copies or converts the associated cover art of the album. Album must have at least one track. """ @@ -415,7 +429,17 @@ class ConvertPlugin(BeetsPlugin): if not pretend: ArtResizer.shared.resize(maxwidth, album.artpath, dest) else: - if link: + if hardlink: + if pretend: + self._log.info(u'ln {0} {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + else: + self._log.info(u'Hardlinking cover art from {0} to {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest)) + util.hardlink(album.artpath, dest) + elif link: if pretend: self._log.info(u'ln -s {0} {1}', util.displayable_path(album.artpath), @@ -458,6 +482,11 @@ class ConvertPlugin(BeetsPlugin): else: link = self.config['link'].get(bool) + if opts.hardlink is not None: + hardlink = opts.hardlink + else: + hardlink = self.config['hardlink'].get(bool) + if opts.album: albums = lib.albums(ui.decargs(args)) items = [i for a in albums for i in a.items()] @@ -478,14 +507,15 @@ class ConvertPlugin(BeetsPlugin): if opts.album and self.config['copy_album_art']: for album in albums: - self.copy_album_art(album, dest, path_formats, pretend, link) + self.copy_album_art(album, dest, path_formats, pretend, link, hardlink) convert = [self.convert_item(dest, opts.keep_new, path_formats, fmt, pretend, - link) + link, + hardlink) for _ in range(threads)] pipe = util.pipeline.Pipeline([iter(items), convert]) pipe.run_parallel() diff --git a/docs/changelog.rst b/docs/changelog.rst index 310e5fb0d..c35114390 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,6 +14,10 @@ New features: * :doc:`/plugins/convert`: Added new ``-l`` (``--link``) flag and ``link`` option which symlinks files that do not need to be converted instead of copying them. :bug:`2324` +* :doc:`/plugins/convert`: Added new ``-H`` (``--hardlink``) flag and ``hardlink`` + option which hardlinks files that do not need to be converted + instead of copying them. + :bug:`2324` * :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 of the MPD protocol. This is enough to get it talking to more complicated clients like ncmpcpp, but there are still some incompatibilities, largely due diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 775434d66..15b4fd2f2 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -50,7 +50,9 @@ them. By default, files that do not need to be transcoded will be copied to their destination. Passing the ``-l`` (``--link``) flag creates symbolic links -instead. Refer to the ``link`` option below for potential issues with this. +instead, passing ``-H`` (``--hardlink``) creates hard links. +Refer to the ``link`` and ``hardlink`` options below +for potential issues with this. Configuration @@ -104,6 +106,15 @@ file. The available options are: enabled. For this reason, it is highly recommended not use to ``link`` and ``embed`` at the same time. Default: ``false``. +- **hardlink**: By default, files that do not need to be transcoded will be + copied to their destination. This option creates hard links instead. Note that + options such as ``embed`` that modify the output files after the transcoding + step will cause the original files to be modified as well if ``hardlink`` is + enabled. For this reason, it is highly recommended not use to ``hardlink`` and + ``embed`` at the same time. + This option overrides ``link``. Only works when converting to a directory + on the same filesystem as the library. + Default: ``false``. You can also configure the format to use for transcoding (see the next section): From aeb7d8846e6397d618ce3151008de831dd7351e7 Mon Sep 17 00:00:00 2001 From: Vexatos Date: Tue, 20 Aug 2019 16:26:43 +0200 Subject: [PATCH 163/613] convert: disable album-art embedding for linked files. Fixed flag precedence of link and hardlink over their options. Fixed formatting issue. --- beetsplug/convert.py | 33 ++++++++++++++++++++++----------- docs/plugins/convert.rst | 8 ++++---- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index cee1e080a..a0afdd722 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -170,11 +170,12 @@ class ConvertPlugin(BeetsPlugin): cmd.parser.add_option('-y', '--yes', action='store_true', dest='yes', help=u'do not ask for confirmation') cmd.parser.add_option('-l', '--link', action='store_true', dest='link', - help=u'symlink files that do not need transcoding. \ - Don\'t use with \'embed\'.') - cmd.parser.add_option('-H', '--hardlink', action='store_true', dest='hardlink', - help=u'hardlink files that do not need transcoding. \ - Overrides --link. Don\'t use with \'embed\'.') + help=u'symlink files that do not \ + need transcoding.') + cmd.parser.add_option('-H', '--hardlink', action='store_true', + dest='hardlink', + help=u'hardlink files that do not \ + need transcoding. Overrides --link.') cmd.parser.add_album_option() cmd.func = self.convert_func return [cmd] @@ -306,6 +307,8 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(original)) util.move(item.path, original) + linked = False + if should_transcode(item, fmt): try: self.encode(command, original, converted, pretend) @@ -322,6 +325,7 @@ class ConvertPlugin(BeetsPlugin): self._log.info(u'Hardlinking {0}', util.displayable_path(item.path)) util.hardlink(original, converted) + linked = True elif link: if pretend: self._log.info(u'ln -s {0} {1}', @@ -332,6 +336,7 @@ class ConvertPlugin(BeetsPlugin): self._log.info(u'Linking {0}', util.displayable_path(item.path)) util.link(original, converted) + linked = True else: if pretend: self._log.info(u'cp {0} {1}', @@ -360,7 +365,7 @@ class ConvertPlugin(BeetsPlugin): item.read() item.store() # Store new path and audio data. - if self.config['embed']: + if self.config['embed'] and not linked: album = item.get_album() if album and album.artpath: self._log.debug(u'embedding album art from {}', @@ -479,13 +484,18 @@ class ConvertPlugin(BeetsPlugin): if opts.link is not None: link = opts.link + + if opts.hardlink is not None: + hardlink = opts.hardlink + else: + hardlink = self.config['hardlink'].get(bool) and not link else: link = self.config['link'].get(bool) - if opts.hardlink is not None: - hardlink = opts.hardlink - else: - hardlink = self.config['hardlink'].get(bool) + if opts.hardlink is not None: + hardlink = opts.hardlink + else: + hardlink = self.config['hardlink'].get(bool) if opts.album: albums = lib.albums(ui.decargs(args)) @@ -507,7 +517,8 @@ class ConvertPlugin(BeetsPlugin): if opts.album and self.config['copy_album_art']: for album in albums: - self.copy_album_art(album, dest, path_formats, pretend, link, hardlink) + self.copy_album_art(album, dest, path_formats, pretend, + link, hardlink) convert = [self.convert_item(dest, opts.keep_new, diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 15b4fd2f2..4bc2b3f3f 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -103,15 +103,15 @@ file. The available options are: to their destination. This option creates symbolic links instead. Note that options such as ``embed`` that modify the output files after the transcoding step will cause the original files to be modified as well if ``link`` is - enabled. For this reason, it is highly recommended not use to ``link`` and - ``embed`` at the same time. + enabled. For this reason, album-art embedding is disabled + for files that are linked. Default: ``false``. - **hardlink**: By default, files that do not need to be transcoded will be copied to their destination. This option creates hard links instead. Note that options such as ``embed`` that modify the output files after the transcoding step will cause the original files to be modified as well if ``hardlink`` is - enabled. For this reason, it is highly recommended not use to ``hardlink`` and - ``embed`` at the same time. + enabled. For this reason, album-art embedding is disabled + for files that are linked. This option overrides ``link``. Only works when converting to a directory on the same filesystem as the library. Default: ``false``. From 7aab50b7b8095c96132db1867fe92c3ded42cf11 Mon Sep 17 00:00:00 2001 From: Vexatos Date: Wed, 21 Aug 2019 12:09:57 +0200 Subject: [PATCH 164/613] convert: Reduce amount of duplicate code for linking. Also slightly reworded documentation. --- beetsplug/convert.py | 114 ++++++++++++++++----------------------- docs/changelog.rst | 10 ++-- docs/plugins/convert.rst | 12 ++--- 3 files changed, 53 insertions(+), 83 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index a0afdd722..e7ac4f3ac 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -307,45 +307,35 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(original)) util.move(item.path, original) - linked = False - if should_transcode(item, fmt): + linked = False try: self.encode(command, original, converted, pretend) except subprocess.CalledProcessError: continue else: - if hardlink: - if pretend: - self._log.info(u'ln {0} {1}', - util.displayable_path(original), - util.displayable_path(converted)) - else: - # No transcoding necessary. - self._log.info(u'Hardlinking {0}', - util.displayable_path(item.path)) - util.hardlink(original, converted) - linked = True - elif link: - if pretend: - self._log.info(u'ln -s {0} {1}', - util.displayable_path(original), - util.displayable_path(converted)) - else: - # No transcoding necessary. - self._log.info(u'Linking {0}', - util.displayable_path(item.path)) - util.link(original, converted) - linked = True + linked = link or hardlink + if pretend: + msg = 'ln' if hardlink else ('ln -s' if link else 'cp') + + self._log.info(u'{2} {0} {1}', + util.displayable_path(original), + util.displayable_path(converted), + msg) else: - if pretend: - self._log.info(u'cp {0} {1}', - util.displayable_path(original), - util.displayable_path(converted)) + # No transcoding necessary. + msg = 'Hardlinking' if hardlink \ + else ('Linking' if link else 'Copying') + + self._log.info(u'{1} {0}', + util.displayable_path(item.path), + msg) + + if hardlink: + util.hardlink(original, converted) + elif link: + util.link(original, converted) else: - # No transcoding necessary. - self._log.info(u'Copying {0}', - util.displayable_path(item.path)) util.copy(original, converted) if pretend: @@ -434,35 +424,26 @@ class ConvertPlugin(BeetsPlugin): if not pretend: ArtResizer.shared.resize(maxwidth, album.artpath, dest) else: - if hardlink: - if pretend: - self._log.info(u'ln {0} {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) - else: - self._log.info(u'Hardlinking cover art from {0} to {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) - util.hardlink(album.artpath, dest) - elif link: - if pretend: - self._log.info(u'ln -s {0} {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) - else: - self._log.info(u'Linking cover art from {0} to {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) - util.link(album.artpath, dest) + if pretend: + msg = 'ln' if hardlink else ('ln -s' if link else 'cp') + + self._log.info(u'{2} {0} {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest), + msg) else: - if pretend: - self._log.info(u'cp {0} {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) + msg = 'Hardlinking' if hardlink \ + else ('Linking' if link else 'Copying') + + self._log.info(u'{2} cover art from {0} to {1}', + util.displayable_path(album.artpath), + util.displayable_path(dest), + msg) + if hardlink: + util.hardlink(album.artpath, dest) + elif link: + util.link(album.artpath, dest) else: - self._log.info(u'Copying cover art from {0} to {1}', - util.displayable_path(album.artpath), - util.displayable_path(dest)) util.copy(album.artpath, dest) def convert_func(self, lib, opts, args): @@ -482,21 +463,16 @@ class ConvertPlugin(BeetsPlugin): else: pretend = self.config['pretend'].get(bool) - if opts.link is not None: + if opts.hardlink is not None: + hardlink = opts.hardlink + link = False + elif opts.link is not None: + hardlink = False link = opts.link - - if opts.hardlink is not None: - hardlink = opts.hardlink - else: - hardlink = self.config['hardlink'].get(bool) and not link else: + hardlink = self.config['hardlink'].get(bool) link = self.config['link'].get(bool) - if opts.hardlink is not None: - hardlink = opts.hardlink - else: - hardlink = self.config['hardlink'].get(bool) - if opts.album: albums = lib.albums(ui.decargs(args)) items = [i for a in albums for i in a.items()] diff --git a/docs/changelog.rst b/docs/changelog.rst index c35114390..458216627 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,12 +11,10 @@ New features: (the MBID), and ``work_disambig`` (the disambiguation string). Thanks to :user:`dosoe`. :bug:`2580` :bug:`3272` -* :doc:`/plugins/convert`: Added new ``-l`` (``--link``) flag and ``link`` option - which symlinks files that do not need to be converted instead of copying them. - :bug:`2324` -* :doc:`/plugins/convert`: Added new ``-H`` (``--hardlink``) flag and ``hardlink`` - option which hardlinks files that do not need to be converted - instead of copying them. +* :doc:`/plugins/convert`: Added new ``-l`` (``--link``) flag and ``link`` + option as well as the ``-H`` (``--hardlink``) flag and ``hardlink`` + option which symlinks or hardlinks files that do not need to + be converted instead of copying them. :bug:`2324` * :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 of the MPD protocol. This is enough to get it talking to more complicated diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 4bc2b3f3f..6e9d00a11 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -51,8 +51,8 @@ them. By default, files that do not need to be transcoded will be copied to their destination. Passing the ``-l`` (``--link``) flag creates symbolic links instead, passing ``-H`` (``--hardlink``) creates hard links. -Refer to the ``link`` and ``hardlink`` options below -for potential issues with this. +Note that album art embedding is disabled for files that are linked. +Refer to the ``link`` and ``hardlink`` options below. Configuration @@ -106,12 +106,8 @@ file. The available options are: enabled. For this reason, album-art embedding is disabled for files that are linked. Default: ``false``. -- **hardlink**: By default, files that do not need to be transcoded will be - copied to their destination. This option creates hard links instead. Note that - options such as ``embed`` that modify the output files after the transcoding - step will cause the original files to be modified as well if ``hardlink`` is - enabled. For this reason, album-art embedding is disabled - for files that are linked. +- **hardlink**: This options works similar to ``link``, but it creates + hard links instead of symlinks. This option overrides ``link``. Only works when converting to a directory on the same filesystem as the library. Default: ``false``. From 67a38fc0e44e3b85f8058dac3ed81ba46288f9e4 Mon Sep 17 00:00:00 2001 From: Chris <> Date: Thu, 22 Aug 2019 12:18:24 +0100 Subject: [PATCH 165/613] Apply `data_source` field to albums #1693 --- beets/importer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/importer.py b/beets/importer.py index d2943b511..c0e25fb51 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -754,6 +754,7 @@ class ImportTask(BaseImportTask): self.record_replaced(lib) self.remove_replaced(lib) self.album = lib.add_album(self.imported_items()) + self.album.set_parse('data_source', self.imported_items()[0].data_source) self.reimport_metadata(lib) def record_replaced(self, lib): From 6146857a116b3fb2961f972e773b8143971587c2 Mon Sep 17 00:00:00 2001 From: Chris <> Date: Thu, 22 Aug 2019 14:53:25 +0100 Subject: [PATCH 166/613] Added check + store on object (no method call) --- beets/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/importer.py b/beets/importer.py index c0e25fb51..1ebe869a1 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -754,7 +754,8 @@ class ImportTask(BaseImportTask): self.record_replaced(lib) self.remove_replaced(lib) self.album = lib.add_album(self.imported_items()) - self.album.set_parse('data_source', self.imported_items()[0].data_source) + if self.imported_items()[0].data_source: + self.album.data_source = self.imported_items()[0].data_source self.reimport_metadata(lib) def record_replaced(self, lib): From ece99bb330a21205bf4b605ae1198474162d0739 Mon Sep 17 00:00:00 2001 From: Chris <> Date: Thu, 22 Aug 2019 15:46:00 +0100 Subject: [PATCH 167/613] Changed test to look for 'key' in Item.keys() --- beets/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/importer.py b/beets/importer.py index 1ebe869a1..385c59185 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -754,7 +754,7 @@ class ImportTask(BaseImportTask): self.record_replaced(lib) self.remove_replaced(lib) self.album = lib.add_album(self.imported_items()) - if self.imported_items()[0].data_source: + if 'data_source' in self.imported_items()[0].keys(): self.album.data_source = self.imported_items()[0].data_source self.reimport_metadata(lib) From ba4430c0bf6f1e30bfb2698972f863ee290b080d Mon Sep 17 00:00:00 2001 From: Chris <> Date: Thu, 22 Aug 2019 19:27:23 +0100 Subject: [PATCH 168/613] Simplified conditional test, now w/o function call --- beets/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/importer.py b/beets/importer.py index 385c59185..e97b0a75c 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -754,7 +754,7 @@ class ImportTask(BaseImportTask): self.record_replaced(lib) self.remove_replaced(lib) self.album = lib.add_album(self.imported_items()) - if 'data_source' in self.imported_items()[0].keys(): + if 'data_source' in self.imported_items()[0]: self.album.data_source = self.imported_items()[0].data_source self.reimport_metadata(lib) From e0276138dbd8ba8d525a5420daa8e79a29e6b482 Mon Sep 17 00:00:00 2001 From: Chris <> Date: Fri, 23 Aug 2019 11:34:34 +0100 Subject: [PATCH 169/613] Changelog added --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 458216627..690696c1d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -63,6 +63,9 @@ New features: Windows. Thanks to :user:`MartyLake`. :bug:`3331` :bug:`3334` +* The 'data_source' field is now also applied as an album-level flexible + attribute during imports, allowing for more refined album level searches. + :bug:`3350` :bug:`1693` Fixes: From 6e5e8a9cb01ab12253c2e9c435e255484ea1b8b7 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 17:53:56 -0700 Subject: [PATCH 170/613] Add Deezer plugin --- beetsplug/deezer.py | 341 +++++++++++++++++++++++++++++++++++++++++++ beetsplug/spotify.py | 20 ++- docs/changelog.rst | 4 + 3 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 beetsplug/deezer.py diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py new file mode 100644 index 000000000..b06777f99 --- /dev/null +++ b/beetsplug/deezer.py @@ -0,0 +1,341 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Rahul Ahuja. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Adds Deezer release and track search support to the autotagger +""" +from __future__ import absolute_import, print_function + +import re +import collections + +import six +import unidecode +import requests + +from beets import ui +from beets.plugins import BeetsPlugin +from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance + + +class DeezerPlugin(BeetsPlugin): + # Base URLs for the Deezer API + # Documentation: https://developers.deezer.com/api/ + search_url = 'https://api.deezer.com/search/' + album_url = 'https://api.deezer.com/album/' + track_url = 'https://api.deezer.com/track/' + + def __init__(self): + super(DeezerPlugin, self).__init__() + self.config.add( + { + 'mode': 'list', + 'tiebreak': 'popularity', + 'show_failures': False, + 'artist_field': 'albumartist', + 'album_field': 'album', + 'track_field': 'title', + 'region_filter': None, + 'regex': [], + 'source_weight': 0.5, + } + ) + + def _get_deezer_id(self, url_type, id_): + """Parse a Deezer ID from its URL if necessary. + + :param url_type: Type of Deezer URL. Either 'album', 'artist', 'playlist', + or 'track'. + :type url_type: str + :param id_: Deezer ID or URL. + :type id_: str + :return: Deezer ID. + :rtype: str + """ + id_regex = r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)' + self._log.debug(u'Searching for {} {}', url_type, id_) + match = re.search(id_regex.format(url_type), str(id_)) + return match.group(3) if match else None + + def album_for_id(self, album_id): + """Fetch an album by its Deezer ID or URL and return an + AlbumInfo object or None if the album is not found. + + :param album_id: Deezer ID or URL for the album. + :type album_id: str + :return: AlbumInfo object for album. + :rtype: beets.autotag.hooks.AlbumInfo or None + """ + deezer_id = self._get_deezer_id('album', album_id) + if deezer_id is None: + return None + + album_data = requests.get(self.album_url + deezer_id).json() + artist, artist_id = self._get_artist(album_data['contributors']) + + release_date = album_data['release_date'] + date_parts = [int(part) for part in release_date.split('-')] + num_date_parts = len(date_parts) + + if num_date_parts == 3: + year, month, day = date_parts + elif num_date_parts == 2: + year, month = date_parts + day = None + elif num_date_parts == 1: + year = date_parts[0] + month = None + day = None + else: + raise ui.UserError( + u"Invalid `release_date` returned " + u"by Deezer API: '{}'".format(release_date) + ) + + tracks_data = requests.get( + self.album_url + deezer_id + '/tracks' + ).json()['data'] + tracks = [] + medium_totals = collections.defaultdict(int) + for i, track_data in enumerate(tracks_data): + track = self._get_track(track_data) + track.index = i + 1 + medium_totals[track.medium] += 1 + tracks.append(track) + for track in tracks: + track.medium_total = medium_totals[track.medium] + + return AlbumInfo( + album=album_data['title'], + album_id=deezer_id, + artist=artist, + artist_credit=self._get_artist([album_data['artist']]), + artist_id=artist_id, + tracks=tracks, + albumtype=album_data['record_type'], + va=len(album_data['contributors']) == 1 + and artist.lower() == 'various artists', + year=year, + month=month, + day=day, + label=album_data['label'], + mediums=max(medium_totals.keys()), + data_source='Deezer', + data_url=album_data['link'], + ) + + def _get_track(self, track_data): + """Convert a Deezer track object dict to a TrackInfo object. + + :param track_data: Deezer Track object dict + :type track_data: dict + :return: TrackInfo object for track + :rtype: beets.autotag.hooks.TrackInfo + """ + artist, artist_id = self._get_artist( + track_data.get('contributors', [track_data['artist']]) + ) + return TrackInfo( + title=track_data['title'], + track_id=track_data['id'], + artist=artist, + artist_id=artist_id, + length=track_data['duration'], + index=track_data['track_position'], + medium=track_data['disk_number'], + medium_index=track_data['track_position'], + data_source='Deezer', + data_url=track_data['link'], + ) + + def track_for_id(self, track_id=None, track_data=None): + """Fetch a track by its Deezer ID or URL and return a + TrackInfo object or None if the track is not found. + + :param track_id: (Optional) Deezer ID or URL for the track. Either + ``track_id`` or ``track_data`` must be provided. + :type track_id: str + :param track_data: (Optional) Simplified track object dict. May be + provided instead of ``track_id`` to avoid unnecessary API calls. + :type track_data: dict + :return: TrackInfo object for track + :rtype: beets.autotag.hooks.TrackInfo or None + """ + if track_data is None: + deezer_id = self._get_deezer_id('track', track_id) + if deezer_id is None: + return None + track_data = requests.get(self.track_url + deezer_id).json() + track = self._get_track(track_data) + + # Get album's tracks to set `track.index` (position on the entire + # release) and `track.medium_total` (total number of tracks on + # the track's disc). + album_tracks_data = requests.get( + self.album_url + str(track_data['album']['id']) + '/tracks' + ).json()['data'] + medium_total = 0 + for i, track_data in enumerate(album_tracks_data, start=1): + if track_data['disc_number'] == track.medium: + medium_total += 1 + if track_data['id'] == track.track_id: + track.index = i + track.medium_total = medium_total + return track + + @staticmethod + def _get_artist(artists): + """Returns an artist string (all artists) and an artist_id (the main + artist) for a list of Deezer artist object dicts. + + :param artists: Iterable of ``contributors`` or ``artist`` returned by the + Deezer Album (https://developers.deezer.com/api/album) or Deezer Track + (https://developers.deezer.com/api/track) APIs. + :type artists: list[dict] + :return: Normalized artist string + :rtype: str + """ + artist_id = None + artist_names = [] + for artist in artists: + if not artist_id: + artist_id = artist['id'] + name = artist['name'] + # Move articles to the front. + name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) + artist_names.append(name) + artist = ', '.join(artist_names).replace(' ,', ',') or None + return artist, artist_id + + def album_distance(self, items, album_info, mapping): + """Returns the Deezer source weight and the maximum source weight + for albums. + """ + dist = Distance() + if album_info.data_source == 'Deezer': + dist.add('source', self.config['source_weight'].as_number()) + return dist + + def track_distance(self, item, track_info): + """Returns the Deezer source weight and the maximum source weight + for individual tracks. + """ + dist = Distance() + if track_info.data_source == 'Deezer': + dist.add('source', self.config['source_weight'].as_number()) + return dist + + def candidates(self, items, artist, album, va_likely): + """Returns a list of AlbumInfo objects for Deezer Search API results + matching an ``album`` and ``artist`` (if not various). + + :param items: List of items comprised by an album to be matched. + :type items: list[beets.library.Item] + :param artist: The artist of the album to be matched. + :type artist: str + :param album: The name of the album to be matched. + :type album: str + :param va_likely: True if the album to be matched likely has + Various Artists. + :type va_likely: bool + :return: Candidate AlbumInfo objects. + :rtype: list[beets.autotag.hooks.AlbumInfo] + """ + query_filters = {'album': album} + if not va_likely: + query_filters['artist'] = artist + response_data = self._search_deezer( + query_type='album', filters=query_filters + ) + if response_data is None: + return [] + return [ + self.album_for_id(album_id=album_data['id']) + for album_data in response_data['data'] + ] + + def item_candidates(self, item, artist, title): + """Returns a list of TrackInfo objects for Deezer Search API results + matching ``title`` and ``artist``. + + :param item: Singleton item to be matched. + :type item: beets.library.Item + :param artist: The artist of the track to be matched. + :type artist: str + :param title: The title of the track to be matched. + :type title: str + :return: Candidate TrackInfo objects. + :rtype: list[beets.autotag.hooks.TrackInfo] + """ + response_data = self._search_deezer( + query_type='track', keywords=title, filters={'artist': artist} + ) + if response_data is None: + return [] + return [ + self.track_for_id(track_data=track_data) + for track_data in response_data['data'] + ] + + @staticmethod + def _construct_search_query(filters=None, keywords=''): + """Construct a query string with the specified filters and keywords to + be provided to the Deezer Search API (https://developers.deezer.com/api/search). + + :param filters: (Optional) Field filters to apply. + :type filters: dict + :param keywords: (Optional) Query keywords to use. + :type keywords: str + :return: Query string to be provided to the Search API. + :rtype: str + """ + query_components = [ + keywords, + ' '.join('{}:"{}"'.format(k, v) for k, v in filters.items()), + ] + query = ' '.join([q for q in query_components if q]) + if not isinstance(query, six.text_type): + query = query.decode('utf8') + return unidecode.unidecode(query) + + def _search_deezer(self, query_type, filters=None, keywords=''): + """Query the Deezer Search API for the specified ``keywords``, applying + the provided ``filters``. + + :param query_type: The Deezer Search API method to use. Valid types are: + 'album', 'artist', 'history', 'playlist', 'podcast', 'radio', 'track', + 'user', and 'track'. + :type query_type: str + :param filters: (Optional) Field filters to apply. + :type filters: dict + :param keywords: (Optional) Query keywords to use. + :type keywords: str + :return: JSON data for the class:`Response ` object or None + if no search results are returned. + :rtype: dict or None + """ + query = self._construct_search_query( + keywords=keywords, filters=filters + ) + if not query: + return None + self._log.debug(u"Searching Deezer for '{}'".format(query)) + response_data = requests.get( + self.search_url + query_type, params={'q': query} + ).json() + num_results = len(response_data['data']) + self._log.debug( + u"Found {} results from Deezer for '{}'", num_results, query + ) + return response_data if num_results > 0 else None diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 0af0dc9aa..5cf5b245a 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -1,5 +1,21 @@ # -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Rahul Ahuja. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +"""Adds Spotify release and track search support to the autotagger, along with +Spotify playlist construction. +""" from __future__ import division, absolute_import, print_function import re @@ -262,11 +278,11 @@ class SpotifyPlugin(BeetsPlugin): requests.get, self.album_url + track_data['album']['id'] ) medium_total = 0 - for i, track_data in enumerate(album_data['tracks']['items']): + for i, track_data in enumerate(album_data['tracks']['items'], start=1): if track_data['disc_number'] == track.medium: medium_total += 1 if track_data['id'] == track.track_id: - track.index = i + 1 + track.index = i track.medium_total = medium_total return track diff --git a/docs/changelog.rst b/docs/changelog.rst index 690696c1d..012d0a4b8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -66,6 +66,10 @@ New features: * The 'data_source' field is now also applied as an album-level flexible attribute during imports, allowing for more refined album level searches. :bug:`3350` :bug:`1693` +* :doc:`/plugins/deezer`: + * Added Deezer plugin as an import metadata provider: you can match tracks + and albums using the Deezer database. + Thanks to :user:`rhlahuja`. Fixes: From e8228d0305029347e5e148e6374864c3f3da2f9a Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 18:02:11 -0700 Subject: [PATCH 171/613] Add changelog hyperlink --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 012d0a4b8..7544db431 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,8 +67,8 @@ New features: attribute during imports, allowing for more refined album level searches. :bug:`3350` :bug:`1693` * :doc:`/plugins/deezer`: - * Added Deezer plugin as an import metadata provider: you can match tracks - and albums using the Deezer database. + * Added Deezer plugin as an import metadata provider: you can now match tracks + and albums using the `Deezer`_ database. Thanks to :user:`rhlahuja`. Fixes: @@ -147,6 +147,7 @@ For packagers: .. _MediaFile: https://github.com/beetbox/mediafile .. _Confuse: https://github.com/beetbox/confuse .. _works: https://musicbrainz.org/doc/Work +.. _Deezer: https://www.deezer.com 1.4.9 (May 30, 2019) From 804397bb124583c9003b6c4be0fb1d02359c1e94 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 18:13:19 -0700 Subject: [PATCH 172/613] Appease flake8 --- beetsplug/deezer.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index b06777f99..fb548756f 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -55,8 +55,8 @@ class DeezerPlugin(BeetsPlugin): def _get_deezer_id(self, url_type, id_): """Parse a Deezer ID from its URL if necessary. - :param url_type: Type of Deezer URL. Either 'album', 'artist', 'playlist', - or 'track'. + :param url_type: Type of Deezer URL. Either 'album', 'artist', + 'playlist', or 'track'. :type url_type: str :param id_: Deezer ID or URL. :type id_: str @@ -199,9 +199,9 @@ class DeezerPlugin(BeetsPlugin): """Returns an artist string (all artists) and an artist_id (the main artist) for a list of Deezer artist object dicts. - :param artists: Iterable of ``contributors`` or ``artist`` returned by the - Deezer Album (https://developers.deezer.com/api/album) or Deezer Track - (https://developers.deezer.com/api/track) APIs. + :param artists: Iterable of ``contributors`` or ``artist`` returned + by the Deezer Album (https://developers.deezer.com/api/album) or + Deezer Track (https://developers.deezer.com/api/track) APIs. :type artists: list[dict] :return: Normalized artist string :rtype: str @@ -291,7 +291,8 @@ class DeezerPlugin(BeetsPlugin): @staticmethod def _construct_search_query(filters=None, keywords=''): """Construct a query string with the specified filters and keywords to - be provided to the Deezer Search API (https://developers.deezer.com/api/search). + be provided to the Deezer Search API + (https://developers.deezer.com/api/search). :param filters: (Optional) Field filters to apply. :type filters: dict @@ -313,9 +314,9 @@ class DeezerPlugin(BeetsPlugin): """Query the Deezer Search API for the specified ``keywords``, applying the provided ``filters``. - :param query_type: The Deezer Search API method to use. Valid types are: - 'album', 'artist', 'history', 'playlist', 'podcast', 'radio', 'track', - 'user', and 'track'. + :param query_type: The Deezer Search API method to use. Valid types + are: 'album', 'artist', 'history', 'playlist', 'podcast', + 'radio', 'track', 'user', and 'track'. :type query_type: str :param filters: (Optional) Field filters to apply. :type filters: dict From 2cf55ee893b2df1d81846775f9af956d2d1365f1 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 18:33:27 -0700 Subject: [PATCH 173/613] Add deezer.rst doc, remove unused options --- beetsplug/deezer.py | 32 +++++++------------------------- docs/plugins/deezer.rst | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 docs/plugins/deezer.rst diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index fb548756f..9447e9c3c 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -38,19 +38,7 @@ class DeezerPlugin(BeetsPlugin): def __init__(self): super(DeezerPlugin, self).__init__() - self.config.add( - { - 'mode': 'list', - 'tiebreak': 'popularity', - 'show_failures': False, - 'artist_field': 'albumartist', - 'album_field': 'album', - 'track_field': 'title', - 'region_filter': None, - 'regex': [], - 'source_weight': 0.5, - } - ) + self.config.add({'source_weight': 0.5}) def _get_deezer_id(self, url_type, id_): """Parse a Deezer ID from its URL if necessary. @@ -103,9 +91,9 @@ class DeezerPlugin(BeetsPlugin): u"by Deezer API: '{}'".format(release_date) ) - tracks_data = requests.get( - self.album_url + deezer_id + '/tracks' - ).json()['data'] + tracks_data = requests.get(self.album_url + deezer_id + '/tracks').json()[ + 'data' + ] tracks = [] medium_totals = collections.defaultdict(int) for i, track_data in enumerate(tracks_data): @@ -255,9 +243,7 @@ class DeezerPlugin(BeetsPlugin): query_filters = {'album': album} if not va_likely: query_filters['artist'] = artist - response_data = self._search_deezer( - query_type='album', filters=query_filters - ) + response_data = self._search_deezer(query_type='album', filters=query_filters) if response_data is None: return [] return [ @@ -326,9 +312,7 @@ class DeezerPlugin(BeetsPlugin): if no search results are returned. :rtype: dict or None """ - query = self._construct_search_query( - keywords=keywords, filters=filters - ) + query = self._construct_search_query(keywords=keywords, filters=filters) if not query: return None self._log.debug(u"Searching Deezer for '{}'".format(query)) @@ -336,7 +320,5 @@ class DeezerPlugin(BeetsPlugin): self.search_url + query_type, params={'q': query} ).json() num_results = len(response_data['data']) - self._log.debug( - u"Found {} results from Deezer for '{}'", num_results, query - ) + self._log.debug(u"Found {} results from Deezer for '{}'", num_results, query) return response_data if num_results > 0 else None diff --git a/docs/plugins/deezer.rst b/docs/plugins/deezer.rst new file mode 100644 index 000000000..94347c99e --- /dev/null +++ b/docs/plugins/deezer.rst @@ -0,0 +1,38 @@ +Spotify Plugin +============== + +The ``deezer`` plugin provides metadata matches for the importer using the +`Deezer_` `Album`_ and `Track`_ APIs. + +.. _Deezer: https://www.deezer.com +.. _Album: https://developers.deezer.com/api/album +.. _Track: https://developers.deezer.com/api/track + +Why Use This Plugin? +-------------------- + +* You're a Beets user. +* You want to autotag music with metadata from the Deezer API. + +Basic Usage +----------- +First, enable the ``deezer`` plugin (see :ref:`using-plugins`). + +You can enter the URL for an album or song on Deezer at the ``enter Id`` +prompt during import:: + + Enter search, enter Id, aBort, eDit, edit Candidates, plaY? i + Enter release ID: https://www.deezer.com/en/album/572261 + +Configuration +------------- +Put these options in config.yaml under the ``deezer:`` section: + +- **source_weight**: Penalty applied to Spotify matches during import. Set to + 0.0 to disable. + Default: ``0.5``. + +Here's an example:: + + deezer: + source_weight: 0.7 From 790ca805d597f9eeda0ba6ef1e9effcc3dfeefbb Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 18:34:06 -0700 Subject: [PATCH 174/613] Formatting --- beetsplug/deezer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 9447e9c3c..4fad84b9f 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -91,9 +91,9 @@ class DeezerPlugin(BeetsPlugin): u"by Deezer API: '{}'".format(release_date) ) - tracks_data = requests.get(self.album_url + deezer_id + '/tracks').json()[ - 'data' - ] + tracks_data = requests.get( + self.album_url + deezer_id + '/tracks' + ).json()['data'] tracks = [] medium_totals = collections.defaultdict(int) for i, track_data in enumerate(tracks_data): @@ -243,7 +243,9 @@ class DeezerPlugin(BeetsPlugin): query_filters = {'album': album} if not va_likely: query_filters['artist'] = artist - response_data = self._search_deezer(query_type='album', filters=query_filters) + response_data = self._search_deezer( + query_type='album', filters=query_filters + ) if response_data is None: return [] return [ @@ -312,7 +314,9 @@ class DeezerPlugin(BeetsPlugin): if no search results are returned. :rtype: dict or None """ - query = self._construct_search_query(keywords=keywords, filters=filters) + query = self._construct_search_query( + keywords=keywords, filters=filters + ) if not query: return None self._log.debug(u"Searching Deezer for '{}'".format(query)) @@ -320,5 +324,7 @@ class DeezerPlugin(BeetsPlugin): self.search_url + query_type, params={'q': query} ).json() num_results = len(response_data['data']) - self._log.debug(u"Found {} results from Deezer for '{}'", num_results, query) + self._log.debug( + u"Found {} results from Deezer for '{}'", num_results, query + ) return response_data if num_results > 0 else None From ca33f190a5a501bb30e61bde28690af378356dbe Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 18:58:03 -0700 Subject: [PATCH 175/613] Add deezer, spotify docs to autotagger index --- docs/plugins/deezer.rst | 2 +- docs/plugins/index.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/plugins/deezer.rst b/docs/plugins/deezer.rst index 94347c99e..8bddd766f 100644 --- a/docs/plugins/deezer.rst +++ b/docs/plugins/deezer.rst @@ -1,4 +1,4 @@ -Spotify Plugin +Deezer Plugin ============== The ``deezer`` plugin provides metadata matches for the importer using the diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 4f3e6fbff..16e564f84 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -105,10 +105,14 @@ Autotagger Extensions * :doc:`chroma`: Use acoustic fingerprinting to identify audio files with missing or incorrect metadata. * :doc:`discogs`: Search for releases in the `Discogs`_ database. +* :doc:`spotify`: Search for releases in the `Spotify`_ database. +* :doc:`deezer`: Search for releases in the `Deezer`_ database. * :doc:`fromfilename`: Guess metadata for untagged tracks from their filenames. .. _Discogs: https://www.discogs.com/ +.. _Spotify: https://www.spotify.com +.. _Deezer: https://www.deezer.com/ Metadata -------- From 8c84daf77af93db75006dc89b3bce253040b0f3d Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 19:12:21 -0700 Subject: [PATCH 176/613] Fix doc links --- docs/plugins/deezer.rst | 2 +- docs/plugins/index.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/plugins/deezer.rst b/docs/plugins/deezer.rst index 8bddd766f..c00d1d68a 100644 --- a/docs/plugins/deezer.rst +++ b/docs/plugins/deezer.rst @@ -2,7 +2,7 @@ Deezer Plugin ============== The ``deezer`` plugin provides metadata matches for the importer using the -`Deezer_` `Album`_ and `Track`_ APIs. +`Deezer`_ `Album`_ and `Track`_ APIs. .. _Deezer: https://www.deezer.com .. _Album: https://developers.deezer.com/api/album diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 16e564f84..3f14b3116 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -104,9 +104,9 @@ Autotagger Extensions * :doc:`chroma`: Use acoustic fingerprinting to identify audio files with missing or incorrect metadata. -* :doc:`discogs`: Search for releases in the `Discogs`_ database. -* :doc:`spotify`: Search for releases in the `Spotify`_ database. -* :doc:`deezer`: Search for releases in the `Deezer`_ database. +* :doc:`/plugins/discogs`: Search for releases in the `Discogs`_ database. +* :doc:`/plugins/spotify`: Search for releases in the `Spotify`_ database. +* :doc:`/plugins/deezer`: Search for releases in the `Deezer`_ database. * :doc:`fromfilename`: Guess metadata for untagged tracks from their filenames. From 2177c7695a149ecd5ae7916301300b323a88b9a3 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 19:44:27 -0700 Subject: [PATCH 177/613] Stringify Deezer ID --- beetsplug/deezer.py | 2 +- docs/plugins/index.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 4fad84b9f..164d3efd3 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -54,7 +54,7 @@ class DeezerPlugin(BeetsPlugin): id_regex = r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)' self._log.debug(u'Searching for {} {}', url_type, id_) match = re.search(id_regex.format(url_type), str(id_)) - return match.group(3) if match else None + return str(match.group(3)) if match else None def album_for_id(self, album_id): """Fetch an album by its Deezer ID or URL and return an diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 3f14b3116..16e564f84 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -104,9 +104,9 @@ Autotagger Extensions * :doc:`chroma`: Use acoustic fingerprinting to identify audio files with missing or incorrect metadata. -* :doc:`/plugins/discogs`: Search for releases in the `Discogs`_ database. -* :doc:`/plugins/spotify`: Search for releases in the `Spotify`_ database. -* :doc:`/plugins/deezer`: Search for releases in the `Deezer`_ database. +* :doc:`discogs`: Search for releases in the `Discogs`_ database. +* :doc:`spotify`: Search for releases in the `Spotify`_ database. +* :doc:`deezer`: Search for releases in the `Deezer`_ database. * :doc:`fromfilename`: Guess metadata for untagged tracks from their filenames. From 43f09296c9055f16a64bfccb50b35b0f03129aac Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 19:50:55 -0700 Subject: [PATCH 178/613] Fix AlbumInfo.album_credit assignment --- beetsplug/deezer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 164d3efd3..e2cad928e 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -54,7 +54,7 @@ class DeezerPlugin(BeetsPlugin): id_regex = r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)' self._log.debug(u'Searching for {} {}', url_type, id_) match = re.search(id_regex.format(url_type), str(id_)) - return str(match.group(3)) if match else None + return match.group(3) if match else None def album_for_id(self, album_id): """Fetch an album by its Deezer ID or URL and return an @@ -108,7 +108,7 @@ class DeezerPlugin(BeetsPlugin): album=album_data['title'], album_id=deezer_id, artist=artist, - artist_credit=self._get_artist([album_data['artist']]), + artist_credit=self._get_artist([album_data['artist']])[0], artist_id=artist_id, tracks=tracks, albumtype=album_data['record_type'], From 240097e377f5fb2c112d4baca7516c7c6db01641 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 19:55:26 -0700 Subject: [PATCH 179/613] Include `deezer` in toctree --- docs/plugins/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 16e564f84..b9f512b1f 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -47,6 +47,7 @@ like this:: bucket chroma convert + deezer discogs duplicates edit From cd1aa3e8aae83596ea9be70d242bda48873e90f8 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 20:10:34 -0700 Subject: [PATCH 180/613] Avoid empty deezer_id string --- beetsplug/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index e2cad928e..23b52ba07 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -66,7 +66,7 @@ class DeezerPlugin(BeetsPlugin): :rtype: beets.autotag.hooks.AlbumInfo or None """ deezer_id = self._get_deezer_id('album', album_id) - if deezer_id is None: + if not deezer_id: return None album_data = requests.get(self.album_url + deezer_id).json() From 70264ee6ee9b44ea05b7c9e1fbef8fce873070b1 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 21:18:08 -0700 Subject: [PATCH 181/613] Handle empty deezer_id upfront --- beetsplug/deezer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 23b52ba07..12a861465 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -54,7 +54,11 @@ class DeezerPlugin(BeetsPlugin): id_regex = r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)' self._log.debug(u'Searching for {} {}', url_type, id_) match = re.search(id_regex.format(url_type), str(id_)) - return match.group(3) if match else None + if match: + deezer_id = match.group(3) + if deezer_id: + return deezer_id + return None def album_for_id(self, album_id): """Fetch an album by its Deezer ID or URL and return an @@ -66,7 +70,7 @@ class DeezerPlugin(BeetsPlugin): :rtype: beets.autotag.hooks.AlbumInfo or None """ deezer_id = self._get_deezer_id('album', album_id) - if not deezer_id: + if deezer_id is None: return None album_data = requests.get(self.album_url + deezer_id).json() From 2ab883a20e7998d9344fccd2f27a4207d33b7828 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 21:23:16 -0700 Subject: [PATCH 182/613] Fix track.index assignment --- beetsplug/deezer.py | 2 +- beetsplug/spotify.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 12a861465..d5aa8defd 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -102,7 +102,7 @@ class DeezerPlugin(BeetsPlugin): medium_totals = collections.defaultdict(int) for i, track_data in enumerate(tracks_data): track = self._get_track(track_data) - track.index = i + 1 + track.index = i medium_totals[track.medium] += 1 tracks.append(track) for track in tracks: diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 5cf5b245a..3eddb2490 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -202,7 +202,7 @@ class SpotifyPlugin(BeetsPlugin): medium_totals = collections.defaultdict(int) for i, track_data in enumerate(response_data['tracks']['items']): track = self._get_track(track_data) - track.index = i + 1 + track.index = i medium_totals[track.medium] += 1 tracks.append(track) for track in tracks: From 9babce582da764539e9fbf80099f7ffa28096d61 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 1 Sep 2019 21:24:56 -0700 Subject: [PATCH 183/613] Fix track data enumeration idx --- beetsplug/deezer.py | 2 +- beetsplug/spotify.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index d5aa8defd..bd098eb6a 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -100,7 +100,7 @@ class DeezerPlugin(BeetsPlugin): ).json()['data'] tracks = [] medium_totals = collections.defaultdict(int) - for i, track_data in enumerate(tracks_data): + for i, track_data in enumerate(tracks_data, start=1): track = self._get_track(track_data) track.index = i medium_totals[track.medium] += 1 diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 3eddb2490..06bca1190 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -200,7 +200,9 @@ class SpotifyPlugin(BeetsPlugin): tracks = [] medium_totals = collections.defaultdict(int) - for i, track_data in enumerate(response_data['tracks']['items']): + for i, track_data in enumerate( + response_data['tracks']['items'], start=1 + ): track = self._get_track(track_data) track.index = i medium_totals[track.medium] += 1 From 4a552595df562e454eab41517cdf04fbeefb42aa Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Mon, 2 Sep 2019 14:27:51 -0700 Subject: [PATCH 184/613] Simplify regex match --- beetsplug/deezer.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index bd098eb6a..654fb59dd 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -54,11 +54,8 @@ class DeezerPlugin(BeetsPlugin): id_regex = r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)' self._log.debug(u'Searching for {} {}', url_type, id_) match = re.search(id_regex.format(url_type), str(id_)) - if match: - deezer_id = match.group(3) - if deezer_id: - return deezer_id - return None + deezer_id = match.group(3) + return deezer_id if deezer_id else None def album_for_id(self, album_id): """Fetch an album by its Deezer ID or URL and return an From 7754dd27c548e02dcdae9fd3613c8cc8e42dacdf Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 2 Sep 2019 20:15:35 -0400 Subject: [PATCH 185/613] Make none_rec_action respect timid. #3242 --- beets/ui/commands.py | 4 ++++ docs/changelog.rst | 3 +++ 2 files changed, 7 insertions(+) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 57b627c77..e89a58bc1 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -481,6 +481,7 @@ def _summary_judgment(rec): queried. May also print to the console if a summary judgment is made. """ + print(rec) if config['import']['quiet']: if rec == Recommendation.strong: return importer.action.APPLY @@ -496,6 +497,9 @@ def _summary_judgment(rec): 'asis': importer.action.ASIS, 'ask': None, }) + # prompt the user if timid is enabled + if config['import']['timid'] and action == importer.action.ASIS: + return None else: return None diff --git a/docs/changelog.rst b/docs/changelog.rst index 690696c1d..da1948758 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -97,6 +97,9 @@ Fixes: * Fixed a bug that caused the UI to display incorrect track numbers for tracks with index 0 when the ``per_disc_numbering`` option was set. :bug:`3346` +* ``none_rec_action`` does not import automatically when ``timid`` is enabled. + Thanks to :user:`RollingStar`. + :bug:`3242` For plugin developers: From 80e51027e6fea955c39b239ebcba1692e8d0200f Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 2 Sep 2019 20:17:00 -0400 Subject: [PATCH 186/613] remove debug line --- beets/ui/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index e89a58bc1..13e431b4b 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -481,7 +481,6 @@ def _summary_judgment(rec): queried. May also print to the console if a summary judgment is made. """ - print(rec) if config['import']['quiet']: if rec == Recommendation.strong: return importer.action.APPLY From bd0cea9f1bfcc2b95206e05ea40423676f37ee6e Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 19:50:04 -0700 Subject: [PATCH 187/613] Factor out APIAutotaggerPlugin --- beets/autotag/__init__.py | 192 +++++++++++++++++++++++++++++++++-- beetsplug/deezer.py | 164 ++++++------------------------ beetsplug/discogs.py | 36 ++----- beetsplug/spotify.py | 205 ++++++++++---------------------------- 4 files changed, 275 insertions(+), 322 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 07d1feffa..0d538a72f 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -18,11 +18,21 @@ from __future__ import division, absolute_import, print_function +import re +from abc import abstractmethod, abstractproperty + from beets import logging from beets import config +from beets.plugins import BeetsPlugin # Parts of external interface. -from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa +from .hooks import ( + AlbumInfo, + TrackInfo, + AlbumMatch, + TrackMatch, + Distance, +) # noqa from .match import tag_item, tag_album, Proposal # noqa from .match import Recommendation # noqa @@ -32,6 +42,7 @@ log = logging.getLogger('beets') # Additional utilities for the main interface. + def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ @@ -72,14 +83,15 @@ def apply_metadata(album_info, mapping): for item, track_info in mapping.items(): # Artist or artist credit. if config['artist_credit']: - item.artist = (track_info.artist_credit or - track_info.artist or - album_info.artist_credit or - album_info.artist) - item.albumartist = (album_info.artist_credit or - album_info.artist) + item.artist = ( + track_info.artist_credit + or track_info.artist + or album_info.artist_credit + or album_info.artist + ) + item.albumartist = album_info.artist_credit or album_info.artist else: - item.artist = (track_info.artist or album_info.artist) + item.artist = track_info.artist or album_info.artist item.albumartist = album_info.artist # Album. @@ -87,8 +99,9 @@ def apply_metadata(album_info, mapping): # Artist sort and credit names. item.artist_sort = track_info.artist_sort or album_info.artist_sort - item.artist_credit = (track_info.artist_credit or - album_info.artist_credit) + item.artist_credit = ( + track_info.artist_credit or album_info.artist_credit + ) item.albumartist_sort = album_info.artist_sort item.albumartist_credit = album_info.artist_credit @@ -179,7 +192,7 @@ def apply_metadata(album_info, mapping): 'work', 'mb_workid', 'work_disambig', - ) + ), } # Don't overwrite fields with empty values unless the @@ -197,3 +210,160 @@ def apply_metadata(album_info, mapping): if value is None and not clobber: continue item[field] = value + + +def album_distance(config, data_source, album_info): + """Returns the ``data_source`` weight and the maximum source weight + for albums. + """ + dist = Distance() + if album_info.data_source == data_source: + dist.add('source', config['source_weight'].as_number()) + return dist + + +def track_distance(config, data_source, track_info): + """Returns the ``data_source`` weight and the maximum source weight + for individual tracks. + """ + dist = Distance() + if track_info.data_source == data_source: + dist.add('source', config['source_weight'].as_number()) + return dist + + +class APIAutotaggerPlugin(BeetsPlugin): + def __init__(self): + super(APIAutotaggerPlugin, self).__init__() + self.config.add({'source_weight': 0.5}) + + @abstractproperty + def id_regex(self): + raise NotImplementedError + + @abstractproperty + def data_source(self): + raise NotImplementedError + + @abstractproperty + def search_url(self): + raise NotImplementedError + + @abstractproperty + def album_url(self): + raise NotImplementedError + + @abstractproperty + def track_url(self): + raise NotImplementedError + + @abstractmethod + def _search_api(self, query_type, filters, keywords=''): + raise NotImplementedError + + @abstractmethod + def album_for_id(self, album_id): + raise NotImplementedError + + @abstractmethod + def track_for_id(self, track_id=None, track_data=None): + raise NotImplementedError + + @staticmethod + def get_artist(artists, id_key='id', name_key='name'): + """Returns an artist string (all artists) and an artist_id (the main + artist) for a list of artist object dicts. + + :param artists: Iterable of artist dicts returned by API. + :type artists: list[dict] + :param id_key: Key corresponding to ``artist_id`` value. + :type id_key: str + :param name_key: Keys corresponding to values to concatenate for ``artist``. + :type name_key: str + :return: Normalized artist string. + :rtype: str + """ + artist_id = None + artist_names = [] + for artist in artists: + if not artist_id: + artist_id = artist[id_key] + name = artist[name_key] + # Move articles to the front. + name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) + artist_names.append(name) + artist = ', '.join(artist_names).replace(' ,', ',') or None + return artist, artist_id + + def _get_id(self, url_type, id_): + """Parse an ID from its URL if necessary. + + :param url_type: Type of URL. Either 'album' or 'track'. + :type url_type: str + :param id_: Album/track ID or URL. + :type id_: str + :return: Album/track ID. + :rtype: str + """ + self._log.debug( + u"Searching {} for {} '{}'", self.data_source, url_type, id_ + ) + match = re.search( + self.id_regex['pattern'].format(url_type=url_type), str(id_) + ) + id_ = match.group(self.id_regex['match_group']) + return id_ if id_ else None + + def candidates(self, items, artist, album, va_likely): + """Returns a list of AlbumInfo objects for Search API results + matching an ``album`` and ``artist`` (if not various). + + :param items: List of items comprised by an album to be matched. + :type items: list[beets.library.Item] + :param artist: The artist of the album to be matched. + :type artist: str + :param album: The name of the album to be matched. + :type album: str + :param va_likely: True if the album to be matched likely has + Various Artists. + :type va_likely: bool + :return: Candidate AlbumInfo objects. + :rtype: list[beets.autotag.hooks.AlbumInfo] + """ + query_filters = {'album': album} + if not va_likely: + query_filters['artist'] = artist + albums = self._search_api(query_type='album', filters=query_filters) + return [self.album_for_id(album_id=album['id']) for album in albums] + + def item_candidates(self, item, artist, title): + """Returns a list of TrackInfo objects for Search API results + matching ``title`` and ``artist``. + + :param item: Singleton item to be matched. + :type item: beets.library.Item + :param artist: The artist of the track to be matched. + :type artist: str + :param title: The title of the track to be matched. + :type title: str + :return: Candidate TrackInfo objects. + :rtype: list[beets.autotag.hooks.TrackInfo] + """ + tracks = self._search_api( + query_type='track', keywords=title, filters={'artist': artist} + ) + return [self.track_for_id(track_data=track) for track in tracks] + + def album_distance(self, items, album_info, mapping): + return album_distance( + data_source=self.data_source, + album_info=album_info, + config=self.config, + ) + + def track_distance(self, item, track_info): + return track_distance( + data_source=self.data_source, + track_info=track_info, + config=self.config, + ) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 654fb59dd..215ea3bf1 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -17,7 +17,6 @@ """ from __future__ import absolute_import, print_function -import re import collections import six @@ -25,37 +24,24 @@ import unidecode import requests from beets import ui -from beets.plugins import BeetsPlugin -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance +from beets.autotag import APIAutotaggerPlugin +from beets.autotag.hooks import AlbumInfo, TrackInfo -class DeezerPlugin(BeetsPlugin): +class DeezerPlugin(APIAutotaggerPlugin): # Base URLs for the Deezer API # Documentation: https://developers.deezer.com/api/ search_url = 'https://api.deezer.com/search/' album_url = 'https://api.deezer.com/album/' track_url = 'https://api.deezer.com/track/' + data_source = 'Deezer' + id_regex = { + 'pattern': r'(^|deezer\.com/([a-z]*/)?{url_type}/)([0-9]*)', + 'match_group': 3, + } def __init__(self): super(DeezerPlugin, self).__init__() - self.config.add({'source_weight': 0.5}) - - def _get_deezer_id(self, url_type, id_): - """Parse a Deezer ID from its URL if necessary. - - :param url_type: Type of Deezer URL. Either 'album', 'artist', - 'playlist', or 'track'. - :type url_type: str - :param id_: Deezer ID or URL. - :type id_: str - :return: Deezer ID. - :rtype: str - """ - id_regex = r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)' - self._log.debug(u'Searching for {} {}', url_type, id_) - match = re.search(id_regex.format(url_type), str(id_)) - deezer_id = match.group(3) - return deezer_id if deezer_id else None def album_for_id(self, album_id): """Fetch an album by its Deezer ID or URL and return an @@ -66,12 +52,12 @@ class DeezerPlugin(BeetsPlugin): :return: AlbumInfo object for album. :rtype: beets.autotag.hooks.AlbumInfo or None """ - deezer_id = self._get_deezer_id('album', album_id) + deezer_id = self._get_id('album', album_id) if deezer_id is None: return None album_data = requests.get(self.album_url + deezer_id).json() - artist, artist_id = self._get_artist(album_data['contributors']) + artist, artist_id = self.get_artist(album_data['contributors']) release_date = album_data['release_date'] date_parts = [int(part) for part in release_date.split('-')] @@ -89,7 +75,7 @@ class DeezerPlugin(BeetsPlugin): else: raise ui.UserError( u"Invalid `release_date` returned " - u"by Deezer API: '{}'".format(release_date) + u"by {} API: '{}'".format(self.data_source, release_date) ) tracks_data = requests.get( @@ -109,7 +95,7 @@ class DeezerPlugin(BeetsPlugin): album=album_data['title'], album_id=deezer_id, artist=artist, - artist_credit=self._get_artist([album_data['artist']])[0], + artist_credit=self.get_artist([album_data['artist']])[0], artist_id=artist_id, tracks=tracks, albumtype=album_data['record_type'], @@ -120,7 +106,7 @@ class DeezerPlugin(BeetsPlugin): day=day, label=album_data['label'], mediums=max(medium_totals.keys()), - data_source='Deezer', + data_source=self.data_source, data_url=album_data['link'], ) @@ -132,7 +118,7 @@ class DeezerPlugin(BeetsPlugin): :return: TrackInfo object for track :rtype: beets.autotag.hooks.TrackInfo """ - artist, artist_id = self._get_artist( + artist, artist_id = self.get_artist( track_data.get('contributors', [track_data['artist']]) ) return TrackInfo( @@ -144,7 +130,7 @@ class DeezerPlugin(BeetsPlugin): index=track_data['track_position'], medium=track_data['disk_number'], medium_index=track_data['track_position'], - data_source='Deezer', + data_source=self.data_source, data_url=track_data['link'], ) @@ -162,7 +148,7 @@ class DeezerPlugin(BeetsPlugin): :rtype: beets.autotag.hooks.TrackInfo or None """ if track_data is None: - deezer_id = self._get_deezer_id('track', track_id) + deezer_id = self._get_id('track', track_id) if deezer_id is None: return None track_data = requests.get(self.track_url + deezer_id).json() @@ -176,107 +162,13 @@ class DeezerPlugin(BeetsPlugin): ).json()['data'] medium_total = 0 for i, track_data in enumerate(album_tracks_data, start=1): - if track_data['disc_number'] == track.medium: + if track_data['disk_number'] == track.medium: medium_total += 1 if track_data['id'] == track.track_id: track.index = i track.medium_total = medium_total return track - @staticmethod - def _get_artist(artists): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of Deezer artist object dicts. - - :param artists: Iterable of ``contributors`` or ``artist`` returned - by the Deezer Album (https://developers.deezer.com/api/album) or - Deezer Track (https://developers.deezer.com/api/track) APIs. - :type artists: list[dict] - :return: Normalized artist string - :rtype: str - """ - artist_id = None - artist_names = [] - for artist in artists: - if not artist_id: - artist_id = artist['id'] - name = artist['name'] - # Move articles to the front. - name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) - artist_names.append(name) - artist = ', '.join(artist_names).replace(' ,', ',') or None - return artist, artist_id - - def album_distance(self, items, album_info, mapping): - """Returns the Deezer source weight and the maximum source weight - for albums. - """ - dist = Distance() - if album_info.data_source == 'Deezer': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def track_distance(self, item, track_info): - """Returns the Deezer source weight and the maximum source weight - for individual tracks. - """ - dist = Distance() - if track_info.data_source == 'Deezer': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def candidates(self, items, artist, album, va_likely): - """Returns a list of AlbumInfo objects for Deezer Search API results - matching an ``album`` and ``artist`` (if not various). - - :param items: List of items comprised by an album to be matched. - :type items: list[beets.library.Item] - :param artist: The artist of the album to be matched. - :type artist: str - :param album: The name of the album to be matched. - :type album: str - :param va_likely: True if the album to be matched likely has - Various Artists. - :type va_likely: bool - :return: Candidate AlbumInfo objects. - :rtype: list[beets.autotag.hooks.AlbumInfo] - """ - query_filters = {'album': album} - if not va_likely: - query_filters['artist'] = artist - response_data = self._search_deezer( - query_type='album', filters=query_filters - ) - if response_data is None: - return [] - return [ - self.album_for_id(album_id=album_data['id']) - for album_data in response_data['data'] - ] - - def item_candidates(self, item, artist, title): - """Returns a list of TrackInfo objects for Deezer Search API results - matching ``title`` and ``artist``. - - :param item: Singleton item to be matched. - :type item: beets.library.Item - :param artist: The artist of the track to be matched. - :type artist: str - :param title: The title of the track to be matched. - :type title: str - :return: Candidate TrackInfo objects. - :rtype: list[beets.autotag.hooks.TrackInfo] - """ - response_data = self._search_deezer( - query_type='track', keywords=title, filters={'artist': artist} - ) - if response_data is None: - return [] - return [ - self.track_for_id(track_data=track_data) - for track_data in response_data['data'] - ] - @staticmethod def _construct_search_query(filters=None, keywords=''): """Construct a query string with the specified filters and keywords to @@ -299,7 +191,7 @@ class DeezerPlugin(BeetsPlugin): query = query.decode('utf8') return unidecode.unidecode(query) - def _search_deezer(self, query_type, filters=None, keywords=''): + def _search_api(self, query_type, filters=None, keywords=''): """Query the Deezer Search API for the specified ``keywords``, applying the provided ``filters``. @@ -320,12 +212,18 @@ class DeezerPlugin(BeetsPlugin): ) if not query: return None - self._log.debug(u"Searching Deezer for '{}'".format(query)) - response_data = requests.get( - self.search_url + query_type, params={'q': query} - ).json() - num_results = len(response_data['data']) self._log.debug( - u"Found {} results from Deezer for '{}'", num_results, query + u"Searching {} for '{}'".format(self.data_source, query) ) - return response_data if num_results > 0 else None + response = requests.get( + self.search_url + query_type, params={'q': query} + ) + response.raise_for_status() + response_data = response.json().get('data', []) + self._log.debug( + u"Found {} results from {} for '{}'", + self.data_source, + len(response_data), + query, + ) + return response_data diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 4996c5d7c..0c9d1b397 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,7 +20,8 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance +from beets.autotag import APIAutotaggerPlugin, album_distance +from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.plugins import BeetsPlugin import confuse from discogs_client import Release, Master, Client @@ -159,10 +160,11 @@ class DiscogsPlugin(BeetsPlugin): def album_distance(self, items, album_info, mapping): """Returns the album distance. """ - dist = Distance() - if album_info.data_source == 'Discogs': - dist.add('source', self.config['source_weight'].as_number()) - return dist + return album_distance( + data_source='Discogs', + album_info=album_info, + config=self.config + ) def candidates(self, items, artist, album, va_likely): """Returns a list of AlbumInfo objects for discogs search results @@ -292,7 +294,7 @@ class DiscogsPlugin(BeetsPlugin): self._log.warning(u"Release does not contain the required fields") return None - artist, artist_id = self.get_artist([a.data for a in result.artists]) + artist, artist_id = APIAutotaggerPlugin.get_artist([a.data for a in result.artists]) album = re.sub(r' +', ' ', result.title) album_id = result.data['id'] # Use `.data` to access the tracklist directly instead of the @@ -368,26 +370,6 @@ class DiscogsPlugin(BeetsPlugin): else: return None - def get_artist(self, artists): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of discogs album or track artists. - """ - artist_id = None - bits = [] - for i, artist in enumerate(artists): - if not artist_id: - artist_id = artist['id'] - name = artist['name'] - # Strip disambiguation number. - name = re.sub(r' \(\d+\)$', '', name) - # Move articles to the front. - name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name) - bits.append(name) - if artist['join'] and i < len(artists) - 1: - bits.append(artist['join']) - artist = ' '.join(bits).replace(' ,', ',') or None - return artist, artist_id - def get_tracks(self, tracklist): """Returns a list of TrackInfo objects for a discogs tracklist. """ @@ -551,7 +533,7 @@ class DiscogsPlugin(BeetsPlugin): title = track['title'] track_id = None medium, medium_index, _ = self.get_track_index(track['position']) - artist, artist_id = self.get_artist(track.get('artists', [])) + artist, artist_id = APIAutotaggerPlugin.get_artist(track.get('artists', [])) length = self.get_track_length(track['duration']) return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, length=length, index=index, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 06bca1190..47d10c62a 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -27,14 +27,14 @@ import collections import six import unidecode import requests +import confuse from beets import ui -from beets.plugins import BeetsPlugin -import confuse -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance +from beets.autotag import APIAutotaggerPlugin +from beets.autotag.hooks import AlbumInfo, TrackInfo -class SpotifyPlugin(BeetsPlugin): +class SpotifyPlugin(APIAutotaggerPlugin): # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' @@ -43,6 +43,14 @@ class SpotifyPlugin(BeetsPlugin): album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' playlist_partial = 'spotify:trackset:Playlist:' + data_source = 'Spotify' + + # Spotify IDs consist of 22 alphanumeric characters + # (zero-left-padded base62 representation of randomly generated UUID4) + id_regex = { + 'pattern': r'(^|open\.spotify\.com/{url_type}/)([0-9A-Za-z]{{22}})', + 'match_group': 2, + } def __init__(self): super(SpotifyPlugin, self).__init__() @@ -59,7 +67,6 @@ class SpotifyPlugin(BeetsPlugin): 'client_id': '4e414367a1d14c75a5c5129a627fcab8', 'client_secret': 'f82bdc09b2254f1a8286815d02fd46dc', 'tokenfile': 'spotify_token.json', - 'source_weight': 0.5, } ) self.config['client_secret'].redact = True @@ -140,26 +147,11 @@ class SpotifyPlugin(BeetsPlugin): self._authenticate() return self._handle_response(request_type, url, params=params) else: - raise ui.UserError(u'Spotify API error:\n{}', response.text) + raise ui.UserError( + u'{} API error:\n{}', self.data_source, response.text + ) return response.json() - def _get_spotify_id(self, url_type, id_): - """Parse a Spotify ID from its URL if necessary. - - :param url_type: Type of Spotify URL, either 'album' or 'track'. - :type url_type: str - :param id_: Spotify ID or URL. - :type id_: str - :return: Spotify ID. - :rtype: str - """ - # Spotify IDs consist of 22 alphanumeric characters - # (zero-left-padded base62 representation of randomly generated UUID4) - id_regex = r'(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})' - self._log.debug(u'Searching for {} {}', url_type, id_) - match = re.search(id_regex.format(url_type), id_) - return match.group(2) if match else None - def album_for_id(self, album_id): """Fetch an album by its Spotify ID or URL and return an AlbumInfo object or None if the album is not found. @@ -169,20 +161,20 @@ class SpotifyPlugin(BeetsPlugin): :return: AlbumInfo object for album :rtype: beets.autotag.hooks.AlbumInfo or None """ - spotify_id = self._get_spotify_id('album', album_id) + spotify_id = self._get_id('album', album_id) if spotify_id is None: return None - response_data = self._handle_response( + album_data = self._handle_response( requests.get, self.album_url + spotify_id ) - artist, artist_id = self._get_artist(response_data['artists']) + artist, artist_id = self.get_artist(album_data['artists']) date_parts = [ - int(part) for part in response_data['release_date'].split('-') + int(part) for part in album_data['release_date'].split('-') ] - release_date_precision = response_data['release_date_precision'] + release_date_precision = album_data['release_date_precision'] if release_date_precision == 'day': year, month, day = date_parts elif release_date_precision == 'month': @@ -195,14 +187,14 @@ class SpotifyPlugin(BeetsPlugin): else: raise ui.UserError( u"Invalid `release_date_precision` returned " - u"by Spotify API: '{}'".format(release_date_precision) + u"by {} API: '{}'".format( + self.data_source, release_date_precision + ) ) tracks = [] medium_totals = collections.defaultdict(int) - for i, track_data in enumerate( - response_data['tracks']['items'], start=1 - ): + for i, track_data in enumerate(album_data['tracks']['items'], start=1): track = self._get_track(track_data) track.index = i medium_totals[track.medium] += 1 @@ -211,21 +203,21 @@ class SpotifyPlugin(BeetsPlugin): track.medium_total = medium_totals[track.medium] return AlbumInfo( - album=response_data['name'], + album=album_data['name'], album_id=spotify_id, artist=artist, artist_id=artist_id, tracks=tracks, - albumtype=response_data['album_type'], - va=len(response_data['artists']) == 1 + albumtype=album_data['album_type'], + va=len(album_data['artists']) == 1 and artist.lower() == 'various artists', year=year, month=month, day=day, - label=response_data['label'], + label=album_data['label'], mediums=max(medium_totals.keys()), - data_source='Spotify', - data_url=response_data['external_urls']['spotify'], + data_source=self.data_source, + data_url=album_data['external_urls']['spotify'], ) def _get_track(self, track_data): @@ -237,7 +229,7 @@ class SpotifyPlugin(BeetsPlugin): :return: TrackInfo object for track :rtype: beets.autotag.hooks.TrackInfo """ - artist, artist_id = self._get_artist(track_data['artists']) + artist, artist_id = self.get_artist(track_data['artists']) return TrackInfo( title=track_data['name'], track_id=track_data['id'], @@ -247,7 +239,7 @@ class SpotifyPlugin(BeetsPlugin): index=track_data['track_number'], medium=track_data['disc_number'], medium_index=track_data['track_number'], - data_source='Spotify', + data_source=self.data_source, data_url=track_data['external_urls']['spotify'], ) @@ -265,7 +257,7 @@ class SpotifyPlugin(BeetsPlugin): :rtype: beets.autotag.hooks.TrackInfo or None """ if track_data is None: - spotify_id = self._get_spotify_id('track', track_id) + spotify_id = self._get_id('track', track_id) if spotify_id is None: return None track_data = self._handle_response( @@ -288,99 +280,6 @@ class SpotifyPlugin(BeetsPlugin): track.medium_total = medium_total return track - @staticmethod - def _get_artist(artists): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of Spotify artist object dicts. - - :param artists: Iterable of simplified Spotify artist objects - (https://developer.spotify.com/documentation/web-api/reference/object-model/#artist-object-simplified) - :type artists: list[dict] - :return: Normalized artist string - :rtype: str - """ - artist_id = None - artist_names = [] - for artist in artists: - if not artist_id: - artist_id = artist['id'] - name = artist['name'] - # Move articles to the front. - name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) - artist_names.append(name) - artist = ', '.join(artist_names).replace(' ,', ',') or None - return artist, artist_id - - def album_distance(self, items, album_info, mapping): - """Returns the Spotify source weight and the maximum source weight - for albums. - """ - dist = Distance() - if album_info.data_source == 'Spotify': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def track_distance(self, item, track_info): - """Returns the Spotify source weight and the maximum source weight - for individual tracks. - """ - dist = Distance() - if track_info.data_source == 'Spotify': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def candidates(self, items, artist, album, va_likely): - """Returns a list of AlbumInfo objects for Spotify Search API results - matching an ``album`` and ``artist`` (if not various). - - :param items: List of items comprised by an album to be matched. - :type items: list[beets.library.Item] - :param artist: The artist of the album to be matched. - :type artist: str - :param album: The name of the album to be matched. - :type album: str - :param va_likely: True if the album to be matched likely has - Various Artists. - :type va_likely: bool - :return: Candidate AlbumInfo objects. - :rtype: list[beets.autotag.hooks.AlbumInfo] - """ - query_filters = {'album': album} - if not va_likely: - query_filters['artist'] = artist - response_data = self._search_spotify( - query_type='album', filters=query_filters - ) - if response_data is None: - return [] - return [ - self.album_for_id(album_id=album_data['id']) - for album_data in response_data['albums']['items'] - ] - - def item_candidates(self, item, artist, title): - """Returns a list of TrackInfo objects for Spotify Search API results - matching ``title`` and ``artist``. - - :param item: Singleton item to be matched. - :type item: beets.library.Item - :param artist: The artist of the track to be matched. - :type artist: str - :param title: The title of the track to be matched. - :type title: str - :return: Candidate TrackInfo objects. - :rtype: list[beets.autotag.hooks.TrackInfo] - """ - response_data = self._search_spotify( - query_type='track', keywords=title, filters={'artist': artist} - ) - if response_data is None: - return [] - return [ - self.track_for_id(track_data=track_data) - for track_data in response_data['tracks']['items'] - ] - @staticmethod def _construct_search_query(filters=None, keywords=''): """Construct a query string with the specified filters and keywords to @@ -403,14 +302,12 @@ class SpotifyPlugin(BeetsPlugin): query = query.decode('utf8') return unidecode.unidecode(query) - def _search_spotify(self, query_type, filters=None, keywords=''): + def _search_api(self, query_type, filters=None, keywords=''): """Query the Spotify Search API for the specified ``keywords``, applying the provided ``filters``. - :param query_type: A comma-separated list of item types to search - across. Valid types are: 'album', 'artist', 'playlist', and - 'track'. Search results include hits from all the specified item - types. + :param query_type: Item type to search across. Valid types are: 'album', + 'artist', 'playlist', and 'track'. :type query_type: str :param filters: (Optional) Field filters to apply. :type filters: dict @@ -425,19 +322,25 @@ class SpotifyPlugin(BeetsPlugin): ) if not query: return None - self._log.debug(u"Searching Spotify for '{}'".format(query)) - response_data = self._handle_response( - requests.get, - self.search_url, - params={'q': query, 'type': query_type}, - ) - num_results = 0 - for result_type_data in response_data.values(): - num_results += len(result_type_data['items']) self._log.debug( - u"Found {} results from Spotify for '{}'", num_results, query + u"Searching {} for '{}'".format(self.data_source, query) ) - return response_data if num_results > 0 else None + response_data = ( + self._handle_response( + requests.get, + self.search_url, + params={'q': query, 'type': query_type}, + ) + .get(query_type + 's', {}) + .get('items', []) + ) + self._log.debug( + u"Found {} results from {} for '{}'", + self.data_source, + len(response_data), + query, + ) + return response_data def commands(self): def queries(lib, opts, args): @@ -529,7 +432,7 @@ class SpotifyPlugin(BeetsPlugin): # Query the Web API for each track, look for the items' JSON data query_filters = {'artist': artist, 'album': album} - response_data = self._search_spotify( + response_data = self._search_api( query_type='track', keywords=keywords, filters=query_filters ) if response_data is None: From f64bd65ddb7aab6aa4073a1e4ae46853336696ff Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 20:11:30 -0700 Subject: [PATCH 188/613] Remove unnecessary indexing --- beetsplug/spotify.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 47d10c62a..34ec4b7ee 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -432,16 +432,15 @@ class SpotifyPlugin(APIAutotaggerPlugin): # Query the Web API for each track, look for the items' JSON data query_filters = {'artist': artist, 'album': album} - response_data = self._search_api( + response_data_tracks = self._search_api( query_type='track', keywords=keywords, filters=query_filters ) - if response_data is None: + if not response_data_tracks: query = self._construct_search_query( keywords=keywords, filters=query_filters ) failures.append(query) continue - response_data_tracks = response_data['tracks']['items'] # Apply market filter if requested region_filter = self.config['region_filter'].get() From 1b05912ab9e5679ab7c4d320f76c2a63f38436b0 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 20:39:46 -0700 Subject: [PATCH 189/613] Appease flake8 --- beets/autotag/__init__.py | 3 ++- beetsplug/discogs.py | 8 ++++++-- beetsplug/spotify.py | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 0d538a72f..738ff4813 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -278,7 +278,8 @@ class APIAutotaggerPlugin(BeetsPlugin): :type artists: list[dict] :param id_key: Key corresponding to ``artist_id`` value. :type id_key: str - :param name_key: Keys corresponding to values to concatenate for ``artist``. + :param name_key: Keys corresponding to values to concatenate + for ``artist``. :type name_key: str :return: Normalized artist string. :rtype: str diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 0c9d1b397..47747420a 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -294,7 +294,9 @@ class DiscogsPlugin(BeetsPlugin): self._log.warning(u"Release does not contain the required fields") return None - artist, artist_id = APIAutotaggerPlugin.get_artist([a.data for a in result.artists]) + artist, artist_id = APIAutotaggerPlugin.get_artist( + [a.data for a in result.artists] + ) album = re.sub(r' +', ' ', result.title) album_id = result.data['id'] # Use `.data` to access the tracklist directly instead of the @@ -533,7 +535,9 @@ class DiscogsPlugin(BeetsPlugin): title = track['title'] track_id = None medium, medium_index, _ = self.get_track_index(track['position']) - artist, artist_id = APIAutotaggerPlugin.get_artist(track.get('artists', [])) + artist, artist_id = APIAutotaggerPlugin.get_artist( + track.get('artists', []) + ) length = self.get_track_length(track['duration']) return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, length=length, index=index, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 34ec4b7ee..66ee2a16d 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -306,8 +306,8 @@ class SpotifyPlugin(APIAutotaggerPlugin): """Query the Spotify Search API for the specified ``keywords``, applying the provided ``filters``. - :param query_type: Item type to search across. Valid types are: 'album', - 'artist', 'playlist', and 'track'. + :param query_type: Item type to search across. Valid types are: + 'album', 'artist', 'playlist', and 'track'. :type query_type: str :param filters: (Optional) Field filters to apply. :type filters: dict From 8010488f37be8d70c89c96aaa790b9a03f97817d Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 21:03:22 -0700 Subject: [PATCH 190/613] Modularize distance --- beets/autotag/__init__.py | 28 +++++++--------------------- beetsplug/discogs.py | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 738ff4813..9b3970f12 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -212,22 +212,12 @@ def apply_metadata(album_info, mapping): item[field] = value -def album_distance(config, data_source, album_info): +def get_distance(config, data_source, info): """Returns the ``data_source`` weight and the maximum source weight - for albums. + for albums or individual tracks. """ dist = Distance() - if album_info.data_source == data_source: - dist.add('source', config['source_weight'].as_number()) - return dist - - -def track_distance(config, data_source, track_info): - """Returns the ``data_source`` weight and the maximum source weight - for individual tracks. - """ - dist = Distance() - if track_info.data_source == data_source: + if info.data_source == data_source: dist.add('source', config['source_weight'].as_number()) return dist @@ -356,15 +346,11 @@ class APIAutotaggerPlugin(BeetsPlugin): return [self.track_for_id(track_data=track) for track in tracks] def album_distance(self, items, album_info, mapping): - return album_distance( - data_source=self.data_source, - album_info=album_info, - config=self.config, + return get_distance( + data_source=self.data_source, info=album_info, config=self.config ) def track_distance(self, item, track_info): - return track_distance( - data_source=self.data_source, - track_info=track_info, - config=self.config, + return get_distance( + data_source=self.data_source, info=track_info, config=self.config ) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 47747420a..47bee68d0 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,7 +20,7 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag import APIAutotaggerPlugin, album_distance +from beets.autotag import APIAutotaggerPlugin, get_distance from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.plugins import BeetsPlugin import confuse @@ -160,9 +160,18 @@ class DiscogsPlugin(BeetsPlugin): def album_distance(self, items, album_info, mapping): """Returns the album distance. """ - return album_distance( + return get_distance( data_source='Discogs', - album_info=album_info, + info=album_info, + config=self.config + ) + + def track_distance(self, item, track_info): + """Returns the track distance. + """ + return get_distance( + data_source='Discogs', + info=track_info, config=self.config ) From 30cfd7ff80d5c2079dfe20d8acd170bd8533fa32 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 21:18:07 -0700 Subject: [PATCH 191/613] Use positional str.format arg --- beets/autotag/__init__.py | 4 +--- beetsplug/deezer.py | 2 +- beetsplug/spotify.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 9b3970f12..132c4ce52 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -299,9 +299,7 @@ class APIAutotaggerPlugin(BeetsPlugin): self._log.debug( u"Searching {} for {} '{}'", self.data_source, url_type, id_ ) - match = re.search( - self.id_regex['pattern'].format(url_type=url_type), str(id_) - ) + match = re.search(self.id_regex['pattern'].format(url_type), str(id_)) id_ = match.group(self.id_regex['match_group']) return id_ if id_ else None diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 215ea3bf1..f68609470 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -36,7 +36,7 @@ class DeezerPlugin(APIAutotaggerPlugin): track_url = 'https://api.deezer.com/track/' data_source = 'Deezer' id_regex = { - 'pattern': r'(^|deezer\.com/([a-z]*/)?{url_type}/)([0-9]*)', + 'pattern': r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)', 'match_group': 3, } diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 66ee2a16d..221181fbe 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -48,7 +48,7 @@ class SpotifyPlugin(APIAutotaggerPlugin): # Spotify IDs consist of 22 alphanumeric characters # (zero-left-padded base62 representation of randomly generated UUID4) id_regex = { - 'pattern': r'(^|open\.spotify\.com/{url_type}/)([0-9A-Za-z]{{22}})', + 'pattern': r'(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})', 'match_group': 2, } From f7c6b5ba7f5b4e640d96b9074f3724ca55acfa9a Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 22:32:55 -0700 Subject: [PATCH 192/613] Fix str format arg order --- beetsplug/deezer.py | 2 +- beetsplug/spotify.py | 42 +++++++++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index f68609470..bd25f8ecc 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -222,8 +222,8 @@ class DeezerPlugin(APIAutotaggerPlugin): response_data = response.json().get('data', []) self._log.debug( u"Found {} results from {} for '{}'", - self.data_source, len(response_data), + self.data_source, query, ) return response_data diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 221181fbe..3ec576bbc 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -116,7 +116,9 @@ class SpotifyPlugin(APIAutotaggerPlugin): self.access_token = response.json()['access_token'] # Save the token for later use. - self._log.debug(u'Spotify access token: {}', self.access_token) + self._log.debug( + u'{} access token: {}', self.data_source, self.access_token + ) with open(self.tokenfile, 'w') as f: json.dump({'access_token': self.access_token}, f) @@ -142,7 +144,8 @@ class SpotifyPlugin(APIAutotaggerPlugin): if response.status_code != 200: if u'token expired' in response.text: self._log.debug( - 'Spotify access token has expired. Reauthenticating.' + '{} access token has expired. Reauthenticating.', + self.data_source, ) self._authenticate() return self._handle_response(request_type, url, params=params) @@ -336,8 +339,8 @@ class SpotifyPlugin(APIAutotaggerPlugin): ) self._log.debug( u"Found {} results from {} for '{}'", - self.data_source, len(response_data), + self.data_source, query, ) return response_data @@ -350,21 +353,23 @@ class SpotifyPlugin(APIAutotaggerPlugin): self._output_match_results(results) spotify_cmd = ui.Subcommand( - 'spotify', help=u'build a Spotify playlist' + 'spotify', help=u'build a {} playlist'.format(self.data_source) ) spotify_cmd.parser.add_option( u'-m', u'--mode', action='store', - help=u'"open" to open Spotify with playlist, ' - u'"list" to print (default)', + help=u'"open" to open {} with playlist, ' + u'"list" to print (default)'.format(self.data_source), ) spotify_cmd.parser.add_option( u'-f', u'--show-failures', action='store_true', dest='show_failures', - help=u'list tracks that did not match a Spotify ID', + help=u'list tracks that did not match a {} ID'.format( + self.data_source + ), ) spotify_cmd.func = queries return [spotify_cmd] @@ -404,7 +409,8 @@ class SpotifyPlugin(APIAutotaggerPlugin): if not items: self._log.debug( - u'Your beets query returned no items, skipping Spotify.' + u'Your beets query returned no items, skipping {}.', + self.data_source, ) return @@ -456,7 +462,8 @@ class SpotifyPlugin(APIAutotaggerPlugin): or self.config['tiebreak'].get() == 'first' ): self._log.debug( - u'Spotify track(s) found, count: {}', + u'{} track(s) found, count: {}', + self.data_source, len(response_data_tracks), ) chosen_result = response_data_tracks[0] @@ -475,16 +482,19 @@ class SpotifyPlugin(APIAutotaggerPlugin): if failure_count > 0: if self.config['show_failures'].get(): self._log.info( - u'{} track(s) did not match a Spotify ID:', failure_count + u'{} track(s) did not match a {} ID:', + failure_count, + self.data_source, ) for track in failures: self._log.info(u'track: {}', track) self._log.info(u'') else: self._log.warning( - u'{} track(s) did not match a Spotify ID;\n' + u'{} track(s) did not match a {} ID:\n' u'use --show-failures to display', failure_count, + self.data_source, ) return results @@ -500,11 +510,17 @@ class SpotifyPlugin(APIAutotaggerPlugin): if results: spotify_ids = [track_data['id'] for track_data in results] if self.config['mode'].get() == 'open': - self._log.info(u'Attempting to open Spotify with playlist') + self._log.info( + u'Attempting to open {} with playlist'.format( + self.data_source + ) + ) spotify_url = self.playlist_partial + ",".join(spotify_ids) webbrowser.open(spotify_url) else: for spotify_id in spotify_ids: print(self.open_track_url + spotify_id) else: - self._log.warning(u'No Spotify tracks found from beets query') + self._log.warning( + u'No {} tracks found from beets query'.format(self.data_source) + ) From a3fb8ebfff869a9cadcc8ce8278e3ffd50e045a5 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 4 Sep 2019 22:56:09 -0700 Subject: [PATCH 193/613] Formatting --- beetsplug/deezer.py | 3 ++- beetsplug/spotify.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index bd25f8ecc..886978536 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -29,12 +29,13 @@ from beets.autotag.hooks import AlbumInfo, TrackInfo class DeezerPlugin(APIAutotaggerPlugin): + data_source = 'Deezer' + # Base URLs for the Deezer API # Documentation: https://developers.deezer.com/api/ search_url = 'https://api.deezer.com/search/' album_url = 'https://api.deezer.com/album/' track_url = 'https://api.deezer.com/track/' - data_source = 'Deezer' id_regex = { 'pattern': r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)', 'match_group': 3, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 3ec576bbc..35ae7e462 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -35,6 +35,8 @@ from beets.autotag.hooks import AlbumInfo, TrackInfo class SpotifyPlugin(APIAutotaggerPlugin): + data_source = 'Spotify' + # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' @@ -42,8 +44,6 @@ class SpotifyPlugin(APIAutotaggerPlugin): search_url = 'https://api.spotify.com/v1/search' album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' - playlist_partial = 'spotify:trackset:Playlist:' - data_source = 'Spotify' # Spotify IDs consist of 22 alphanumeric characters # (zero-left-padded base62 representation of randomly generated UUID4) @@ -515,7 +515,9 @@ class SpotifyPlugin(APIAutotaggerPlugin): self.data_source ) ) - spotify_url = self.playlist_partial + ",".join(spotify_ids) + spotify_url = 'spotify:trackset:Playlist:' + ','.join( + spotify_ids + ) webbrowser.open(spotify_url) else: for spotify_id in spotify_ids: From 0d6df42d5fb540e6cbf3a303e1cc7bdfcd5dcc98 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 6 Sep 2019 12:08:26 -0700 Subject: [PATCH 194/613] Use Abstract Base Class --- beets/autotag/__init__.py | 4 ++-- beetsplug/deezer.py | 4 +++- beetsplug/spotify.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 132c4ce52..88621b34c 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -19,7 +19,7 @@ from __future__ import division, absolute_import, print_function import re -from abc import abstractmethod, abstractproperty +from abc import ABC, abstractmethod, abstractproperty from beets import logging from beets import config @@ -222,7 +222,7 @@ def get_distance(config, data_source, info): return dist -class APIAutotaggerPlugin(BeetsPlugin): +class APIAutotaggerPlugin(ABC): def __init__(self): super(APIAutotaggerPlugin, self).__init__() self.config.add({'source_weight': 0.5}) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 886978536..a8b4651e6 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -26,9 +26,10 @@ import requests from beets import ui from beets.autotag import APIAutotaggerPlugin from beets.autotag.hooks import AlbumInfo, TrackInfo +from beets.plugins import BeetsPlugin -class DeezerPlugin(APIAutotaggerPlugin): +class DeezerPlugin(APIAutotaggerPlugin, BeetsPlugin): data_source = 'Deezer' # Base URLs for the Deezer API @@ -36,6 +37,7 @@ class DeezerPlugin(APIAutotaggerPlugin): search_url = 'https://api.deezer.com/search/' album_url = 'https://api.deezer.com/album/' track_url = 'https://api.deezer.com/track/' + id_regex = { 'pattern': r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)', 'match_group': 3, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 35ae7e462..43bd8eb76 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -32,9 +32,10 @@ import confuse from beets import ui from beets.autotag import APIAutotaggerPlugin from beets.autotag.hooks import AlbumInfo, TrackInfo +from beets.plugins import BeetsPlugin -class SpotifyPlugin(APIAutotaggerPlugin): +class SpotifyPlugin(APIAutotaggerPlugin, BeetsPlugin): data_source = 'Spotify' # Base URLs for the Spotify API From 12a8e0a792163f78f75db45e753beba77bb3bc04 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 6 Sep 2019 12:41:24 -0700 Subject: [PATCH 195/613] Fix Spotify API error formatting --- beetsplug/spotify.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 43bd8eb76..4ca85454d 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -152,7 +152,9 @@ class SpotifyPlugin(APIAutotaggerPlugin, BeetsPlugin): return self._handle_response(request_type, url, params=params) else: raise ui.UserError( - u'{} API error:\n{}', self.data_source, response.text + u'{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( + self.data_source, response.text, url, params + ) ) return response.json() From 4a6fa5657b1948e9407186b076f465d75d1259c2 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 6 Sep 2019 13:11:28 -0700 Subject: [PATCH 196/613] Formatting --- beetsplug/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index a8b4651e6..0a1855a81 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -37,7 +37,7 @@ class DeezerPlugin(APIAutotaggerPlugin, BeetsPlugin): search_url = 'https://api.deezer.com/search/' album_url = 'https://api.deezer.com/album/' track_url = 'https://api.deezer.com/track/' - + id_regex = { 'pattern': r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)', 'match_group': 3, From 46065c3c8ecaee641ce2b189c59b78b4ec0a8628 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 6 Sep 2019 15:20:05 -0700 Subject: [PATCH 197/613] Use `six.with_metaclass` --- beets/autotag/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 88621b34c..17f731c28 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -19,11 +19,12 @@ from __future__ import division, absolute_import, print_function import re -from abc import ABC, abstractmethod, abstractproperty +from abc import ABCMeta, abstractmethod, abstractproperty + +import six from beets import logging from beets import config -from beets.plugins import BeetsPlugin # Parts of external interface. from .hooks import ( @@ -222,7 +223,8 @@ def get_distance(config, data_source, info): return dist -class APIAutotaggerPlugin(ABC): +@six.with_metaclass(ABCMeta) +class APIAutotaggerPlugin(object): def __init__(self): super(APIAutotaggerPlugin, self).__init__() self.config.add({'source_weight': 0.5}) From 867242da656863ae74bc1227361b8c0433b46aea Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 6 Sep 2019 15:25:06 -0700 Subject: [PATCH 198/613] `with_metaclass` --> `add_metaclass` --- beets/autotag/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 17f731c28..cec63ade1 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -19,7 +19,7 @@ from __future__ import division, absolute_import, print_function import re -from abc import ABCMeta, abstractmethod, abstractproperty +import abc import six @@ -223,41 +223,41 @@ def get_distance(config, data_source, info): return dist -@six.with_metaclass(ABCMeta) +@six.add_metaclass(abc.ABCMeta) class APIAutotaggerPlugin(object): def __init__(self): super(APIAutotaggerPlugin, self).__init__() self.config.add({'source_weight': 0.5}) - @abstractproperty + @abc.abstractproperty def id_regex(self): raise NotImplementedError - @abstractproperty + @abc.abstractproperty def data_source(self): raise NotImplementedError - @abstractproperty + @abc.abstractproperty def search_url(self): raise NotImplementedError - @abstractproperty + @abc.abstractproperty def album_url(self): raise NotImplementedError - @abstractproperty + @abc.abstractproperty def track_url(self): raise NotImplementedError - @abstractmethod + @abc.abstractmethod def _search_api(self, query_type, filters, keywords=''): raise NotImplementedError - @abstractmethod + @abc.abstractmethod def album_for_id(self, album_id): raise NotImplementedError - @abstractmethod + @abc.abstractmethod def track_for_id(self, track_id=None, track_data=None): raise NotImplementedError From 112941b944b5f588ad671d59cc4f73e9b5ef2d10 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 6 Sep 2019 17:24:26 -0700 Subject: [PATCH 199/613] Guard against None match --- beets/autotag/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index cec63ade1..8c0e62067 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -302,8 +302,11 @@ class APIAutotaggerPlugin(object): u"Searching {} for {} '{}'", self.data_source, url_type, id_ ) match = re.search(self.id_regex['pattern'].format(url_type), str(id_)) - id_ = match.group(self.id_regex['match_group']) - return id_ if id_ else None + if match: + id_ = match.group(self.id_regex['match_group']) + if id_: + return id_ + return None def candidates(self, items, artist, album, va_likely): """Returns a list of AlbumInfo objects for Search API results From bdb756550097c80dd8b4de7e502e4f54ef75d14d Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 7 Sep 2019 00:48:19 -0700 Subject: [PATCH 200/613] Avoid nested capturing groups --- beetsplug/deezer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 0a1855a81..787139e7c 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -39,8 +39,8 @@ class DeezerPlugin(APIAutotaggerPlugin, BeetsPlugin): track_url = 'https://api.deezer.com/track/' id_regex = { - 'pattern': r'(^|deezer\.com/([a-z]*/)?{}/)([0-9]*)', - 'match_group': 3, + 'pattern': r'(^|deezer\.com/)([a-z]*/)?({}/)([0-9]*)', + 'match_group': 4, } def __init__(self): From 0a700c75a22cd9afb48ef90e15abd2cfd8d7e301 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 7 Sep 2019 01:07:44 -0700 Subject: [PATCH 201/613] Optional capturing groups --- beetsplug/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 787139e7c..cab3b9a5f 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -39,7 +39,7 @@ class DeezerPlugin(APIAutotaggerPlugin, BeetsPlugin): track_url = 'https://api.deezer.com/track/' id_regex = { - 'pattern': r'(^|deezer\.com/)([a-z]*/)?({}/)([0-9]*)', + 'pattern': r'(^|deezer\.com/)?([a-z]*/)?({}/)?([0-9]*)', 'match_group': 4, } From c9468350ec9e3a0942177ef9a54b1b0bec39db2f Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 7 Sep 2019 11:49:54 -0400 Subject: [PATCH 202/613] Timid always prompts. Clarify docstring. --- beets/ui/commands.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 13e431b4b..7941be46b 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -477,10 +477,11 @@ def summarize_items(items, singleton): def _summary_judgment(rec): """Determines whether a decision should be made without even asking the user. This occurs in quiet mode and when an action is chosen for - NONE recommendations. Return an action or None if the user should be - queried. May also print to the console if a summary judgment is - made. + NONE recommendations. Return None if the user should be queried. + Otherwise, returns an action. May also print to the console if a + summary judgment is made. """ + if config['import']['quiet']: if rec == Recommendation.strong: return importer.action.APPLY @@ -489,17 +490,14 @@ def _summary_judgment(rec): 'skip': importer.action.SKIP, 'asis': importer.action.ASIS, }) - + elif config['import']['timid']: + return None elif rec == Recommendation.none: action = config['import']['none_rec_action'].as_choice({ 'skip': importer.action.SKIP, 'asis': importer.action.ASIS, 'ask': None, }) - # prompt the user if timid is enabled - if config['import']['timid'] and action == importer.action.ASIS: - return None - else: return None From 959a05845811c8ae70d993220fdc7db1a5efd6dd Mon Sep 17 00:00:00 2001 From: Christopher Larson Date: Mon, 15 Jul 2019 07:20:37 -0700 Subject: [PATCH 203/613] library: show album id in empty album error This makes it possible to recover from this case, as you can correct whatever caused it, by either fixing album ids on tracks or removing the empty album id. Signed-off-by: Christopher Larson --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 4c7fb894c..59791959d 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1119,7 +1119,7 @@ class Album(LibModel): """ item = self.items().get() if not item: - raise ValueError(u'empty album') + raise ValueError(u'empty album for album id %d' % self.id) return os.path.dirname(item.path) def _albumtotal(self): From 732e372ed2687853b524f66786eecb3e07152429 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Mon, 9 Sep 2019 17:31:42 -0700 Subject: [PATCH 204/613] Rename/move to `plugins.MetadataSourcePlugin`, fix formatting --- beets/autotag/__init__.py | 182 +++----------------------------------- beets/plugins.py | 148 +++++++++++++++++++++++++++++++ beetsplug/deezer.py | 8 +- beetsplug/spotify.py | 7 +- docs/changelog.rst | 11 ++- docs/plugins/deezer.rst | 8 +- 6 files changed, 173 insertions(+), 191 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 8c0e62067..07d1feffa 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -18,22 +18,11 @@ from __future__ import division, absolute_import, print_function -import re -import abc - -import six - from beets import logging from beets import config # Parts of external interface. -from .hooks import ( - AlbumInfo, - TrackInfo, - AlbumMatch, - TrackMatch, - Distance, -) # noqa +from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa from .match import tag_item, tag_album, Proposal # noqa from .match import Recommendation # noqa @@ -43,7 +32,6 @@ log = logging.getLogger('beets') # Additional utilities for the main interface. - def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ @@ -84,15 +72,14 @@ def apply_metadata(album_info, mapping): for item, track_info in mapping.items(): # Artist or artist credit. if config['artist_credit']: - item.artist = ( - track_info.artist_credit - or track_info.artist - or album_info.artist_credit - or album_info.artist - ) - item.albumartist = album_info.artist_credit or album_info.artist + item.artist = (track_info.artist_credit or + track_info.artist or + album_info.artist_credit or + album_info.artist) + item.albumartist = (album_info.artist_credit or + album_info.artist) else: - item.artist = track_info.artist or album_info.artist + item.artist = (track_info.artist or album_info.artist) item.albumartist = album_info.artist # Album. @@ -100,9 +87,8 @@ def apply_metadata(album_info, mapping): # Artist sort and credit names. item.artist_sort = track_info.artist_sort or album_info.artist_sort - item.artist_credit = ( - track_info.artist_credit or album_info.artist_credit - ) + item.artist_credit = (track_info.artist_credit or + album_info.artist_credit) item.albumartist_sort = album_info.artist_sort item.albumartist_credit = album_info.artist_credit @@ -193,7 +179,7 @@ def apply_metadata(album_info, mapping): 'work', 'mb_workid', 'work_disambig', - ), + ) } # Don't overwrite fields with empty values unless the @@ -211,149 +197,3 @@ def apply_metadata(album_info, mapping): if value is None and not clobber: continue item[field] = value - - -def get_distance(config, data_source, info): - """Returns the ``data_source`` weight and the maximum source weight - for albums or individual tracks. - """ - dist = Distance() - if info.data_source == data_source: - dist.add('source', config['source_weight'].as_number()) - return dist - - -@six.add_metaclass(abc.ABCMeta) -class APIAutotaggerPlugin(object): - def __init__(self): - super(APIAutotaggerPlugin, self).__init__() - self.config.add({'source_weight': 0.5}) - - @abc.abstractproperty - def id_regex(self): - raise NotImplementedError - - @abc.abstractproperty - def data_source(self): - raise NotImplementedError - - @abc.abstractproperty - def search_url(self): - raise NotImplementedError - - @abc.abstractproperty - def album_url(self): - raise NotImplementedError - - @abc.abstractproperty - def track_url(self): - raise NotImplementedError - - @abc.abstractmethod - def _search_api(self, query_type, filters, keywords=''): - raise NotImplementedError - - @abc.abstractmethod - def album_for_id(self, album_id): - raise NotImplementedError - - @abc.abstractmethod - def track_for_id(self, track_id=None, track_data=None): - raise NotImplementedError - - @staticmethod - def get_artist(artists, id_key='id', name_key='name'): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of artist object dicts. - - :param artists: Iterable of artist dicts returned by API. - :type artists: list[dict] - :param id_key: Key corresponding to ``artist_id`` value. - :type id_key: str - :param name_key: Keys corresponding to values to concatenate - for ``artist``. - :type name_key: str - :return: Normalized artist string. - :rtype: str - """ - artist_id = None - artist_names = [] - for artist in artists: - if not artist_id: - artist_id = artist[id_key] - name = artist[name_key] - # Move articles to the front. - name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) - artist_names.append(name) - artist = ', '.join(artist_names).replace(' ,', ',') or None - return artist, artist_id - - def _get_id(self, url_type, id_): - """Parse an ID from its URL if necessary. - - :param url_type: Type of URL. Either 'album' or 'track'. - :type url_type: str - :param id_: Album/track ID or URL. - :type id_: str - :return: Album/track ID. - :rtype: str - """ - self._log.debug( - u"Searching {} for {} '{}'", self.data_source, url_type, id_ - ) - match = re.search(self.id_regex['pattern'].format(url_type), str(id_)) - if match: - id_ = match.group(self.id_regex['match_group']) - if id_: - return id_ - return None - - def candidates(self, items, artist, album, va_likely): - """Returns a list of AlbumInfo objects for Search API results - matching an ``album`` and ``artist`` (if not various). - - :param items: List of items comprised by an album to be matched. - :type items: list[beets.library.Item] - :param artist: The artist of the album to be matched. - :type artist: str - :param album: The name of the album to be matched. - :type album: str - :param va_likely: True if the album to be matched likely has - Various Artists. - :type va_likely: bool - :return: Candidate AlbumInfo objects. - :rtype: list[beets.autotag.hooks.AlbumInfo] - """ - query_filters = {'album': album} - if not va_likely: - query_filters['artist'] = artist - albums = self._search_api(query_type='album', filters=query_filters) - return [self.album_for_id(album_id=album['id']) for album in albums] - - def item_candidates(self, item, artist, title): - """Returns a list of TrackInfo objects for Search API results - matching ``title`` and ``artist``. - - :param item: Singleton item to be matched. - :type item: beets.library.Item - :param artist: The artist of the track to be matched. - :type artist: str - :param title: The title of the track to be matched. - :type title: str - :return: Candidate TrackInfo objects. - :rtype: list[beets.autotag.hooks.TrackInfo] - """ - tracks = self._search_api( - query_type='track', keywords=title, filters={'artist': artist} - ) - return [self.track_for_id(track_data=track) for track in tracks] - - def album_distance(self, items, album_info, mapping): - return get_distance( - data_source=self.data_source, info=album_info, config=self.config - ) - - def track_distance(self, item, track_info): - return get_distance( - data_source=self.data_source, info=track_info, config=self.config - ) diff --git a/beets/plugins.py b/beets/plugins.py index 7c98225ca..c5db5f4bd 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -20,12 +20,14 @@ from __future__ import division, absolute_import, print_function import traceback import re import inspect +import abc from collections import defaultdict from functools import wraps import beets from beets import logging +from beets.autotag.hooks import Distance import mediafile import six @@ -576,3 +578,149 @@ def notify_info_yielded(event): yield v return decorated return decorator + + +def get_distance(config, data_source, info): + """Returns the ``data_source`` weight and the maximum source weight + for albums or individual tracks. + """ + dist = Distance() + if info.data_source == data_source: + dist.add('source', config['source_weight'].as_number()) + return dist + + +@six.add_metaclass(abc.ABCMeta) +class MetadataSourcePlugin(object): + def __init__(self): + super(MetadataSourcePlugin, self).__init__() + self.config.add({'source_weight': 0.5}) + + @abc.abstractproperty + def id_regex(self): + raise NotImplementedError + + @abc.abstractproperty + def data_source(self): + raise NotImplementedError + + @abc.abstractproperty + def search_url(self): + raise NotImplementedError + + @abc.abstractproperty + def album_url(self): + raise NotImplementedError + + @abc.abstractproperty + def track_url(self): + raise NotImplementedError + + @abc.abstractmethod + def _search_api(self, query_type, filters, keywords=''): + raise NotImplementedError + + @abc.abstractmethod + def album_for_id(self, album_id): + raise NotImplementedError + + @abc.abstractmethod + def track_for_id(self, track_id=None, track_data=None): + raise NotImplementedError + + @staticmethod + def get_artist(artists, id_key='id', name_key='name'): + """Returns an artist string (all artists) and an artist_id (the main + artist) for a list of artist object dicts. + + :param artists: Iterable of artist dicts returned by API. + :type artists: list[dict] + :param id_key: Key corresponding to ``artist_id`` value. + :type id_key: str + :param name_key: Keys corresponding to values to concatenate + for ``artist``. + :type name_key: str + :return: Normalized artist string. + :rtype: str + """ + artist_id = None + artist_names = [] + for artist in artists: + if not artist_id: + artist_id = artist[id_key] + name = artist[name_key] + # Move articles to the front. + name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) + artist_names.append(name) + artist = ', '.join(artist_names).replace(' ,', ',') or None + return artist, artist_id + + def _get_id(self, url_type, id_): + """Parse an ID from its URL if necessary. + + :param url_type: Type of URL. Either 'album' or 'track'. + :type url_type: str + :param id_: Album/track ID or URL. + :type id_: str + :return: Album/track ID. + :rtype: str + """ + self._log.debug( + u"Searching {} for {} '{}'", self.data_source, url_type, id_ + ) + match = re.search(self.id_regex['pattern'].format(url_type), str(id_)) + if match: + id_ = match.group(self.id_regex['match_group']) + if id_: + return id_ + return None + + def candidates(self, items, artist, album, va_likely): + """Returns a list of AlbumInfo objects for Search API results + matching an ``album`` and ``artist`` (if not various). + + :param items: List of items comprised by an album to be matched. + :type items: list[beets.library.Item] + :param artist: The artist of the album to be matched. + :type artist: str + :param album: The name of the album to be matched. + :type album: str + :param va_likely: True if the album to be matched likely has + Various Artists. + :type va_likely: bool + :return: Candidate AlbumInfo objects. + :rtype: list[beets.autotag.hooks.AlbumInfo] + """ + query_filters = {'album': album} + if not va_likely: + query_filters['artist'] = artist + albums = self._search_api(query_type='album', filters=query_filters) + return [self.album_for_id(album_id=album['id']) for album in albums] + + def item_candidates(self, item, artist, title): + """Returns a list of TrackInfo objects for Search API results + matching ``title`` and ``artist``. + + :param item: Singleton item to be matched. + :type item: beets.library.Item + :param artist: The artist of the track to be matched. + :type artist: str + :param title: The title of the track to be matched. + :type title: str + :return: Candidate TrackInfo objects. + :rtype: list[beets.autotag.hooks.TrackInfo] + """ + tracks = self._search_api( + query_type='track', keywords=title, filters={'artist': artist} + ) + return [self.track_for_id(track_data=track) for track in tracks] + + def album_distance(self, items, album_info, mapping): + return get_distance( + data_source=self.data_source, info=album_info, config=self.config + ) + + def track_distance(self, item, track_info): + return get_distance( + data_source=self.data_source, info=track_info, config=self.config + ) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index cab3b9a5f..f84a6d30e 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -24,12 +24,10 @@ import unidecode import requests from beets import ui -from beets.autotag import APIAutotaggerPlugin -from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import BeetsPlugin +from beets.plugins import MetadataSourcePlugin, BeetsPlugin -class DeezerPlugin(APIAutotaggerPlugin, BeetsPlugin): +class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Deezer' # Base URLs for the Deezer API @@ -224,7 +222,7 @@ class DeezerPlugin(APIAutotaggerPlugin, BeetsPlugin): response.raise_for_status() response_data = response.json().get('data', []) self._log.debug( - u"Found {} results from {} for '{}'", + u"Found {} result(s) from {} for '{}'", len(response_data), self.data_source, query, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 4ca85454d..8fe0d394c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -30,12 +30,11 @@ import requests import confuse from beets import ui -from beets.autotag import APIAutotaggerPlugin from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import BeetsPlugin +from beets.plugins import MetadataSourcePlugin, BeetsPlugin -class SpotifyPlugin(APIAutotaggerPlugin, BeetsPlugin): +class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Spotify' # Base URLs for the Spotify API @@ -341,7 +340,7 @@ class SpotifyPlugin(APIAutotaggerPlugin, BeetsPlugin): .get('items', []) ) self._log.debug( - u"Found {} results from {} for '{}'", + u"Found {} result(s) from {} for '{}'", len(response_data), self.data_source, query, diff --git a/docs/changelog.rst b/docs/changelog.rst index 7544db431..c9cd03dab 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -66,10 +66,9 @@ New features: * The 'data_source' field is now also applied as an album-level flexible attribute during imports, allowing for more refined album level searches. :bug:`3350` :bug:`1693` -* :doc:`/plugins/deezer`: - * Added Deezer plugin as an import metadata provider: you can now match tracks - and albums using the `Deezer`_ database. - Thanks to :user:`rhlahuja`. +* :doc:`/plugins/deezer`: Added Deezer plugin as an import metadata provider: + you can now match tracks and albums using the `Deezer`_ database. + Thanks to :user:`rhlahuja`. Fixes: @@ -128,6 +127,10 @@ For plugin developers: longer separate R128 backend instances. Instead the targetlevel is passed to ``compute_album_gain`` and ``compute_track_gain``. :bug:`3065` +* The ``beets.plugins.MetadataSourcePlugin`` base class has been added to + simplify development of plugins which query album, track, and search + APIs to provide metadata matches for the importer. Refer to the Spotify and + Deezer plugins for examples of using this template class. For packagers: diff --git a/docs/plugins/deezer.rst b/docs/plugins/deezer.rst index c00d1d68a..cb964c612 100644 --- a/docs/plugins/deezer.rst +++ b/docs/plugins/deezer.rst @@ -8,12 +8,6 @@ The ``deezer`` plugin provides metadata matches for the importer using the .. _Album: https://developers.deezer.com/api/album .. _Track: https://developers.deezer.com/api/track -Why Use This Plugin? --------------------- - -* You're a Beets user. -* You want to autotag music with metadata from the Deezer API. - Basic Usage ----------- First, enable the ``deezer`` plugin (see :ref:`using-plugins`). @@ -28,7 +22,7 @@ Configuration ------------- Put these options in config.yaml under the ``deezer:`` section: -- **source_weight**: Penalty applied to Spotify matches during import. Set to +- **source_weight**: Penalty applied to Deezer matches during import. Set to 0.0 to disable. Default: ``0.5``. From 68e91b18b0ff1801e7d663442b1aa9cce3005ca2 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Mon, 9 Sep 2019 17:33:42 -0700 Subject: [PATCH 205/613] Fix discogs.py `MetadataSourcePlugin` refs --- beetsplug/discogs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 47bee68d0..bccf1f7e2 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,9 +20,8 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag import APIAutotaggerPlugin, get_distance from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import BeetsPlugin +from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance import confuse from discogs_client import Release, Master, Client from discogs_client.exceptions import DiscogsAPIError @@ -303,7 +302,7 @@ class DiscogsPlugin(BeetsPlugin): self._log.warning(u"Release does not contain the required fields") return None - artist, artist_id = APIAutotaggerPlugin.get_artist( + artist, artist_id = MetadataSourcePlugin.get_artist( [a.data for a in result.artists] ) album = re.sub(r' +', ' ', result.title) @@ -544,7 +543,7 @@ class DiscogsPlugin(BeetsPlugin): title = track['title'] track_id = None medium, medium_index, _ = self.get_track_index(track['position']) - artist, artist_id = APIAutotaggerPlugin.get_artist( + artist, artist_id = MetadataSourcePlugin.get_artist( track.get('artists', []) ) length = self.get_track_length(track['duration']) From c531b1628e88460374511449f94c7c9260b01146 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Mon, 9 Sep 2019 18:28:20 -0700 Subject: [PATCH 206/613] Avoid circular import --- beets/autotag/hooks.py | 10 ++++++++++ beets/plugins.py | 14 +++----------- beetsplug/discogs.py | 4 ++-- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 686423360..9cb6866ab 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -652,3 +652,13 @@ def item_candidates(item, artist, title): # Plugin candidates. for candidate in plugins.item_candidates(item, artist, title): yield candidate + + +def get_distance(config, data_source, info): + """Returns the ``data_source`` weight and the maximum source weight + for albums or individual tracks. + """ + dist = Distance() + if info.data_source == data_source: + dist.add('source', config['source_weight'].as_number()) + return dist diff --git a/beets/plugins.py b/beets/plugins.py index c5db5f4bd..19374868f 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -27,10 +27,12 @@ from functools import wraps import beets from beets import logging -from beets.autotag.hooks import Distance import mediafile import six +from .autotag.hooks import get_distance + + PLUGIN_NAMESPACE = 'beetsplug' # Plugins using the Last.fm API can share the same API key. @@ -580,16 +582,6 @@ def notify_info_yielded(event): return decorator -def get_distance(config, data_source, info): - """Returns the ``data_source`` weight and the maximum source weight - for albums or individual tracks. - """ - dist = Distance() - if info.data_source == data_source: - dist.add('source', config['source_weight'].as_number()) - return dist - - @six.add_metaclass(abc.ABCMeta) class MetadataSourcePlugin(object): def __init__(self): diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index bccf1f7e2..7addbc2ca 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,8 +20,8 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance +from beets.autotag.hooks import AlbumInfo, TrackInfo, get_distance +from beets.plugins import MetadataSourcePlugin, BeetsPlugin import confuse from discogs_client import Release, Master, Client from discogs_client.exceptions import DiscogsAPIError From 1de0af669df6fd0fb2bf52f2d1e319298256595e Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Mon, 9 Sep 2019 18:45:12 -0700 Subject: [PATCH 207/613] Try absolute import --- beets/plugins.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 19374868f..e76c99dfd 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -27,11 +27,10 @@ from functools import wraps import beets from beets import logging +from beets.autotag.hooks import get_distance import mediafile import six -from .autotag.hooks import get_distance - PLUGIN_NAMESPACE = 'beetsplug' From 84b13475e03eda560570c95a1f8ac9bc1ff936a3 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Mon, 9 Sep 2019 19:13:24 -0700 Subject: [PATCH 208/613] Move `get_distance` to `beets.autotag` --- beets/autotag/__init__.py | 18 +++++++++++++++++- beets/autotag/hooks.py | 10 ---------- beets/plugins.py | 2 +- beetsplug/discogs.py | 2 +- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 07d1feffa..2c9c03ffb 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -22,7 +22,13 @@ from beets import logging from beets import config # Parts of external interface. -from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch # noqa +from .hooks import ( + AlbumInfo, + TrackInfo, + AlbumMatch, + TrackMatch, + Distance, +) # noqa from .match import tag_item, tag_album, Proposal # noqa from .match import Recommendation # noqa @@ -197,3 +203,13 @@ def apply_metadata(album_info, mapping): if value is None and not clobber: continue item[field] = value + + +def get_distance(config, data_source, info): + """Returns the ``data_source`` weight and the maximum source weight + for albums or individual tracks. + """ + dist = Distance() + if info.data_source == data_source: + dist.add('source', config['source_weight'].as_number()) + return dist diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9cb6866ab..686423360 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -652,13 +652,3 @@ def item_candidates(item, artist, title): # Plugin candidates. for candidate in plugins.item_candidates(item, artist, title): yield candidate - - -def get_distance(config, data_source, info): - """Returns the ``data_source`` weight and the maximum source weight - for albums or individual tracks. - """ - dist = Distance() - if info.data_source == data_source: - dist.add('source', config['source_weight'].as_number()) - return dist diff --git a/beets/plugins.py b/beets/plugins.py index e76c99dfd..cfeb5aa0b 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -27,7 +27,7 @@ from functools import wraps import beets from beets import logging -from beets.autotag.hooks import get_distance +from beets.autotag import get_distance import mediafile import six diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 7addbc2ca..e7ac73d64 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,7 +20,7 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag.hooks import AlbumInfo, TrackInfo, get_distance +from beets.autotag import AlbumInfo, TrackInfo, get_distance from beets.plugins import MetadataSourcePlugin, BeetsPlugin import confuse from discogs_client import Release, Master, Client From 2b0cf3e0021e9a95f48eea876bb2ef3abd139f8c Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Tue, 10 Sep 2019 22:39:06 -0700 Subject: [PATCH 209/613] Try absolute import --- beets/plugins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index cfeb5aa0b..b429adfe7 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -27,7 +27,7 @@ from functools import wraps import beets from beets import logging -from beets.autotag import get_distance +import beets.autotag import mediafile import six @@ -707,11 +707,11 @@ class MetadataSourcePlugin(object): return [self.track_for_id(track_data=track) for track in tracks] def album_distance(self, items, album_info, mapping): - return get_distance( + return beets.autotag.get_distance( data_source=self.data_source, info=album_info, config=self.config ) def track_distance(self, item, track_info): - return get_distance( + return beets.autotag.get_distance( data_source=self.data_source, info=track_info, config=self.config ) From dfdf8ded336b79acb979648685014ea957007341 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Tue, 10 Sep 2019 22:55:41 -0700 Subject: [PATCH 210/613] Add missing import --- beetsplug/deezer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index f84a6d30e..a4dfb2bed 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -24,6 +24,7 @@ import unidecode import requests from beets import ui +from beets.autotag import AlbumInfo, TrackInfo from beets.plugins import MetadataSourcePlugin, BeetsPlugin From 876c0f733feb0fa0483055a411157d45dde07142 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Tue, 10 Sep 2019 23:52:35 -0700 Subject: [PATCH 211/613] Appease flake8 --- beets/autotag/__init__.py | 4 ++-- beets/plugins.py | 2 +- beetsplug/deezer.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 2c9c03ffb..ccb23bf9e 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -22,13 +22,13 @@ from beets import logging from beets import config # Parts of external interface. -from .hooks import ( +from .hooks import ( # noqa AlbumInfo, TrackInfo, AlbumMatch, TrackMatch, Distance, -) # noqa +) from .match import tag_item, tag_album, Proposal # noqa from .match import Recommendation # noqa diff --git a/beets/plugins.py b/beets/plugins.py index b429adfe7..fc71e8401 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -686,7 +686,7 @@ class MetadataSourcePlugin(object): if not va_likely: query_filters['artist'] = artist albums = self._search_api(query_type='album', filters=query_filters) - return [self.album_for_id(album_id=album['id']) for album in albums] + return [self.album_for_id(album_id=a['id']) for a in albums] def item_candidates(self, item, artist, title): """Returns a list of TrackInfo objects for Search API results diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index a4dfb2bed..a9a8e1b5b 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -15,7 +15,7 @@ """Adds Deezer release and track search support to the autotagger """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, division import collections From ed80e915fe44a9fdf57256bc7424396bd277f3cc Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 11 Sep 2019 00:07:43 -0700 Subject: [PATCH 212/613] Move `get_distance` --> `beets.plugins` --- beets/autotag/__init__.py | 10 ---------- beets/plugins.py | 15 ++++++++++++--- beetsplug/discogs.py | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index ccb23bf9e..b8bdea479 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -203,13 +203,3 @@ def apply_metadata(album_info, mapping): if value is None and not clobber: continue item[field] = value - - -def get_distance(config, data_source, info): - """Returns the ``data_source`` weight and the maximum source weight - for albums or individual tracks. - """ - dist = Distance() - if info.data_source == data_source: - dist.add('source', config['source_weight'].as_number()) - return dist diff --git a/beets/plugins.py b/beets/plugins.py index fc71e8401..b0752203f 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -27,7 +27,6 @@ from functools import wraps import beets from beets import logging -import beets.autotag import mediafile import six @@ -581,6 +580,16 @@ def notify_info_yielded(event): return decorator +def get_distance(config, data_source, info): + """Returns the ``data_source`` weight and the maximum source weight + for albums or individual tracks. + """ + dist = beets.autotag.Distance() + if info.data_source == data_source: + dist.add('source', config['source_weight'].as_number()) + return dist + + @six.add_metaclass(abc.ABCMeta) class MetadataSourcePlugin(object): def __init__(self): @@ -707,11 +716,11 @@ class MetadataSourcePlugin(object): return [self.track_for_id(track_data=track) for track in tracks] def album_distance(self, items, album_info, mapping): - return beets.autotag.get_distance( + return get_distance( data_source=self.data_source, info=album_info, config=self.config ) def track_distance(self, item, track_info): - return beets.autotag.get_distance( + return get_distance( data_source=self.data_source, info=track_info, config=self.config ) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index e7ac73d64..c6aab991d 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,8 +20,8 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag import AlbumInfo, TrackInfo, get_distance -from beets.plugins import MetadataSourcePlugin, BeetsPlugin +from beets.autotag import AlbumInfo, TrackInfo +from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance import confuse from discogs_client import Release, Master, Client from discogs_client.exceptions import DiscogsAPIError From 6cfe7adb6cb14c70e51fa01da51f3d846a470d34 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 11 Sep 2019 00:26:48 -0700 Subject: [PATCH 213/613] Use qualified import --- beetsplug/discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index c6aab991d..bccf1f7e2 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,7 +20,7 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag import AlbumInfo, TrackInfo +from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance import confuse from discogs_client import Release, Master, Client From 0b2837dd4f2433fffbb14428756831e622447345 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Wed, 11 Sep 2019 00:37:23 -0700 Subject: [PATCH 214/613] Revert Spotify, Discogs changes --- beetsplug/discogs.py | 50 ++++---- beetsplug/spotify.py | 276 ++++++++++++++++++++++++++----------------- 2 files changed, 196 insertions(+), 130 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index bccf1f7e2..4996c5d7c 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,8 +20,8 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance +from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance +from beets.plugins import BeetsPlugin import confuse from discogs_client import Release, Master, Client from discogs_client.exceptions import DiscogsAPIError @@ -159,20 +159,10 @@ class DiscogsPlugin(BeetsPlugin): def album_distance(self, items, album_info, mapping): """Returns the album distance. """ - return get_distance( - data_source='Discogs', - info=album_info, - config=self.config - ) - - def track_distance(self, item, track_info): - """Returns the track distance. - """ - return get_distance( - data_source='Discogs', - info=track_info, - config=self.config - ) + dist = Distance() + if album_info.data_source == 'Discogs': + dist.add('source', self.config['source_weight'].as_number()) + return dist def candidates(self, items, artist, album, va_likely): """Returns a list of AlbumInfo objects for discogs search results @@ -302,9 +292,7 @@ class DiscogsPlugin(BeetsPlugin): self._log.warning(u"Release does not contain the required fields") return None - artist, artist_id = MetadataSourcePlugin.get_artist( - [a.data for a in result.artists] - ) + artist, artist_id = self.get_artist([a.data for a in result.artists]) album = re.sub(r' +', ' ', result.title) album_id = result.data['id'] # Use `.data` to access the tracklist directly instead of the @@ -380,6 +368,26 @@ class DiscogsPlugin(BeetsPlugin): else: return None + def get_artist(self, artists): + """Returns an artist string (all artists) and an artist_id (the main + artist) for a list of discogs album or track artists. + """ + artist_id = None + bits = [] + for i, artist in enumerate(artists): + if not artist_id: + artist_id = artist['id'] + name = artist['name'] + # Strip disambiguation number. + name = re.sub(r' \(\d+\)$', '', name) + # Move articles to the front. + name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name) + bits.append(name) + if artist['join'] and i < len(artists) - 1: + bits.append(artist['join']) + artist = ' '.join(bits).replace(' ,', ',') or None + return artist, artist_id + def get_tracks(self, tracklist): """Returns a list of TrackInfo objects for a discogs tracklist. """ @@ -543,9 +551,7 @@ class DiscogsPlugin(BeetsPlugin): title = track['title'] track_id = None medium, medium_index, _ = self.get_track_index(track['position']) - artist, artist_id = MetadataSourcePlugin.get_artist( - track.get('artists', []) - ) + artist, artist_id = self.get_artist(track.get('artists', [])) length = self.get_track_length(track['duration']) return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, length=length, index=index, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 8fe0d394c..0af0dc9aa 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -1,21 +1,5 @@ # -*- coding: utf-8 -*- -# This file is part of beets. -# Copyright 2019, Rahul Ahuja. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -"""Adds Spotify release and track search support to the autotagger, along with -Spotify playlist construction. -""" from __future__ import division, absolute_import, print_function import re @@ -27,16 +11,14 @@ import collections import six import unidecode import requests -import confuse from beets import ui -from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import MetadataSourcePlugin, BeetsPlugin +from beets.plugins import BeetsPlugin +import confuse +from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance -class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): - data_source = 'Spotify' - +class SpotifyPlugin(BeetsPlugin): # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' @@ -44,13 +26,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): search_url = 'https://api.spotify.com/v1/search' album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' - - # Spotify IDs consist of 22 alphanumeric characters - # (zero-left-padded base62 representation of randomly generated UUID4) - id_regex = { - 'pattern': r'(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})', - 'match_group': 2, - } + playlist_partial = 'spotify:trackset:Playlist:' def __init__(self): super(SpotifyPlugin, self).__init__() @@ -67,6 +43,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): 'client_id': '4e414367a1d14c75a5c5129a627fcab8', 'client_secret': 'f82bdc09b2254f1a8286815d02fd46dc', 'tokenfile': 'spotify_token.json', + 'source_weight': 0.5, } ) self.config['client_secret'].redact = True @@ -116,9 +93,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self.access_token = response.json()['access_token'] # Save the token for later use. - self._log.debug( - u'{} access token: {}', self.data_source, self.access_token - ) + self._log.debug(u'Spotify access token: {}', self.access_token) with open(self.tokenfile, 'w') as f: json.dump({'access_token': self.access_token}, f) @@ -144,19 +119,31 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): if response.status_code != 200: if u'token expired' in response.text: self._log.debug( - '{} access token has expired. Reauthenticating.', - self.data_source, + 'Spotify access token has expired. Reauthenticating.' ) self._authenticate() return self._handle_response(request_type, url, params=params) else: - raise ui.UserError( - u'{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( - self.data_source, response.text, url, params - ) - ) + raise ui.UserError(u'Spotify API error:\n{}', response.text) return response.json() + def _get_spotify_id(self, url_type, id_): + """Parse a Spotify ID from its URL if necessary. + + :param url_type: Type of Spotify URL, either 'album' or 'track'. + :type url_type: str + :param id_: Spotify ID or URL. + :type id_: str + :return: Spotify ID. + :rtype: str + """ + # Spotify IDs consist of 22 alphanumeric characters + # (zero-left-padded base62 representation of randomly generated UUID4) + id_regex = r'(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})' + self._log.debug(u'Searching for {} {}', url_type, id_) + match = re.search(id_regex.format(url_type), id_) + return match.group(2) if match else None + def album_for_id(self, album_id): """Fetch an album by its Spotify ID or URL and return an AlbumInfo object or None if the album is not found. @@ -166,20 +153,20 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): :return: AlbumInfo object for album :rtype: beets.autotag.hooks.AlbumInfo or None """ - spotify_id = self._get_id('album', album_id) + spotify_id = self._get_spotify_id('album', album_id) if spotify_id is None: return None - album_data = self._handle_response( + response_data = self._handle_response( requests.get, self.album_url + spotify_id ) - artist, artist_id = self.get_artist(album_data['artists']) + artist, artist_id = self._get_artist(response_data['artists']) date_parts = [ - int(part) for part in album_data['release_date'].split('-') + int(part) for part in response_data['release_date'].split('-') ] - release_date_precision = album_data['release_date_precision'] + release_date_precision = response_data['release_date_precision'] if release_date_precision == 'day': year, month, day = date_parts elif release_date_precision == 'month': @@ -192,37 +179,35 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): else: raise ui.UserError( u"Invalid `release_date_precision` returned " - u"by {} API: '{}'".format( - self.data_source, release_date_precision - ) + u"by Spotify API: '{}'".format(release_date_precision) ) tracks = [] medium_totals = collections.defaultdict(int) - for i, track_data in enumerate(album_data['tracks']['items'], start=1): + for i, track_data in enumerate(response_data['tracks']['items']): track = self._get_track(track_data) - track.index = i + track.index = i + 1 medium_totals[track.medium] += 1 tracks.append(track) for track in tracks: track.medium_total = medium_totals[track.medium] return AlbumInfo( - album=album_data['name'], + album=response_data['name'], album_id=spotify_id, artist=artist, artist_id=artist_id, tracks=tracks, - albumtype=album_data['album_type'], - va=len(album_data['artists']) == 1 + albumtype=response_data['album_type'], + va=len(response_data['artists']) == 1 and artist.lower() == 'various artists', year=year, month=month, day=day, - label=album_data['label'], + label=response_data['label'], mediums=max(medium_totals.keys()), - data_source=self.data_source, - data_url=album_data['external_urls']['spotify'], + data_source='Spotify', + data_url=response_data['external_urls']['spotify'], ) def _get_track(self, track_data): @@ -234,7 +219,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): :return: TrackInfo object for track :rtype: beets.autotag.hooks.TrackInfo """ - artist, artist_id = self.get_artist(track_data['artists']) + artist, artist_id = self._get_artist(track_data['artists']) return TrackInfo( title=track_data['name'], track_id=track_data['id'], @@ -244,7 +229,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): index=track_data['track_number'], medium=track_data['disc_number'], medium_index=track_data['track_number'], - data_source=self.data_source, + data_source='Spotify', data_url=track_data['external_urls']['spotify'], ) @@ -262,7 +247,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): :rtype: beets.autotag.hooks.TrackInfo or None """ if track_data is None: - spotify_id = self._get_id('track', track_id) + spotify_id = self._get_spotify_id('track', track_id) if spotify_id is None: return None track_data = self._handle_response( @@ -277,14 +262,107 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): requests.get, self.album_url + track_data['album']['id'] ) medium_total = 0 - for i, track_data in enumerate(album_data['tracks']['items'], start=1): + for i, track_data in enumerate(album_data['tracks']['items']): if track_data['disc_number'] == track.medium: medium_total += 1 if track_data['id'] == track.track_id: - track.index = i + track.index = i + 1 track.medium_total = medium_total return track + @staticmethod + def _get_artist(artists): + """Returns an artist string (all artists) and an artist_id (the main + artist) for a list of Spotify artist object dicts. + + :param artists: Iterable of simplified Spotify artist objects + (https://developer.spotify.com/documentation/web-api/reference/object-model/#artist-object-simplified) + :type artists: list[dict] + :return: Normalized artist string + :rtype: str + """ + artist_id = None + artist_names = [] + for artist in artists: + if not artist_id: + artist_id = artist['id'] + name = artist['name'] + # Move articles to the front. + name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) + artist_names.append(name) + artist = ', '.join(artist_names).replace(' ,', ',') or None + return artist, artist_id + + def album_distance(self, items, album_info, mapping): + """Returns the Spotify source weight and the maximum source weight + for albums. + """ + dist = Distance() + if album_info.data_source == 'Spotify': + dist.add('source', self.config['source_weight'].as_number()) + return dist + + def track_distance(self, item, track_info): + """Returns the Spotify source weight and the maximum source weight + for individual tracks. + """ + dist = Distance() + if track_info.data_source == 'Spotify': + dist.add('source', self.config['source_weight'].as_number()) + return dist + + def candidates(self, items, artist, album, va_likely): + """Returns a list of AlbumInfo objects for Spotify Search API results + matching an ``album`` and ``artist`` (if not various). + + :param items: List of items comprised by an album to be matched. + :type items: list[beets.library.Item] + :param artist: The artist of the album to be matched. + :type artist: str + :param album: The name of the album to be matched. + :type album: str + :param va_likely: True if the album to be matched likely has + Various Artists. + :type va_likely: bool + :return: Candidate AlbumInfo objects. + :rtype: list[beets.autotag.hooks.AlbumInfo] + """ + query_filters = {'album': album} + if not va_likely: + query_filters['artist'] = artist + response_data = self._search_spotify( + query_type='album', filters=query_filters + ) + if response_data is None: + return [] + return [ + self.album_for_id(album_id=album_data['id']) + for album_data in response_data['albums']['items'] + ] + + def item_candidates(self, item, artist, title): + """Returns a list of TrackInfo objects for Spotify Search API results + matching ``title`` and ``artist``. + + :param item: Singleton item to be matched. + :type item: beets.library.Item + :param artist: The artist of the track to be matched. + :type artist: str + :param title: The title of the track to be matched. + :type title: str + :return: Candidate TrackInfo objects. + :rtype: list[beets.autotag.hooks.TrackInfo] + """ + response_data = self._search_spotify( + query_type='track', keywords=title, filters={'artist': artist} + ) + if response_data is None: + return [] + return [ + self.track_for_id(track_data=track_data) + for track_data in response_data['tracks']['items'] + ] + @staticmethod def _construct_search_query(filters=None, keywords=''): """Construct a query string with the specified filters and keywords to @@ -307,12 +385,14 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): query = query.decode('utf8') return unidecode.unidecode(query) - def _search_api(self, query_type, filters=None, keywords=''): + def _search_spotify(self, query_type, filters=None, keywords=''): """Query the Spotify Search API for the specified ``keywords``, applying the provided ``filters``. - :param query_type: Item type to search across. Valid types are: - 'album', 'artist', 'playlist', and 'track'. + :param query_type: A comma-separated list of item types to search + across. Valid types are: 'album', 'artist', 'playlist', and + 'track'. Search results include hits from all the specified item + types. :type query_type: str :param filters: (Optional) Field filters to apply. :type filters: dict @@ -327,25 +407,19 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): ) if not query: return None + self._log.debug(u"Searching Spotify for '{}'".format(query)) + response_data = self._handle_response( + requests.get, + self.search_url, + params={'q': query, 'type': query_type}, + ) + num_results = 0 + for result_type_data in response_data.values(): + num_results += len(result_type_data['items']) self._log.debug( - u"Searching {} for '{}'".format(self.data_source, query) + u"Found {} results from Spotify for '{}'", num_results, query ) - response_data = ( - self._handle_response( - requests.get, - self.search_url, - params={'q': query, 'type': query_type}, - ) - .get(query_type + 's', {}) - .get('items', []) - ) - self._log.debug( - u"Found {} result(s) from {} for '{}'", - len(response_data), - self.data_source, - query, - ) - return response_data + return response_data if num_results > 0 else None def commands(self): def queries(lib, opts, args): @@ -355,23 +429,21 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self._output_match_results(results) spotify_cmd = ui.Subcommand( - 'spotify', help=u'build a {} playlist'.format(self.data_source) + 'spotify', help=u'build a Spotify playlist' ) spotify_cmd.parser.add_option( u'-m', u'--mode', action='store', - help=u'"open" to open {} with playlist, ' - u'"list" to print (default)'.format(self.data_source), + help=u'"open" to open Spotify with playlist, ' + u'"list" to print (default)', ) spotify_cmd.parser.add_option( u'-f', u'--show-failures', action='store_true', dest='show_failures', - help=u'list tracks that did not match a {} ID'.format( - self.data_source - ), + help=u'list tracks that did not match a Spotify ID', ) spotify_cmd.func = queries return [spotify_cmd] @@ -411,8 +483,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): if not items: self._log.debug( - u'Your beets query returned no items, skipping {}.', - self.data_source, + u'Your beets query returned no items, skipping Spotify.' ) return @@ -440,15 +511,16 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): # Query the Web API for each track, look for the items' JSON data query_filters = {'artist': artist, 'album': album} - response_data_tracks = self._search_api( + response_data = self._search_spotify( query_type='track', keywords=keywords, filters=query_filters ) - if not response_data_tracks: + if response_data is None: query = self._construct_search_query( keywords=keywords, filters=query_filters ) failures.append(query) continue + response_data_tracks = response_data['tracks']['items'] # Apply market filter if requested region_filter = self.config['region_filter'].get() @@ -464,8 +536,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): or self.config['tiebreak'].get() == 'first' ): self._log.debug( - u'{} track(s) found, count: {}', - self.data_source, + u'Spotify track(s) found, count: {}', len(response_data_tracks), ) chosen_result = response_data_tracks[0] @@ -484,19 +555,16 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): if failure_count > 0: if self.config['show_failures'].get(): self._log.info( - u'{} track(s) did not match a {} ID:', - failure_count, - self.data_source, + u'{} track(s) did not match a Spotify ID:', failure_count ) for track in failures: self._log.info(u'track: {}', track) self._log.info(u'') else: self._log.warning( - u'{} track(s) did not match a {} ID:\n' + u'{} track(s) did not match a Spotify ID;\n' u'use --show-failures to display', failure_count, - self.data_source, ) return results @@ -512,19 +580,11 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): if results: spotify_ids = [track_data['id'] for track_data in results] if self.config['mode'].get() == 'open': - self._log.info( - u'Attempting to open {} with playlist'.format( - self.data_source - ) - ) - spotify_url = 'spotify:trackset:Playlist:' + ','.join( - spotify_ids - ) + self._log.info(u'Attempting to open Spotify with playlist') + spotify_url = self.playlist_partial + ",".join(spotify_ids) webbrowser.open(spotify_url) else: for spotify_id in spotify_ids: print(self.open_track_url + spotify_id) else: - self._log.warning( - u'No {} tracks found from beets query'.format(self.data_source) - ) + self._log.warning(u'No Spotify tracks found from beets query') From dd85a88ba5e7f9f3644969539e1a5a11d4baa8ad Mon Sep 17 00:00:00 2001 From: Ian Pickering Date: Wed, 11 Sep 2019 20:14:30 -0700 Subject: [PATCH 215/613] Correct documentation of `incremental_skip_later` flag The behavior is actually the opposite of what is documented. --- docs/reference/config.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 687f6c3f9..7dcd53801 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -508,10 +508,10 @@ incremental_skip_later ~~~~~~~~~~~~~~~~~~~~~~ Either ``yes`` or ``no``, controlling whether skipped directories are -recorded in the incremental list. When set to ``yes``, skipped directories -will be recorded, and skipped later. When set to ``no``, skipped +recorded in the incremental list. When set to ``yes``, skipped directories won't be recorded, and beets will try to import them again -later. Defaults to ``no``. +later. When set to ``no``, skipped directories will be recorded, and +skipped later. Defaults to ``no``. .. _from_scratch: From bebb725352de0e25b1486b39d0d4c4defda77c38 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 14 Sep 2019 09:35:21 -0400 Subject: [PATCH 216/613] Docs tweaks for #3355 --- docs/changelog.rst | 2 ++ docs/plugins/deezer.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ac2f43c32..cf14ae974 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -69,6 +69,7 @@ New features: * :doc:`/plugins/deezer`: Added Deezer plugin as an import metadata provider: you can now match tracks and albums using the `Deezer`_ database. Thanks to :user:`rhlahuja`. + :bug:`3355` Fixes: @@ -134,6 +135,7 @@ For plugin developers: simplify development of plugins which query album, track, and search APIs to provide metadata matches for the importer. Refer to the Spotify and Deezer plugins for examples of using this template class. + :bug:`3355` For packagers: diff --git a/docs/plugins/deezer.rst b/docs/plugins/deezer.rst index cb964c612..f283df1b2 100644 --- a/docs/plugins/deezer.rst +++ b/docs/plugins/deezer.rst @@ -10,6 +10,7 @@ The ``deezer`` plugin provides metadata matches for the importer using the Basic Usage ----------- + First, enable the ``deezer`` plugin (see :ref:`using-plugins`). You can enter the URL for an album or song on Deezer at the ``enter Id`` @@ -20,6 +21,7 @@ prompt during import:: Configuration ------------- + Put these options in config.yaml under the ``deezer:`` section: - **source_weight**: Penalty applied to Deezer matches during import. Set to From a5fadf0dcc60eb8e66af43563a32b3e073b40a88 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 15 Sep 2019 15:59:24 -0700 Subject: [PATCH 217/613] Integrate MetadataSourcePlugin --- beetsplug/deezer.py | 2 +- beetsplug/discogs.py | 50 ++++---- beetsplug/spotify.py | 276 +++++++++++++++++-------------------------- 3 files changed, 131 insertions(+), 197 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index a9a8e1b5b..ca9bd446d 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -38,7 +38,7 @@ class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): track_url = 'https://api.deezer.com/track/' id_regex = { - 'pattern': r'(^|deezer\.com/)?([a-z]*/)?({}/)?([0-9]*)', + 'pattern': r'(^|deezer\.com/)([a-z]*/)?({}/)([0-9]*)', 'match_group': 4, } diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 4996c5d7c..bccf1f7e2 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -20,8 +20,8 @@ from __future__ import division, absolute_import, print_function import beets.ui from beets import config -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance -from beets.plugins import BeetsPlugin +from beets.autotag.hooks import AlbumInfo, TrackInfo +from beets.plugins import MetadataSourcePlugin, BeetsPlugin, get_distance import confuse from discogs_client import Release, Master, Client from discogs_client.exceptions import DiscogsAPIError @@ -159,10 +159,20 @@ class DiscogsPlugin(BeetsPlugin): def album_distance(self, items, album_info, mapping): """Returns the album distance. """ - dist = Distance() - if album_info.data_source == 'Discogs': - dist.add('source', self.config['source_weight'].as_number()) - return dist + return get_distance( + data_source='Discogs', + info=album_info, + config=self.config + ) + + def track_distance(self, item, track_info): + """Returns the track distance. + """ + return get_distance( + data_source='Discogs', + info=track_info, + config=self.config + ) def candidates(self, items, artist, album, va_likely): """Returns a list of AlbumInfo objects for discogs search results @@ -292,7 +302,9 @@ class DiscogsPlugin(BeetsPlugin): self._log.warning(u"Release does not contain the required fields") return None - artist, artist_id = self.get_artist([a.data for a in result.artists]) + artist, artist_id = MetadataSourcePlugin.get_artist( + [a.data for a in result.artists] + ) album = re.sub(r' +', ' ', result.title) album_id = result.data['id'] # Use `.data` to access the tracklist directly instead of the @@ -368,26 +380,6 @@ class DiscogsPlugin(BeetsPlugin): else: return None - def get_artist(self, artists): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of discogs album or track artists. - """ - artist_id = None - bits = [] - for i, artist in enumerate(artists): - if not artist_id: - artist_id = artist['id'] - name = artist['name'] - # Strip disambiguation number. - name = re.sub(r' \(\d+\)$', '', name) - # Move articles to the front. - name = re.sub(r'(?i)^(.*?), (a|an|the)$', r'\2 \1', name) - bits.append(name) - if artist['join'] and i < len(artists) - 1: - bits.append(artist['join']) - artist = ' '.join(bits).replace(' ,', ',') or None - return artist, artist_id - def get_tracks(self, tracklist): """Returns a list of TrackInfo objects for a discogs tracklist. """ @@ -551,7 +543,9 @@ class DiscogsPlugin(BeetsPlugin): title = track['title'] track_id = None medium, medium_index, _ = self.get_track_index(track['position']) - artist, artist_id = self.get_artist(track.get('artists', [])) + artist, artist_id = MetadataSourcePlugin.get_artist( + track.get('artists', []) + ) length = self.get_track_length(track['duration']) return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, length=length, index=index, diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 0af0dc9aa..8fe0d394c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -1,5 +1,21 @@ # -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Rahul Ahuja. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +"""Adds Spotify release and track search support to the autotagger, along with +Spotify playlist construction. +""" from __future__ import division, absolute_import, print_function import re @@ -11,14 +27,16 @@ import collections import six import unidecode import requests +import confuse from beets import ui -from beets.plugins import BeetsPlugin -import confuse -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance +from beets.autotag.hooks import AlbumInfo, TrackInfo +from beets.plugins import MetadataSourcePlugin, BeetsPlugin -class SpotifyPlugin(BeetsPlugin): +class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): + data_source = 'Spotify' + # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' @@ -26,7 +44,13 @@ class SpotifyPlugin(BeetsPlugin): search_url = 'https://api.spotify.com/v1/search' album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' - playlist_partial = 'spotify:trackset:Playlist:' + + # Spotify IDs consist of 22 alphanumeric characters + # (zero-left-padded base62 representation of randomly generated UUID4) + id_regex = { + 'pattern': r'(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})', + 'match_group': 2, + } def __init__(self): super(SpotifyPlugin, self).__init__() @@ -43,7 +67,6 @@ class SpotifyPlugin(BeetsPlugin): 'client_id': '4e414367a1d14c75a5c5129a627fcab8', 'client_secret': 'f82bdc09b2254f1a8286815d02fd46dc', 'tokenfile': 'spotify_token.json', - 'source_weight': 0.5, } ) self.config['client_secret'].redact = True @@ -93,7 +116,9 @@ class SpotifyPlugin(BeetsPlugin): self.access_token = response.json()['access_token'] # Save the token for later use. - self._log.debug(u'Spotify access token: {}', self.access_token) + self._log.debug( + u'{} access token: {}', self.data_source, self.access_token + ) with open(self.tokenfile, 'w') as f: json.dump({'access_token': self.access_token}, f) @@ -119,31 +144,19 @@ class SpotifyPlugin(BeetsPlugin): if response.status_code != 200: if u'token expired' in response.text: self._log.debug( - 'Spotify access token has expired. Reauthenticating.' + '{} access token has expired. Reauthenticating.', + self.data_source, ) self._authenticate() return self._handle_response(request_type, url, params=params) else: - raise ui.UserError(u'Spotify API error:\n{}', response.text) + raise ui.UserError( + u'{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( + self.data_source, response.text, url, params + ) + ) return response.json() - def _get_spotify_id(self, url_type, id_): - """Parse a Spotify ID from its URL if necessary. - - :param url_type: Type of Spotify URL, either 'album' or 'track'. - :type url_type: str - :param id_: Spotify ID or URL. - :type id_: str - :return: Spotify ID. - :rtype: str - """ - # Spotify IDs consist of 22 alphanumeric characters - # (zero-left-padded base62 representation of randomly generated UUID4) - id_regex = r'(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})' - self._log.debug(u'Searching for {} {}', url_type, id_) - match = re.search(id_regex.format(url_type), id_) - return match.group(2) if match else None - def album_for_id(self, album_id): """Fetch an album by its Spotify ID or URL and return an AlbumInfo object or None if the album is not found. @@ -153,20 +166,20 @@ class SpotifyPlugin(BeetsPlugin): :return: AlbumInfo object for album :rtype: beets.autotag.hooks.AlbumInfo or None """ - spotify_id = self._get_spotify_id('album', album_id) + spotify_id = self._get_id('album', album_id) if spotify_id is None: return None - response_data = self._handle_response( + album_data = self._handle_response( requests.get, self.album_url + spotify_id ) - artist, artist_id = self._get_artist(response_data['artists']) + artist, artist_id = self.get_artist(album_data['artists']) date_parts = [ - int(part) for part in response_data['release_date'].split('-') + int(part) for part in album_data['release_date'].split('-') ] - release_date_precision = response_data['release_date_precision'] + release_date_precision = album_data['release_date_precision'] if release_date_precision == 'day': year, month, day = date_parts elif release_date_precision == 'month': @@ -179,35 +192,37 @@ class SpotifyPlugin(BeetsPlugin): else: raise ui.UserError( u"Invalid `release_date_precision` returned " - u"by Spotify API: '{}'".format(release_date_precision) + u"by {} API: '{}'".format( + self.data_source, release_date_precision + ) ) tracks = [] medium_totals = collections.defaultdict(int) - for i, track_data in enumerate(response_data['tracks']['items']): + for i, track_data in enumerate(album_data['tracks']['items'], start=1): track = self._get_track(track_data) - track.index = i + 1 + track.index = i medium_totals[track.medium] += 1 tracks.append(track) for track in tracks: track.medium_total = medium_totals[track.medium] return AlbumInfo( - album=response_data['name'], + album=album_data['name'], album_id=spotify_id, artist=artist, artist_id=artist_id, tracks=tracks, - albumtype=response_data['album_type'], - va=len(response_data['artists']) == 1 + albumtype=album_data['album_type'], + va=len(album_data['artists']) == 1 and artist.lower() == 'various artists', year=year, month=month, day=day, - label=response_data['label'], + label=album_data['label'], mediums=max(medium_totals.keys()), - data_source='Spotify', - data_url=response_data['external_urls']['spotify'], + data_source=self.data_source, + data_url=album_data['external_urls']['spotify'], ) def _get_track(self, track_data): @@ -219,7 +234,7 @@ class SpotifyPlugin(BeetsPlugin): :return: TrackInfo object for track :rtype: beets.autotag.hooks.TrackInfo """ - artist, artist_id = self._get_artist(track_data['artists']) + artist, artist_id = self.get_artist(track_data['artists']) return TrackInfo( title=track_data['name'], track_id=track_data['id'], @@ -229,7 +244,7 @@ class SpotifyPlugin(BeetsPlugin): index=track_data['track_number'], medium=track_data['disc_number'], medium_index=track_data['track_number'], - data_source='Spotify', + data_source=self.data_source, data_url=track_data['external_urls']['spotify'], ) @@ -247,7 +262,7 @@ class SpotifyPlugin(BeetsPlugin): :rtype: beets.autotag.hooks.TrackInfo or None """ if track_data is None: - spotify_id = self._get_spotify_id('track', track_id) + spotify_id = self._get_id('track', track_id) if spotify_id is None: return None track_data = self._handle_response( @@ -262,107 +277,14 @@ class SpotifyPlugin(BeetsPlugin): requests.get, self.album_url + track_data['album']['id'] ) medium_total = 0 - for i, track_data in enumerate(album_data['tracks']['items']): + for i, track_data in enumerate(album_data['tracks']['items'], start=1): if track_data['disc_number'] == track.medium: medium_total += 1 if track_data['id'] == track.track_id: - track.index = i + 1 + track.index = i track.medium_total = medium_total return track - @staticmethod - def _get_artist(artists): - """Returns an artist string (all artists) and an artist_id (the main - artist) for a list of Spotify artist object dicts. - - :param artists: Iterable of simplified Spotify artist objects - (https://developer.spotify.com/documentation/web-api/reference/object-model/#artist-object-simplified) - :type artists: list[dict] - :return: Normalized artist string - :rtype: str - """ - artist_id = None - artist_names = [] - for artist in artists: - if not artist_id: - artist_id = artist['id'] - name = artist['name'] - # Move articles to the front. - name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) - artist_names.append(name) - artist = ', '.join(artist_names).replace(' ,', ',') or None - return artist, artist_id - - def album_distance(self, items, album_info, mapping): - """Returns the Spotify source weight and the maximum source weight - for albums. - """ - dist = Distance() - if album_info.data_source == 'Spotify': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def track_distance(self, item, track_info): - """Returns the Spotify source weight and the maximum source weight - for individual tracks. - """ - dist = Distance() - if track_info.data_source == 'Spotify': - dist.add('source', self.config['source_weight'].as_number()) - return dist - - def candidates(self, items, artist, album, va_likely): - """Returns a list of AlbumInfo objects for Spotify Search API results - matching an ``album`` and ``artist`` (if not various). - - :param items: List of items comprised by an album to be matched. - :type items: list[beets.library.Item] - :param artist: The artist of the album to be matched. - :type artist: str - :param album: The name of the album to be matched. - :type album: str - :param va_likely: True if the album to be matched likely has - Various Artists. - :type va_likely: bool - :return: Candidate AlbumInfo objects. - :rtype: list[beets.autotag.hooks.AlbumInfo] - """ - query_filters = {'album': album} - if not va_likely: - query_filters['artist'] = artist - response_data = self._search_spotify( - query_type='album', filters=query_filters - ) - if response_data is None: - return [] - return [ - self.album_for_id(album_id=album_data['id']) - for album_data in response_data['albums']['items'] - ] - - def item_candidates(self, item, artist, title): - """Returns a list of TrackInfo objects for Spotify Search API results - matching ``title`` and ``artist``. - - :param item: Singleton item to be matched. - :type item: beets.library.Item - :param artist: The artist of the track to be matched. - :type artist: str - :param title: The title of the track to be matched. - :type title: str - :return: Candidate TrackInfo objects. - :rtype: list[beets.autotag.hooks.TrackInfo] - """ - response_data = self._search_spotify( - query_type='track', keywords=title, filters={'artist': artist} - ) - if response_data is None: - return [] - return [ - self.track_for_id(track_data=track_data) - for track_data in response_data['tracks']['items'] - ] - @staticmethod def _construct_search_query(filters=None, keywords=''): """Construct a query string with the specified filters and keywords to @@ -385,14 +307,12 @@ class SpotifyPlugin(BeetsPlugin): query = query.decode('utf8') return unidecode.unidecode(query) - def _search_spotify(self, query_type, filters=None, keywords=''): + def _search_api(self, query_type, filters=None, keywords=''): """Query the Spotify Search API for the specified ``keywords``, applying the provided ``filters``. - :param query_type: A comma-separated list of item types to search - across. Valid types are: 'album', 'artist', 'playlist', and - 'track'. Search results include hits from all the specified item - types. + :param query_type: Item type to search across. Valid types are: + 'album', 'artist', 'playlist', and 'track'. :type query_type: str :param filters: (Optional) Field filters to apply. :type filters: dict @@ -407,19 +327,25 @@ class SpotifyPlugin(BeetsPlugin): ) if not query: return None - self._log.debug(u"Searching Spotify for '{}'".format(query)) - response_data = self._handle_response( - requests.get, - self.search_url, - params={'q': query, 'type': query_type}, - ) - num_results = 0 - for result_type_data in response_data.values(): - num_results += len(result_type_data['items']) self._log.debug( - u"Found {} results from Spotify for '{}'", num_results, query + u"Searching {} for '{}'".format(self.data_source, query) ) - return response_data if num_results > 0 else None + response_data = ( + self._handle_response( + requests.get, + self.search_url, + params={'q': query, 'type': query_type}, + ) + .get(query_type + 's', {}) + .get('items', []) + ) + self._log.debug( + u"Found {} result(s) from {} for '{}'", + len(response_data), + self.data_source, + query, + ) + return response_data def commands(self): def queries(lib, opts, args): @@ -429,21 +355,23 @@ class SpotifyPlugin(BeetsPlugin): self._output_match_results(results) spotify_cmd = ui.Subcommand( - 'spotify', help=u'build a Spotify playlist' + 'spotify', help=u'build a {} playlist'.format(self.data_source) ) spotify_cmd.parser.add_option( u'-m', u'--mode', action='store', - help=u'"open" to open Spotify with playlist, ' - u'"list" to print (default)', + help=u'"open" to open {} with playlist, ' + u'"list" to print (default)'.format(self.data_source), ) spotify_cmd.parser.add_option( u'-f', u'--show-failures', action='store_true', dest='show_failures', - help=u'list tracks that did not match a Spotify ID', + help=u'list tracks that did not match a {} ID'.format( + self.data_source + ), ) spotify_cmd.func = queries return [spotify_cmd] @@ -483,7 +411,8 @@ class SpotifyPlugin(BeetsPlugin): if not items: self._log.debug( - u'Your beets query returned no items, skipping Spotify.' + u'Your beets query returned no items, skipping {}.', + self.data_source, ) return @@ -511,16 +440,15 @@ class SpotifyPlugin(BeetsPlugin): # Query the Web API for each track, look for the items' JSON data query_filters = {'artist': artist, 'album': album} - response_data = self._search_spotify( + response_data_tracks = self._search_api( query_type='track', keywords=keywords, filters=query_filters ) - if response_data is None: + if not response_data_tracks: query = self._construct_search_query( keywords=keywords, filters=query_filters ) failures.append(query) continue - response_data_tracks = response_data['tracks']['items'] # Apply market filter if requested region_filter = self.config['region_filter'].get() @@ -536,7 +464,8 @@ class SpotifyPlugin(BeetsPlugin): or self.config['tiebreak'].get() == 'first' ): self._log.debug( - u'Spotify track(s) found, count: {}', + u'{} track(s) found, count: {}', + self.data_source, len(response_data_tracks), ) chosen_result = response_data_tracks[0] @@ -555,16 +484,19 @@ class SpotifyPlugin(BeetsPlugin): if failure_count > 0: if self.config['show_failures'].get(): self._log.info( - u'{} track(s) did not match a Spotify ID:', failure_count + u'{} track(s) did not match a {} ID:', + failure_count, + self.data_source, ) for track in failures: self._log.info(u'track: {}', track) self._log.info(u'') else: self._log.warning( - u'{} track(s) did not match a Spotify ID;\n' + u'{} track(s) did not match a {} ID:\n' u'use --show-failures to display', failure_count, + self.data_source, ) return results @@ -580,11 +512,19 @@ class SpotifyPlugin(BeetsPlugin): if results: spotify_ids = [track_data['id'] for track_data in results] if self.config['mode'].get() == 'open': - self._log.info(u'Attempting to open Spotify with playlist') - spotify_url = self.playlist_partial + ",".join(spotify_ids) + self._log.info( + u'Attempting to open {} with playlist'.format( + self.data_source + ) + ) + spotify_url = 'spotify:trackset:Playlist:' + ','.join( + spotify_ids + ) webbrowser.open(spotify_url) else: for spotify_id in spotify_ids: print(self.open_track_url + spotify_id) else: - self._log.warning(u'No Spotify tracks found from beets query') + self._log.warning( + u'No {} tracks found from beets query'.format(self.data_source) + ) From 01e8643cece12010c726ecc52168ecac7a415e46 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 15 Sep 2019 16:24:32 -0700 Subject: [PATCH 218/613] Revert to optional capturing group --- beetsplug/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index ca9bd446d..6be4e1679 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -38,7 +38,7 @@ class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): track_url = 'https://api.deezer.com/track/' id_regex = { - 'pattern': r'(^|deezer\.com/)([a-z]*/)?({}/)([0-9]*)', + 'pattern': r'(^|deezer\.com/)(\w+/)?({}/)?(\d+)', 'match_group': 4, } From ca5806fb6e10c9fada20a764ea7879e4d31547fa Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 15 Sep 2019 16:34:20 -0700 Subject: [PATCH 219/613] Restrict country code to alpha characters --- beetsplug/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 6be4e1679..a4337d273 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -38,7 +38,7 @@ class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): track_url = 'https://api.deezer.com/track/' id_regex = { - 'pattern': r'(^|deezer\.com/)(\w+/)?({}/)?(\d+)', + 'pattern': r'(^|deezer\.com/)([a-z]*/)?({}/)?(\d+)', 'match_group': 4, } From b4edc1f832f6add99a912e271eb3e795d8fbad56 Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 00:29:43 +0200 Subject: [PATCH 220/613] Add bpm, musical_key and genre to plugin. --- beetsplug/beatport.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 3462f118a..8ef356fe8 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -255,6 +255,16 @@ class BeatportTrack(BeatportObject): self.url = "https://beatport.com/track/{0}/{1}" \ .format(data['slug'], data['id']) self.track_number = data.get('trackNumber') + if 'bpm' in data: + self.bpm = data['bpm'] + if 'key' in data: + self.musical_key = six.text_type(data['key'].get('shortName')) + + # Use 'subgenre' and if not present, 'genre' as a fallback. + if 'subGenres' in data: + self.genre = six.text_type(data['subGenres'][0].get('name')) + if not self.genre and 'genres' in data: + self.genre = six.text_type(data['genres'][0].get('name')) class BeatportPlugin(BeetsPlugin): @@ -433,7 +443,8 @@ class BeatportPlugin(BeetsPlugin): artist=artist, artist_id=artist_id, length=length, index=track.track_number, medium_index=track.track_number, - data_source=u'Beatport', data_url=track.url) + data_source=u'Beatport', data_url=track.url, + bpm=track.bpm, musical_key=track.musical_key) def _get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main From 067358711e554ca95dffac5770541ff21ba134af Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 00:31:43 +0200 Subject: [PATCH 221/613] Add attributes to hooks. --- beets/autotag/__init__.py | 3 +++ beets/autotag/hooks.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index b8bdea479..38db0a07b 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -185,6 +185,9 @@ def apply_metadata(album_info, mapping): 'work', 'mb_workid', 'work_disambig', + 'bpm', + 'musical_key', + 'genre' ) } diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 686423360..f59aaea42 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -179,7 +179,8 @@ class TrackInfo(object): disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, track_alt=None, - work=None, mb_workid=None, work_disambig=None): + work=None, mb_workid=None, work_disambig=None, bpm=None, + musical_key=None, genre=None): self.title = title self.track_id = track_id self.release_track_id = release_track_id @@ -204,6 +205,9 @@ class TrackInfo(object): self.work = work self.mb_workid = mb_workid self.work_disambig = work_disambig + self.bpm = bpm + self.musical_key = musical_key + self.genre = genre # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): From 6c8535088a2561e6d637c33aa07e5dac4e6ea2d4 Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 00:32:52 +0200 Subject: [PATCH 222/613] Add test file. --- test/test_beatport.py | 567 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 test/test_beatport.py diff --git a/test/test_beatport.py b/test/test_beatport.py new file mode 100644 index 000000000..a1591d58c --- /dev/null +++ b/test/test_beatport.py @@ -0,0 +1,567 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Adrian Sampson. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Tests for the 'beatport' plugin. +""" +from __future__ import division, absolute_import, print_function + +import unittest +from test import _common +from test.helper import TestHelper +import six +from datetime import timedelta + +from beetsplug import beatport +from beets import library + + +class BeatportTest(_common.TestCase, TestHelper): + def _make_release_response(self): + """Returns a dict that mimics a response from the beatport API. + + The results were retrived from: + https://oauth-api.beatport.com/catalog/3/releases?id=1742984 + The list of elements on the returned dict is incomplete, including just + those required for the tests on this class. + """ + results = { + "id": 1742984, + "type": "release", + "name": "Charade", + "slug": "charade", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "audioFormat": "", + "category": "Release", + "currentStatus": "General Content", + "catalogNumber": "GR089", + "description": "", + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + } + return results + + def _make_tracks_response(self): + """Return a list that mimics a response from the beatport API. + + The results were retrived from: + https://oauth-api.beatport.com/catalog/3/tracks?releaseId=1742984 + The list of elements on the returned list is incomplete, including just + those required for the tests on this class. + """ + results = [{ + "id": 7817567, + "type": "track", + "sku": "track-7817567", + "name": "Mirage a Trois", + "trackNumber": 1, + "mixName": "Original Mix", + "title": "Mirage a Trois (Original Mix)", + "slug": "mirage-a-trois-original-mix", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "currentStatus": "General Content", + "length": "7:05", + "lengthMs": 425421, + "bpm": 90, + "key": { + "standard": { + "letter": "G", + "sharp": False, + "flat": False, + "chord": "minor" + }, + "shortName": "Gmin" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + "release": { + "id": 1742984, + "name": "Charade", + "type": "release", + "slug": "charade" + }, + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings", + "status": True + } + }, { + "id": 7817568, + "type": "track", + "sku": "track-7817568", + "name": "Aeon Bahamut", + "trackNumber": 2, + "mixName": "Original Mix", + "title": "Aeon Bahamut (Original Mix)", + "slug": "aeon-bahamut-original-mix", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "currentStatus": "General Content", + "length": "7:38", + "lengthMs": 458000, + "bpm": 100, + "key": { + "standard": { + "letter": "G", + "sharp": False, + "flat": False, + "chord": "major" + }, + "shortName": "Gmaj" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + "release": { + "id": 1742984, + "name": "Charade", + "type": "release", + "slug": "charade" + }, + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings", + "status": True + } + }, { + "id": 7817569, + "type": "track", + "sku": "track-7817569", + "name": "Trancendental Medication", + "trackNumber": 3, + "mixName": "Original Mix", + "title": "Trancendental Medication (Original Mix)", + "slug": "trancendental-medication-original-mix", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "currentStatus": "General Content", + "length": "1:08", + "lengthMs": 68571, + "bpm": 141, + "key": { + "standard": { + "letter": "F", + "sharp": False, + "flat": False, + "chord": "major" + }, + "shortName": "Fmaj" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + "release": { + "id": 1742984, + "name": "Charade", + "type": "release", + "slug": "charade" + }, + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings", + "status": True + } + }, { + "id": 7817570, + "type": "track", + "sku": "track-7817570", + "name": "A List of Instructions for When I'm Human", + "trackNumber": 4, + "mixName": "Original Mix", + "title": "A List of Instructions for When I'm Human (Original Mix)", + "slug": "a-list-of-instructions-for-when-im-human-original-mix", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "currentStatus": "General Content", + "length": "6:57", + "lengthMs": 417913, + "bpm": 88, + "key": { + "standard": { + "letter": "A", + "sharp": False, + "flat": False, + "chord": "minor" + }, + "shortName": "Amin" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + "release": { + "id": 1742984, + "name": "Charade", + "type": "release", + "slug": "charade" + }, + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings", + "status": True + } + }, { + "id": 7817571, + "type": "track", + "sku": "track-7817571", + "name": "The Great Shenanigan", + "trackNumber": 5, + "mixName": "Original Mix", + "title": "The Great Shenanigan (Original Mix)", + "slug": "the-great-shenanigan-original-mix", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "currentStatus": "General Content", + "length": "9:49", + "lengthMs": 589875, + "bpm": 123, + "key": { + "standard": { + "letter": "E", + "sharp": False, + "flat": True, + "chord": "major" + }, + "shortName": "E♭maj" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + "release": { + "id": 1742984, + "name": "Charade", + "type": "release", + "slug": "charade" + }, + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings", + "status": True + } + }, { + "id": 7817572, + "type": "track", + "sku": "track-7817572", + "name": "Charade", + "trackNumber": 6, + "mixName": "Original Mix", + "title": "Charade (Original Mix)", + "slug": "charade-original-mix", + "releaseDate": "2016-04-11", + "publishDate": "2016-04-11", + "currentStatus": "General Content", + "length": "7:05", + "lengthMs": 425423, + "bpm": 123, + "key": { + "standard": { + "letter": "A", + "sharp": False, + "flat": False, + "chord": "major" + }, + "shortName": "Amaj" + }, + "artists": [{ + "id": 326158, + "name": "Supersillyus", + "slug": "supersillyus", + "type": "artist" + }], + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + "release": { + "id": 1742984, + "name": "Charade", + "type": "release", + "slug": "charade" + }, + "label": { + "id": 24539, + "name": "Gravitas Recordings", + "type": "label", + "slug": "gravitas-recordings", + "status": True + } + }] + return results + + def setUp(self): + self.setup_beets() + self.load_plugins('beatport') + self.lib = library.Library(':memory:') + + # Set up 'album'. + response_release = self._make_release_response() + self.album = beatport.BeatportRelease(response_release) + + # Set up 'tracks'. + response_tracks = self._make_tracks_response() + self.tracks = [beatport.BeatportTrack(t) for t in response_tracks] + + # Set up 'test_album'. + self.test_album = self.mk_test_album() + # print(self.test_album.keys()) + + # Set up 'test_tracks' + self.test_tracks = self.test_album.items() + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() + + def mk_test_album(self): + items = [_common.item() for _ in range(6)] + for item in items: + item.album = 'Charade' + item.catalognum = 'GR089' + item.label = 'Gravitas Recordings' + item.artist = 'Supersillyus' + item.year = 2016 + item.comp = False + item.label_name = 'Gravitas Recordings' + item.genre = 'Glitch Hop' + item.year = 2016 + item.month = 4 + item.day = 11 + item.mix_name = 'Original Mix' + + items[0].title = 'Mirage a Trois' + items[1].title = 'Aeon Bahamut' + items[2].title = 'Trancendental Medication' + items[3].title = 'A List of Instructions for When I\'m Human' + items[4].title = 'The Great Shenanigan' + items[5].title = 'Charade' + + items[0].length = timedelta(minutes=7, seconds=5).total_seconds() + items[1].length = timedelta(minutes=7, seconds=38).total_seconds() + items[2].length = timedelta(minutes=1, seconds=8).total_seconds() + items[3].length = timedelta(minutes=6, seconds=57).total_seconds() + items[4].length = timedelta(minutes=9, seconds=49).total_seconds() + items[5].length = timedelta(minutes=7, seconds=5).total_seconds() + + items[0].url = 'mirage-a-trois-original-mix' + items[1].url = 'aeon-bahamut-original-mix' + items[2].url = 'trancendental-medication-original-mix' + items[3].url = 'a-list-of-instructions-for-when-im-human-original-mix' + items[4].url = 'the-great-shenanigan-original-mix' + items[5].url = 'charade-original-mix' + + counter = 0 + for item in items: + counter += 1 + item.track_number = counter + + items[0].bpm = 90 + items[1].bpm = 100 + items[2].bpm = 141 + items[3].bpm = 88 + items[4].bpm = 123 + items[5].bpm = 123 + + items[0].musical_key = 'Gmin' + items[1].musical_key = 'Gmaj' + items[2].musical_key = 'Fmaj' + items[3].musical_key = 'Amin' + items[4].musical_key = 'E♭maj' + items[5].musical_key = 'Amaj' + + for item in items: + self.lib.add(item) + + album = self.lib.add_album(items) + album.store() + + return album + + # Test BeatportRelease. + def test_album_name_applied(self): + self.assertEqual(self.album.name, self.test_album['album']) + + def test_catalog_number_applied(self): + self.assertEqual(self.album.catalog_number, + self.test_album['catalognum']) + + def test_label_applied(self): + self.assertEqual(self.album.label_name, self.test_album['label']) + + def test_category_applied(self): + self.assertEqual(self.album.category, 'Release') + + def test_album_url_applied(self): + self.assertEqual(self.album.url, + 'https://beatport.com/release/charade/1742984') + + # Test BeatportTrack. + def test_title_applied(self): + for track, test_track in zip(self.tracks, self.test_tracks): + self.assertEqual(track.name, test_track.title) + + def test_mix_name_applied(self): + for track, test_track in zip(self.tracks, self.test_tracks): + self.assertEqual(track.mix_name, test_track.mix_name) + + def test_length_applied(self): + for track, test_track in zip(self.tracks, self.test_tracks): + self.assertEqual(int(track.length.total_seconds()), + int(test_track.length)) + + def test_track_url_applied(self): + # Specify beatport ids here because an 'item.id' is beets-internal. + ids = [ + 7817567, + 7817568, + 7817569, + 7817570, + 7817571, + 7817572, + ] + # Concatenate with 'id' to pass strict equality test. + for track, test_track, id in zip(self.tracks, self.test_tracks, ids): + self.assertEqual( + track.url, 'https://beatport.com/track/' + + test_track.url + '/' + six.text_type(id)) + + def test_bpm_applied(self): + for track, test_track in zip(self.tracks, self.test_tracks): + self.assertEqual(track.bpm, test_track.bpm) + + def test_musical_key_applied(self): + for track, test_track in zip(self.tracks, self.test_tracks): + self.assertEqual(track.musical_key, test_track.musical_key) + + def test_genre_applied(self): + for track, test_track in zip(self.tracks, self.test_tracks): + self.assertEqual(track.genre, test_track.genre) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 4038e36343da41f631b23970a96ed150fef2886c Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 00:33:57 +0200 Subject: [PATCH 223/613] Update URL to use HTTPS and add documentation. --- docs/plugins/beatport.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index 709dbb0a8..0fcd7676b 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -4,7 +4,9 @@ Beatport Plugin The ``beatport`` plugin adds support for querying the `Beatport`_ catalogue during the autotagging process. This can potentially be helpful for users whose collection includes a lot of diverse electronic music releases, for which -both MusicBrainz and (to a lesser degree) Discogs show no matches. +both MusicBrainz and (to a lesser degree) `Discogs`_ show no matches. + +.. _Discogs: https://discogs.com Installation ------------ @@ -21,15 +23,18 @@ run the :ref:`import-cmd` command after enabling the plugin, it will ask you to authorize with Beatport by visiting the site in a browser. On the site you will be asked to enter your username and password to authorize beets to query the Beatport API. You will then be displayed with a single line of -text that you should paste into your terminal. This will store the -authentication data for subsequent runs and you will not be required to -repeat the above steps. +text that you should paste as a whole into your terminal. This will store the +authentication data (under ``~/.config/beatport_token.json``) for subsequent +runs and you will not be required to repeat the above steps. Matches from Beatport should now show up alongside matches from MusicBrainz and other sources. If you have a Beatport ID or a URL for a release or track you want to tag, you -can just enter one of the two at the "enter Id" prompt in the importer. +can just enter one of the two at the "enter Id" prompt in the importer. You can +also search for an id like so: + + beet import path/to/music/library --search-id id .. _requests: https://docs.python-requests.org/en/latest/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib From c2b750d7e487dd5ff01e93c02d515e25afcab271 Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 00:39:16 +0200 Subject: [PATCH 224/613] Add feature and add For plugin developers. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cf14ae974..aa87929c6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -70,6 +70,9 @@ New features: you can now match tracks and albums using the `Deezer`_ database. Thanks to :user:`rhlahuja`. :bug:`3355` +* :doc:`/plugins/beatport`: The plugin now gets the musical key, BPM and the + genre for each track. + :bug:`2080` Fixes: @@ -136,6 +139,8 @@ For plugin developers: APIs to provide metadata matches for the importer. Refer to the Spotify and Deezer plugins for examples of using this template class. :bug:`3355` +* The autotag hooks have been modified such that they now take 'bpm', + 'musical_key' and a per-track based 'genre' as attributes. For packagers: From 045f5723e24e40f1ed78edd20ed2b54fc52f0b83 Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 21:57:09 +0200 Subject: [PATCH 225/613] Remove hard-coded path. --- docs/plugins/beatport.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index 0fcd7676b..980add451 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -24,8 +24,8 @@ to authorize with Beatport by visiting the site in a browser. On the site you will be asked to enter your username and password to authorize beets to query the Beatport API. You will then be displayed with a single line of text that you should paste as a whole into your terminal. This will store the -authentication data (under ``~/.config/beatport_token.json``) for subsequent -runs and you will not be required to repeat the above steps. +authentication data for subsequentruns and you will not be required to repeat +the above steps. Matches from Beatport should now show up alongside matches from MusicBrainz and other sources. From 15f08aba4fd03ee11eaa6d28f66580073b4dc804 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 19 Sep 2019 16:13:42 -0400 Subject: [PATCH 226/613] Restore missing space --- docs/plugins/beatport.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index 980add451..d645e2043 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -24,7 +24,7 @@ to authorize with Beatport by visiting the site in a browser. On the site you will be asked to enter your username and password to authorize beets to query the Beatport API. You will then be displayed with a single line of text that you should paste as a whole into your terminal. This will store the -authentication data for subsequentruns and you will not be required to repeat +authentication data for subsequent runs and you will not be required to repeat the above steps. Matches from Beatport should now show up alongside matches From 88aea76fcb7a064d265ca119be44f28c86773442 Mon Sep 17 00:00:00 2001 From: temrix Date: Thu, 19 Sep 2019 22:35:40 +0200 Subject: [PATCH 227/613] Add requests_oauthlib to test deps. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 8736f0f3c..9dac2d59a 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = python-mpd2 coverage discogs-client + requests_oauthlib [_flake8] deps = From f98010ad23b2462a321f8a33c280a6fdc112b739 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 16:09:08 +0200 Subject: [PATCH 228/613] Add 'look before you leap' defensive code. --- beetsplug/beatport.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 8ef356fe8..7385684f4 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -150,9 +150,11 @@ class BeatportClient(object): :rtype: :py:class:`BeatportRelease` """ response = self._get('/catalog/3/releases', id=beatport_id) - release = BeatportRelease(response[0]) - release.tracks = self.get_release_tracks(beatport_id) - return release + if response: + release = BeatportRelease(response[0]) + release.tracks = self.get_release_tracks(beatport_id) + return release + return None def get_release_tracks(self, beatport_id): """ Get all tracks for a given release. @@ -261,10 +263,10 @@ class BeatportTrack(BeatportObject): self.musical_key = six.text_type(data['key'].get('shortName')) # Use 'subgenre' and if not present, 'genre' as a fallback. - if 'subGenres' in data: + if 'subGenres' in data and data['subGenres']: self.genre = six.text_type(data['subGenres'][0].get('name')) - if not self.genre and 'genres' in data: - self.genre = six.text_type(data['genres'][0].get('name')) + elif 'genres' in data and data['genres']: + self.genre = six.text_type(data['genres'][0].get('name')) class BeatportPlugin(BeetsPlugin): @@ -375,27 +377,33 @@ class BeatportPlugin(BeetsPlugin): def album_for_id(self, release_id): """Fetches a release by its Beatport ID and returns an AlbumInfo object - or None if the release is not found. + or None if the query is not a valid ID or release is not found. """ self._log.debug(u'Searching for release {0}', release_id) match = re.search(r'(^|beatport\.com/release/.+/)(\d+)$', release_id) if not match: + self._log.debug(u'Not a valid Beatport release ID.') return None release = self.client.get_release(match.group(2)) - album = self._get_album_info(release) - return album + if release is not None: + album = self._get_album_info(release) + return album + return None def track_for_id(self, track_id): """Fetches a track by its Beatport ID and returns a TrackInfo object - or None if the track is not found. + or None if the track is not a valid Beatport ID or track is not found. """ self._log.debug(u'Searching for track {0}', track_id) match = re.search(r'(^|beatport\.com/track/.+/)(\d+)$', track_id) if not match: + self._log.debug(u'Not a valid Beatport track ID.') return None bp_track = self.client.get_track(match.group(2)) - track = self._get_track_info(bp_track) - return track + if bp_track is not None: + track = self._get_track_info(bp_track) + return track + return None def _get_releases(self, query): """Returns a list of AlbumInfo objects for a beatport search query. From dde905f9a805b102d452cc84ec52f25208fdddb2 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 16:41:16 +0200 Subject: [PATCH 229/613] Correct typo in docstring. --- test/test_beatport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_beatport.py b/test/test_beatport.py index a1591d58c..823668c82 100644 --- a/test/test_beatport.py +++ b/test/test_beatport.py @@ -31,7 +31,7 @@ class BeatportTest(_common.TestCase, TestHelper): def _make_release_response(self): """Returns a dict that mimics a response from the beatport API. - The results were retrived from: + The results were retrieved from: https://oauth-api.beatport.com/catalog/3/releases?id=1742984 The list of elements on the returned dict is incomplete, including just those required for the tests on this class. @@ -72,7 +72,7 @@ class BeatportTest(_common.TestCase, TestHelper): def _make_tracks_response(self): """Return a list that mimics a response from the beatport API. - The results were retrived from: + The results were retrieved from: https://oauth-api.beatport.com/catalog/3/tracks?releaseId=1742984 The list of elements on the returned list is incomplete, including just those required for the tests on this class. From 2691781d4e53909c7bb993d16b4ede972f218e95 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 16:41:57 +0200 Subject: [PATCH 230/613] Remove forgotten print statement. --- test/test_beatport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_beatport.py b/test/test_beatport.py index 823668c82..4589a9eba 100644 --- a/test/test_beatport.py +++ b/test/test_beatport.py @@ -425,7 +425,6 @@ class BeatportTest(_common.TestCase, TestHelper): # Set up 'test_album'. self.test_album = self.mk_test_album() - # print(self.test_album.keys()) # Set up 'test_tracks' self.test_tracks = self.test_album.items() From 486cfa1df1a149689017cb4d2646894ad0ce3c04 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 16:54:47 +0200 Subject: [PATCH 231/613] Add tests for empty responses. --- test/test_beatport.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/test/test_beatport.py b/test/test_beatport.py index 4589a9eba..8e830df76 100644 --- a/test/test_beatport.py +++ b/test/test_beatport.py @@ -558,6 +558,70 @@ class BeatportTest(_common.TestCase, TestHelper): self.assertEqual(track.genre, test_track.genre) +class BeatportResponseEmptyTest(_common.TestCase, TestHelper): + def _make_tracks_response(self): + results = [{ + "id": 7817567, + "name": "Mirage a Trois", + "genres": [{ + "id": 9, + "name": "Breaks", + "slug": "breaks", + "type": "genre" + }], + "subGenres": [{ + "id": 209, + "name": "Glitch Hop", + "slug": "glitch-hop", + "type": "subgenre" + }], + }] + return results + + def setUp(self): + self.setup_beets() + self.load_plugins('beatport') + self.lib = library.Library(':memory:') + + # Set up 'tracks'. + self.response_tracks = self._make_tracks_response() + self.tracks = [beatport.BeatportTrack(t) for t in self.response_tracks] + + # Make alias to be congruent with class `BeatportTest`. + self.test_tracks = self.response_tracks + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() + + def test_response_tracks_empty(self): + response_tracks = [] + tracks = [beatport.BeatportTrack(t) for t in response_tracks] + self.assertEqual(tracks, []) + + def test_sub_genre_empty_fallback(self): + """No 'sub_genre' is provided. Test if fallback to 'genre' works. + """ + self.response_tracks[0]['subGenres'] = [] + tracks = [beatport.BeatportTrack(t) for t in self.response_tracks] + + self.test_tracks[0]['subGenres'] = [] + + self.assertEqual(tracks[0].genre, + self.test_tracks[0]['genres'][0]['name']) + + def test_genre_empty(self): + """No 'genre' is provided. Test if 'sub_genre' is applied. + """ + self.response_tracks[0]['genres'] = [] + tracks = [beatport.BeatportTrack(t) for t in self.response_tracks] + + self.test_tracks[0]['genres'] = [] + + self.assertEqual(tracks[0].genre, + self.test_tracks[0]['subGenres'][0]['name']) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 2cd38fc7df178d37276d24aa5f9acaa027fcddf0 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 17:12:46 +0200 Subject: [PATCH 232/613] Add bugfix. --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index aa87929c6..261b74505 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -107,6 +107,8 @@ Fixes: * ``none_rec_action`` does not import automatically when ``timid`` is enabled. Thanks to :user:`RollingStar`. :bug:`3242` +* Fix a bug that caused a crash when tagging items with the beatport plugin. + :bug:`3374` For plugin developers: From 9a4175dcd05b4565addf614ab2e69044fffd9960 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 21:13:41 +0200 Subject: [PATCH 233/613] Return value directly. --- beetsplug/beatport.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 7385684f4..1f2fbe451 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -385,9 +385,8 @@ class BeatportPlugin(BeetsPlugin): self._log.debug(u'Not a valid Beatport release ID.') return None release = self.client.get_release(match.group(2)) - if release is not None: - album = self._get_album_info(release) - return album + if release: + return self._get_album_info(release) return None def track_for_id(self, track_id): @@ -401,8 +400,7 @@ class BeatportPlugin(BeetsPlugin): return None bp_track = self.client.get_track(match.group(2)) if bp_track is not None: - track = self._get_track_info(bp_track) - return track + return self._get_track_info(bp_track) return None def _get_releases(self, query): From dfb6fc3f5bfa4d471ccdcf18d1ad0f6f568be194 Mon Sep 17 00:00:00 2001 From: temrix Date: Sat, 21 Sep 2019 21:14:11 +0200 Subject: [PATCH 234/613] Test for presence and non-emptiness in one go. --- beetsplug/beatport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 1f2fbe451..c3933ed73 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -263,9 +263,9 @@ class BeatportTrack(BeatportObject): self.musical_key = six.text_type(data['key'].get('shortName')) # Use 'subgenre' and if not present, 'genre' as a fallback. - if 'subGenres' in data and data['subGenres']: + if data.get('subGenres'): self.genre = six.text_type(data['subGenres'][0].get('name')) - elif 'genres' in data and data['genres']: + elif data.get('genres'): self.genre = six.text_type(data['genres'][0].get('name')) From 4bc057fd5edc70015043b81c21559055989c8343 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 27 Sep 2019 17:11:47 -0700 Subject: [PATCH 235/613] Exclude invalid musical keys --- beetsplug/beatport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index c3933ed73..8665fc392 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -259,7 +259,7 @@ class BeatportTrack(BeatportObject): self.track_number = data.get('trackNumber') if 'bpm' in data: self.bpm = data['bpm'] - if 'key' in data: + if data.get('key'): self.musical_key = six.text_type(data['key'].get('shortName')) # Use 'subgenre' and if not present, 'genre' as a fallback. From 13792bd8abe8f2c27d23f7b34dbebd10b27e64aa Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 27 Sep 2019 17:33:49 -0700 Subject: [PATCH 236/613] Always set musical_key --- beetsplug/beatport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 8665fc392..de3554d95 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -259,8 +259,7 @@ class BeatportTrack(BeatportObject): self.track_number = data.get('trackNumber') if 'bpm' in data: self.bpm = data['bpm'] - if data.get('key'): - self.musical_key = six.text_type(data['key'].get('shortName')) + self.musical_key = six.text_type(data.get('key', {}).get('shortName')) # Use 'subgenre' and if not present, 'genre' as a fallback. if data.get('subGenres'): From 7b9ebcbf2f9e3287c5aaa83573dd28e7eb8b8d03 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Fri, 27 Sep 2019 17:48:28 -0700 Subject: [PATCH 237/613] Properly guard against None key --- beetsplug/beatport.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index de3554d95..655cd83b7 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -259,7 +259,9 @@ class BeatportTrack(BeatportObject): self.track_number = data.get('trackNumber') if 'bpm' in data: self.bpm = data['bpm'] - self.musical_key = six.text_type(data.get('key', {}).get('shortName')) + self.musical_key = six.text_type( + (data.get('key') or {}).get('shortName') + ) # Use 'subgenre' and if not present, 'genre' as a fallback. if data.get('subGenres'): From 80f54ecb9ee0e684b22919d6a9aa410ff56323ad Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 28 Sep 2019 10:01:19 -0700 Subject: [PATCH 238/613] Update changelog.rst --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 261b74505..af9f29c1f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -73,6 +73,8 @@ New features: * :doc:`/plugins/beatport`: The plugin now gets the musical key, BPM and the genre for each track. :bug:`2080` +* :doc:`/plugins/beatport`: Fix default assignment of the musical key. + :bug:`3377` Fixes: From 3ffaaa1f37805e18e33d694f0af2d19217680d22 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 28 Sep 2019 11:37:58 -0700 Subject: [PATCH 239/613] Default bpm to None --- beetsplug/beatport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 655cd83b7..be76b902f 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -257,8 +257,7 @@ class BeatportTrack(BeatportObject): self.url = "https://beatport.com/track/{0}/{1}" \ .format(data['slug'], data['id']) self.track_number = data.get('trackNumber') - if 'bpm' in data: - self.bpm = data['bpm'] + self.bpm = data.get('bpm') self.musical_key = six.text_type( (data.get('key') or {}).get('shortName') ) From 5f51b7b38e0378e2889691c05b66f99a5d80bd8f Mon Sep 17 00:00:00 2001 From: Alexander Miller Date: Tue, 1 Oct 2019 07:42:10 +0200 Subject: [PATCH 240/613] docs/plugins/discogs.rst: Update plugin documentation Add Configuration section and describe the 'source_weight' option. --- docs/plugins/discogs.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 622a085b4..866532233 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -43,6 +43,13 @@ documentation), login to `Discogs`_, and visit the token`` button, and place the generated token in your configuration, as the ``user_token`` config option in the ``discogs`` section. +Configuration +------------- + +- **source_weight**: Penalty applied to Discogs matches during import. Set to + 0.0 to disable. + Default: ``0.5``. + Troubleshooting --------------- From c4ad4c69cb520c3ab4bdfc33ef8dbb94ea11b74a Mon Sep 17 00:00:00 2001 From: Alexander Miller Date: Tue, 1 Oct 2019 20:57:49 +0200 Subject: [PATCH 241/613] docs/plugins/index.rst: Describe configuration of metadata source plugins --- docs/plugins/index.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index b9f512b1f..24e427dce 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -35,6 +35,27 @@ like this:: pip install beets[fetchart,lyrics,lastgenre] +.. _metadata-source-plugin-configuration: + +Using Metadata Source Plugins +----------------------------- + +Some plugins provide sources for metadata in addition to MusicBrainz. These +plugins share the following configuration option: + +- **source_weight**: Penalty applied to matches during import. Set to 0.0 to + disable. + Default: ``0.5``. + +For example, to equally consider matches from Discogs and MusicBrainz add the +following to your configuration:: + + plugins: discogs + + discogs: + source_weight: 0.0 + + .. toctree:: :hidden: From 279dd314ae38bc7334a5d2303e1a464537d0df1a Mon Sep 17 00:00:00 2001 From: Alexander Miller Date: Tue, 1 Oct 2019 21:16:57 +0200 Subject: [PATCH 242/613] docs/plugins: Centralize documentation of source_weight option --- docs/plugins/beatport.rst | 5 +++++ docs/plugins/deezer.rst | 11 +---------- docs/plugins/discogs.rst | 4 +--- docs/plugins/spotify.rst | 6 +++--- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index d645e2043..cbf5b4312 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -36,6 +36,11 @@ also search for an id like so: beet import path/to/music/library --search-id id +Configuration +------------- + +This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`. + .. _requests: https://docs.python-requests.org/en/latest/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib .. _Beatport: https://beetport.com diff --git a/docs/plugins/deezer.rst b/docs/plugins/deezer.rst index f283df1b2..29f561e6a 100644 --- a/docs/plugins/deezer.rst +++ b/docs/plugins/deezer.rst @@ -22,13 +22,4 @@ prompt during import:: Configuration ------------- -Put these options in config.yaml under the ``deezer:`` section: - -- **source_weight**: Penalty applied to Deezer matches during import. Set to - 0.0 to disable. - Default: ``0.5``. - -Here's an example:: - - deezer: - source_weight: 0.7 +This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`. diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 866532233..6c16083ee 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -46,9 +46,7 @@ token`` button, and place the generated token in your configuration, as the Configuration ------------- -- **source_weight**: Penalty applied to Discogs matches during import. Set to - 0.0 to disable. - Default: ``0.5``. +This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`. Troubleshooting --------------- diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index 5d6ae8f47..96b198f64 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -52,6 +52,9 @@ prompt during import:: Configuration ------------- +This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`. In addition, the following +configuration options are provided. + The default options should work as-is, but there are some options you can put in config.yaml under the ``spotify:`` section: @@ -79,9 +82,6 @@ in config.yaml under the ``spotify:`` section: track/album/artist fields before sending them to Spotify. Can be useful for changing certain abbreviations, like ft. -> feat. See the examples below. Default: None. -- **source_weight**: Penalty applied to Spotify matches during import. Set to - 0.0 to disable. - Default: ``0.5``. Here's an example:: From f14137fcc2450869dc96825e81f362bdedc08cb3 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 18:04:12 -0700 Subject: [PATCH 243/613] Add BPSyncPlugin --- beets/autotag/__init__.py | 2 +- beets/autotag/hooks.py | 4 +- beets/library.py | 12 +++ beets/plugins.py | 13 ++- beetsplug/beatport.py | 33 +++---- beetsplug/bpsync.py | 192 ++++++++++++++++++++++++++++++++++++++ beetsplug/mbsync.py | 17 +--- test/test_beatport.py | 16 ++-- 8 files changed, 238 insertions(+), 51 deletions(-) create mode 100644 beetsplug/bpsync.py diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 38db0a07b..ee1bf051c 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -186,7 +186,7 @@ def apply_metadata(album_info, mapping): 'mb_workid', 'work_disambig', 'bpm', - 'musical_key', + 'initial_key', 'genre' ) } diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index f59aaea42..c0f0ace7b 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -180,7 +180,7 @@ class TrackInfo(object): data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, track_alt=None, work=None, mb_workid=None, work_disambig=None, bpm=None, - musical_key=None, genre=None): + initial_key=None, genre=None): self.title = title self.track_id = track_id self.release_track_id = release_track_id @@ -206,7 +206,7 @@ class TrackInfo(object): self.mb_workid = mb_workid self.work_disambig = work_disambig self.bpm = bpm - self.musical_key = musical_key + self.initial_key = initial_key self.genre = genre # As above, work around a bug in python-musicbrainz-ngs. diff --git a/beets/library.py b/beets/library.py index 59791959d..239d96e81 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1648,6 +1648,18 @@ class DefaultTemplateFunctions(object): return falseval +def apply_item_changes(lib, item, move, pretend, write): + """Store, move and write the item according to the arguments. + """ + if not pretend: + # Move the item if it's in the library. + if move and lib.directory in util.ancestry(item.path): + item.move(with_album=False) + + if write: + item.try_write() + item.store() + # Get the name of tmpl_* functions in the above class. DefaultTemplateFunctions._func_names = \ [s for s in dir(DefaultTemplateFunctions) diff --git a/beets/plugins.py b/beets/plugins.py index b0752203f..9a0f2cc73 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -26,7 +26,7 @@ from functools import wraps import beets -from beets import logging +from beets import library, logging, ui, util import mediafile import six @@ -635,11 +635,11 @@ class MetadataSourcePlugin(object): :param artists: Iterable of artist dicts returned by API. :type artists: list[dict] - :param id_key: Key corresponding to ``artist_id`` value. - :type id_key: str - :param name_key: Keys corresponding to values to concatenate + :param id_key: Key or index corresponding to ``artist_id`` value. + :type id_key: str or int + :param name_key: Key or index corresponding to values to concatenate for ``artist``. - :type name_key: str + :type name_key: str or int :return: Normalized artist string. :rtype: str """ @@ -649,6 +649,8 @@ class MetadataSourcePlugin(object): if not artist_id: artist_id = artist[id_key] name = artist[name_key] + # Strip disambiguation number. + name = re.sub(r' \(\d+\)$', '', name) # Move articles to the front. name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) artist_names.append(name) @@ -724,3 +726,4 @@ class MetadataSourcePlugin(object): return get_distance( data_source=self.data_source, info=track_info, config=self.config ) + diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index be76b902f..360a40173 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -29,7 +29,7 @@ from requests_oauthlib.oauth1_session import (TokenRequestDenied, TokenMissing, import beets import beets.ui from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance -from beets.plugins import BeetsPlugin +from beets.plugins import BeetsPlugin, MetadataSourcePlugin import confuse @@ -228,6 +228,7 @@ class BeatportRelease(BeatportObject): if 'slug' in data: self.url = "https://beatport.com/release/{0}/{1}".format( data['slug'], data['id']) + self.genre = data.get('genre') @six.python_2_unicode_compatible @@ -258,7 +259,7 @@ class BeatportTrack(BeatportObject): .format(data['slug'], data['id']) self.track_number = data.get('trackNumber') self.bpm = data.get('bpm') - self.musical_key = six.text_type( + self.initial_key = six.text_type( (data.get('key') or {}).get('shortName') ) @@ -270,6 +271,8 @@ class BeatportTrack(BeatportObject): class BeatportPlugin(BeetsPlugin): + data_source = 'Beatport' + def __init__(self): super(BeatportPlugin, self).__init__() self.config.add({ @@ -337,7 +340,7 @@ class BeatportPlugin(BeetsPlugin): for albums. """ dist = Distance() - if album_info.data_source == 'Beatport': + if album_info.data_source == self.data_source: dist.add('source', self.config['source_weight'].as_number()) return dist @@ -346,7 +349,7 @@ class BeatportPlugin(BeetsPlugin): for individual tracks. """ dist = Distance() - if track_info.data_source == 'Beatport': + if track_info.data_source == self.data_source: dist.add('source', self.config['source_weight'].as_number()) return dist @@ -435,7 +438,8 @@ class BeatportPlugin(BeetsPlugin): day=release.release_date.day, label=release.label_name, catalognum=release.catalog_number, media=u'Digital', - data_source=u'Beatport', data_url=release.url) + data_source=self.data_source, data_url=release.url, + genre=release.genre) def _get_track_info(self, track): """Returns a TrackInfo object for a Beatport Track object. @@ -449,26 +453,15 @@ class BeatportPlugin(BeetsPlugin): artist=artist, artist_id=artist_id, length=length, index=track.track_number, medium_index=track.track_number, - data_source=u'Beatport', data_url=track.url, - bpm=track.bpm, musical_key=track.musical_key) + data_source=self.data_source, data_url=track.url, + bpm=track.bpm, initial_key=track.initial_key, + genre=track.genre) def _get_artist(self, artists): """Returns an artist string (all artists) and an artist_id (the main artist) for a list of Beatport release or track artists. """ - artist_id = None - bits = [] - for artist in artists: - if not artist_id: - artist_id = artist[0] - name = artist[1] - # Strip disambiguation number. - name = re.sub(r' \(\d+\)$', '', name) - # Move articles to the front. - name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I) - bits.append(name) - artist = ', '.join(bits).replace(' ,', ',') or None - return artist, artist_id + return MetadataSourcePlugin.get_artist(artists=artists, id_key=0, name_key=1) def _get_tracks(self, query): """Returns a list of TrackInfo objects for a Beatport query. diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py new file mode 100644 index 000000000..0cd4c903a --- /dev/null +++ b/beetsplug/bpsync.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Rahul Ahuja. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Update library's tags using MusicBrainz. +""" +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets import autotag, library, ui, util + +from .beatport import BeatportPlugin + + +class BPSyncPlugin(BeetsPlugin): + def __init__(self): + super(BPSyncPlugin, self).__init__() + self.beatport_plugin = BeatportPlugin() + self.beatport_plugin.setup() + + def commands(self): + cmd = ui.Subcommand('bpsync', help=u'update metadata from Beatport') + cmd.parser.add_option( + u'-p', + u'--pretend', + action='store_true', + help=u'show all changes but do nothing', + ) + cmd.parser.add_option( + u'-m', + u'--move', + action='store_true', + dest='move', + help=u"move files in the library directory", + ) + cmd.parser.add_option( + u'-M', + u'--nomove', + action='store_false', + dest='move', + help=u"don't move files in library", + ) + cmd.parser.add_option( + u'-W', + u'--nowrite', + action='store_false', + default=None, + dest='write', + help=u"don't write updated metadata to files", + ) + cmd.parser.add_format_option() + cmd.func = self.func + return [cmd] + + def func(self, lib, opts, args): + """Command handler for the bpsync function. + """ + move = ui.should_move(opts.move) + pretend = opts.pretend + write = ui.should_write(opts.write) + query = ui.decargs(args) + + self.singletons(lib, query, move, pretend, write) + self.albums(lib, query, move, pretend, write) + + def singletons(self, lib, query, move, pretend, write): + """Retrieve and apply info from the autotagger for items matched by + query. + """ + for item in lib.items(query + [u'singleton:true']): + if not item.mb_trackid: + self._log.info( + u'Skipping singleton with no mb_trackid: {}', item + ) + continue + + if not self.is_beatport_track(item): + self._log.info( + u'Skipping non-{} singleton: {}', + self.beatport_plugin.data_source, + item, + ) + continue + + # Apply. + track_info = self.beatport_plugin.track_for_id(item.mb_trackid) + with lib.transaction(): + autotag.apply_item_metadata(item, track_info) + library.apply_item_changes(lib, item, move, pretend, write) + + @staticmethod + def is_beatport_track(track): + return ( + track.get('data_source') == BeatportPlugin.data_source + and track.mb_trackid.isnumeric() + ) + + def get_album_tracks(self, album): + if not album.mb_albumid: + self._log.info(u'Skipping album with no mb_albumid: {}', album) + return False + if not album.mb_albumid.isnumeric(): + self._log.info( + u'Skipping album with invalid {} ID: {}', + self.beatport_plugin.data_source, + album, + ) + return False + tracks = list(album.items()) + if album.get('data_source') == self.beatport_plugin.data_source: + return tracks + if not all(self.is_beatport_track(track) for track in tracks): + self._log.info( + u'Skipping non-{} release: {}', + self.beatport_plugin.data_source, + album, + ) + return False + return tracks + + def albums(self, lib, query, move, pretend, write): + """Retrieve and apply info from the autotagger for albums matched by + query and their items. + """ + # Process matching albums. + for album in lib.albums(query): + # Do we have a valid Beatport album? + items = self.get_album_tracks(album) + if not items: + continue + + # Get the Beatport album information. + album_info = self.beatport_plugin.album_for_id(album.mb_albumid) + if not album_info: + self._log.info( + u'Release ID {} not found for album {}', + album.mb_albumid, + album, + ) + continue + + beatport_track_id_to_info = { + track.track_id: track for track in album_info.tracks + } + library_track_id_to_item = { + int(item.mb_trackid): item for item in items + } + item_to_info_mapping = { + library_track_id_to_item[track_id]: track_info + for track_id, track_info in beatport_track_id_to_info.items() + } + + self._log.info(u'applying changes to {}', album) + with lib.transaction(): + autotag.apply_metadata(album_info, item_to_info_mapping) + changed = False + # Find any changed item to apply Beatport changes to album. + any_changed_item = items[0] + for item in items: + item_changed = ui.show_model_changes(item) + changed |= item_changed + if item_changed: + any_changed_item = item + library.apply_item_changes( + lib, item, move, pretend, write + ) + + if not changed: + # No change to any item. + continue + + if not pretend: + # Update album structure to reflect an item in it. + for key in library.Album.item_keys: + album[key] = any_changed_item[key] + album.store() + + # Move album art (and any inconsistent items). + if move and lib.directory in util.ancestry(items[0].path): + self._log.debug(u'moving album {}', album) + album.move() diff --git a/beetsplug/mbsync.py b/beetsplug/mbsync.py index b8121d9c9..c3ef1674c 100644 --- a/beetsplug/mbsync.py +++ b/beetsplug/mbsync.py @@ -27,19 +27,6 @@ import re MBID_REGEX = r"(\d|\w){8}-(\d|\w){4}-(\d|\w){4}-(\d|\w){4}-(\d|\w){12}" -def apply_item_changes(lib, item, move, pretend, write): - """Store, move and write the item according to the arguments. - """ - if not pretend: - # Move the item if it's in the library. - if move and lib.directory in util.ancestry(item.path): - item.move(with_album=False) - - if write: - item.try_write() - item.store() - - class MBSyncPlugin(BeetsPlugin): def __init__(self): super(MBSyncPlugin, self).__init__() @@ -103,7 +90,7 @@ class MBSyncPlugin(BeetsPlugin): # Apply. with lib.transaction(): autotag.apply_item_metadata(item, track_info) - apply_item_changes(lib, item, move, pretend, write) + library.apply_item_changes(lib, item, move, pretend, write) def albums(self, lib, query, move, pretend, write): """Retrieve and apply info from the autotagger for albums matched by @@ -175,7 +162,7 @@ class MBSyncPlugin(BeetsPlugin): changed |= item_changed if item_changed: any_changed_item = item - apply_item_changes(lib, item, move, pretend, write) + library.apply_item_changes(lib, item, move, pretend, write) if not changed: # No change to any item. diff --git a/test/test_beatport.py b/test/test_beatport.py index 8e830df76..fb39627f8 100644 --- a/test/test_beatport.py +++ b/test/test_beatport.py @@ -482,12 +482,12 @@ class BeatportTest(_common.TestCase, TestHelper): items[4].bpm = 123 items[5].bpm = 123 - items[0].musical_key = 'Gmin' - items[1].musical_key = 'Gmaj' - items[2].musical_key = 'Fmaj' - items[3].musical_key = 'Amin' - items[4].musical_key = 'E♭maj' - items[5].musical_key = 'Amaj' + items[0].initial_key = 'Gmin' + items[1].initial_key = 'Gmaj' + items[2].initial_key = 'Fmaj' + items[3].initial_key = 'Amin' + items[4].initial_key = 'E♭maj' + items[5].initial_key = 'Amaj' for item in items: self.lib.add(item) @@ -549,9 +549,9 @@ class BeatportTest(_common.TestCase, TestHelper): for track, test_track in zip(self.tracks, self.test_tracks): self.assertEqual(track.bpm, test_track.bpm) - def test_musical_key_applied(self): + def test_initial_key_applied(self): for track, test_track in zip(self.tracks, self.test_tracks): - self.assertEqual(track.musical_key, test_track.musical_key) + self.assertEqual(track.initial_key, test_track.initial_key) def test_genre_applied(self): for track, test_track in zip(self.tracks, self.test_tracks): From a1885a571bb4e63ae8d7debeaf1bc03036a3493a Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 19:03:04 -0700 Subject: [PATCH 244/613] Add documentation, fix circular import --- beets/plugins.py | 2 +- beetsplug/bpsync.py | 21 ++++++++++----------- docs/changelog.rst | 5 +++++ docs/plugins/bpsync.rst | 37 +++++++++++++++++++++++++++++++++++++ docs/plugins/index.rst | 3 ++- 5 files changed, 55 insertions(+), 13 deletions(-) create mode 100644 docs/plugins/bpsync.rst diff --git a/beets/plugins.py b/beets/plugins.py index 9a0f2cc73..ff94a123c 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -26,7 +26,7 @@ from functools import wraps import beets -from beets import library, logging, ui, util +from beets import logging import mediafile import six diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py index 0cd4c903a..a69c9c3d7 100644 --- a/beetsplug/bpsync.py +++ b/beetsplug/bpsync.py @@ -13,7 +13,7 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Update library's tags using MusicBrainz. +"""Update library's tags using Beatport. """ from __future__ import division, absolute_import, print_function @@ -176,17 +176,16 @@ class BPSyncPlugin(BeetsPlugin): lib, item, move, pretend, write ) - if not changed: + if not changed or pretend: # No change to any item. continue - if not pretend: - # Update album structure to reflect an item in it. - for key in library.Album.item_keys: - album[key] = any_changed_item[key] - album.store() + # Update album structure to reflect an item in it. + for key in library.Album.item_keys: + album[key] = any_changed_item[key] + album.store() - # Move album art (and any inconsistent items). - if move and lib.directory in util.ancestry(items[0].path): - self._log.debug(u'moving album {}', album) - album.move() + # Move album art (and any inconsistent items). + if move and lib.directory in util.ancestry(items[0].path): + self._log.debug(u'moving album {}', album) + album.move() diff --git a/docs/changelog.rst b/docs/changelog.rst index af9f29c1f..b96a16cde 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -75,6 +75,11 @@ New features: :bug:`2080` * :doc:`/plugins/beatport`: Fix default assignment of the musical key. :bug:`3377` +* :doc:`/plugins/bpsync`: Add `bpsync` plugin to sync metadata changes + from the Beatport database. +* :doc:`/plugins/beatport`: Fix assignment of `genre` and rename `musical_key` + to `initial_key`. + :bug:`3387` Fixes: diff --git a/docs/plugins/bpsync.rst b/docs/plugins/bpsync.rst new file mode 100644 index 000000000..e43e33e5a --- /dev/null +++ b/docs/plugins/bpsync.rst @@ -0,0 +1,37 @@ +BPSync Plugin +============= + +This plugin provides the ``bpsync`` command, which lets you fetch metadata +from Beatport for albums and tracks that already have Beatport IDs. This +is useful for updating tags as they are fixed in the Beatport database, or +when you change your mind about some config options that change how tags are +written to files. If you have a music library that is already nicely tagged by +a program that also uses Beatport, this can speed up the initial import if you +just import "as-is" and then use ``bpsync`` to get up-to-date tags that are written +to the files according to your beets configuration. + + +Usage +----- + +Enable the ``bpsync`` plugin in your configuration (see :ref:`using-plugins`) +and then run ``beet bpsync QUERY`` to fetch updated metadata for a part of your +collection (or omit the query to run over your whole library). + +This plugin treats albums and singletons (non-album tracks) separately. It +first processes all matching singletons and then proceeds on to full albums. +The same query is used to search for both kinds of entities. + +The command has a few command-line options: + +* To preview the changes that would be made without applying them, use the + ``-p`` (``--pretend``) flag. +* By default, files will be moved (renamed) according to their metadata if + they are inside your beets library directory. To disable this, use the + ``-M`` (``--nomove``) command-line option. +* If you have the ``import.write`` configuration option enabled, then this + plugin will write new metadata to files' tags. To disable this, use the + ``-W`` (``--nowrite``) option. +* To customize the output of unrecognized items, use the ``-f`` + (``--format``) option. The default output is ``format_item`` or + ``format_album`` for items and albums, respectively. diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 24e427dce..b51370612 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -142,6 +142,7 @@ Metadata * :doc:`absubmit`: Analyse audio with the `streaming_extractor_music`_ program and submit the metadata to the AcousticBrainz server * :doc:`acousticbrainz`: Fetch various AcousticBrainz metadata * :doc:`bpm`: Measure tempo using keystrokes. +* :doc:`bpsync`: Fetch updated metadata from Beatport. * :doc:`edit`: Edit metadata from a text editor. * :doc:`embedart`: Embed album art images into files' metadata. * :doc:`fetchart`: Fetch album cover art from various sources. @@ -154,7 +155,7 @@ Metadata * :doc:`lastgenre`: Fetch genres based on Last.fm tags. * :doc:`lastimport`: Collect play counts from Last.fm. * :doc:`lyrics`: Automatically fetch song lyrics. -* :doc:`mbsync`: Fetch updated metadata from MusicBrainz +* :doc:`mbsync`: Fetch updated metadata from MusicBrainz. * :doc:`metasync`: Fetch metadata from local or remote sources * :doc:`mpdstats`: Connect to `MPD`_ and update the beets library with play statistics (last_played, play_count, skip_count, rating). From 14b8f30eadd10736dd3a8c891d586ff9a7af90d8 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 19:13:40 -0700 Subject: [PATCH 245/613] Appease flake8 --- beets/plugins.py | 1 - beetsplug/beatport.py | 4 +++- beetsplug/mbsync.py | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index ff94a123c..738b48b5e 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -726,4 +726,3 @@ class MetadataSourcePlugin(object): return get_distance( data_source=self.data_source, info=track_info, config=self.config ) - diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 360a40173..c2eb98665 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -461,7 +461,9 @@ class BeatportPlugin(BeetsPlugin): """Returns an artist string (all artists) and an artist_id (the main artist) for a list of Beatport release or track artists. """ - return MetadataSourcePlugin.get_artist(artists=artists, id_key=0, name_key=1) + return MetadataSourcePlugin.get_artist( + artists=artists, id_key=0, name_key=1 + ) def _get_tracks(self, query): """Returns a list of TrackInfo objects for a Beatport query. diff --git a/beetsplug/mbsync.py b/beetsplug/mbsync.py index c3ef1674c..18474e6ca 100644 --- a/beetsplug/mbsync.py +++ b/beetsplug/mbsync.py @@ -162,7 +162,9 @@ class MBSyncPlugin(BeetsPlugin): changed |= item_changed if item_changed: any_changed_item = item - library.apply_item_changes(lib, item, move, pretend, write) + library.apply_item_changes( + lib, item, move, pretend, write + ) if not changed: # No change to any item. From 571e8902fd7bd223336651880b669d80050b52ec Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 19:23:29 -0700 Subject: [PATCH 246/613] Fix indentation --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b96a16cde..963863ae8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -78,7 +78,7 @@ New features: * :doc:`/plugins/bpsync`: Add `bpsync` plugin to sync metadata changes from the Beatport database. * :doc:`/plugins/beatport`: Fix assignment of `genre` and rename `musical_key` - to `initial_key`. + to `initial_key`. :bug:`3387` Fixes: From ca57100f27227da1acea77f867b8a195fab0cb19 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 19:55:53 -0700 Subject: [PATCH 247/613] Add `bpsync` to toctree --- docs/plugins/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index b51370612..56864b2e0 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -65,6 +65,7 @@ following to your configuration:: beatport bpd bpm + bpsync bucket chroma convert From 5e6b8f5264022fe5cb5119453665612da1667ba0 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 22:04:30 -0700 Subject: [PATCH 248/613] DRY album/track distances --- beetsplug/beatport.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index c2eb98665..6a45ab93a 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -28,8 +28,8 @@ from requests_oauthlib.oauth1_session import (TokenRequestDenied, TokenMissing, import beets import beets.ui -from beets.autotag.hooks import AlbumInfo, TrackInfo, Distance -from beets.plugins import BeetsPlugin, MetadataSourcePlugin +from beets.autotag.hooks import AlbumInfo, TrackInfo +from beets.plugins import BeetsPlugin, MetadataSourcePlugin, get_distance import confuse @@ -336,22 +336,24 @@ class BeatportPlugin(BeetsPlugin): return self.config['tokenfile'].get(confuse.Filename(in_app_dir=True)) def album_distance(self, items, album_info, mapping): - """Returns the beatport source weight and the maximum source weight + """Returns the Beatport source weight and the maximum source weight for albums. """ - dist = Distance() - if album_info.data_source == self.data_source: - dist.add('source', self.config['source_weight'].as_number()) - return dist + return get_distance( + data_source=self.data_source, + info=album_info, + config=self.config + ) def track_distance(self, item, track_info): - """Returns the beatport source weight and the maximum source weight + """Returns the Beatport source weight and the maximum source weight for individual tracks. """ - dist = Distance() - if track_info.data_source == self.data_source: - dist.add('source', self.config['source_weight'].as_number()) - return dist + return get_distance( + data_source=self.data_source, + info=track_info, + config=self.config + ) def candidates(self, items, artist, release, va_likely): """Returns a list of AlbumInfo objects for beatport search results From ea03c7fac2199158b21cdcbc9013ba01afc7b9b6 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 22:16:32 -0700 Subject: [PATCH 249/613] Better readability --- beets/library.py | 15 ++++++++------- beetsplug/bpsync.py | 3 +-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/beets/library.py b/beets/library.py index 239d96e81..d38d3ebb5 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1651,14 +1651,15 @@ class DefaultTemplateFunctions(object): def apply_item_changes(lib, item, move, pretend, write): """Store, move and write the item according to the arguments. """ - if not pretend: - # Move the item if it's in the library. - if move and lib.directory in util.ancestry(item.path): - item.move(with_album=False) + if pretend: + return + # Move the item if it's in the library. + if move and lib.directory in util.ancestry(item.path): + item.move(with_album=False) - if write: - item.try_write() - item.store() + if write: + item.try_write() + item.store() # Get the name of tmpl_* functions in the above class. DefaultTemplateFunctions._func_names = \ diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py index a69c9c3d7..479698faf 100644 --- a/beetsplug/bpsync.py +++ b/beetsplug/bpsync.py @@ -176,8 +176,7 @@ class BPSyncPlugin(BeetsPlugin): lib, item, move, pretend, write ) - if not changed or pretend: - # No change to any item. + if pretend or not changed: continue # Update album structure to reflect an item in it. From 6c49afaf2241a801cf4e71de605b1e832105ecfa Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 22:49:32 -0700 Subject: [PATCH 250/613] Better naming --- beetsplug/bpsync.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py index 479698faf..ce4470611 100644 --- a/beetsplug/bpsync.py +++ b/beetsplug/bpsync.py @@ -100,10 +100,10 @@ class BPSyncPlugin(BeetsPlugin): library.apply_item_changes(lib, item, move, pretend, write) @staticmethod - def is_beatport_track(track): + def is_beatport_track(item): return ( - track.get('data_source') == BeatportPlugin.data_source - and track.mb_trackid.isnumeric() + item.get('data_source') == BeatportPlugin.data_source + and item.mb_trackid.isnumeric() ) def get_album_tracks(self, album): @@ -117,17 +117,17 @@ class BPSyncPlugin(BeetsPlugin): album, ) return False - tracks = list(album.items()) + items = list(album.items()) if album.get('data_source') == self.beatport_plugin.data_source: - return tracks - if not all(self.is_beatport_track(track) for track in tracks): + return items + if not all(self.is_beatport_track(item) for item in items): self._log.info( u'Skipping non-{} release: {}', self.beatport_plugin.data_source, album, ) return False - return tracks + return items def albums(self, lib, query, move, pretend, write): """Retrieve and apply info from the autotagger for albums matched by @@ -150,20 +150,20 @@ class BPSyncPlugin(BeetsPlugin): ) continue - beatport_track_id_to_info = { + beatport_trackid_to_trackinfo = { track.track_id: track for track in album_info.tracks } - library_track_id_to_item = { + library_trackid_to_item = { int(item.mb_trackid): item for item in items } - item_to_info_mapping = { - library_track_id_to_item[track_id]: track_info - for track_id, track_info in beatport_track_id_to_info.items() + item_to_trackinfo = { + library_trackid_to_item[track_id]: track_info + for track_id, track_info in beatport_trackid_to_trackinfo.items() } self._log.info(u'applying changes to {}', album) with lib.transaction(): - autotag.apply_metadata(album_info, item_to_info_mapping) + autotag.apply_metadata(album_info, item_to_trackinfo) changed = False # Find any changed item to apply Beatport changes to album. any_changed_item = items[0] From 0685305efb2d873b09db532b8dfbe7426d011533 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 22:53:35 -0700 Subject: [PATCH 251/613] Only sync tracks in library --- beetsplug/bpsync.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py index ce4470611..82867ab34 100644 --- a/beetsplug/bpsync.py +++ b/beetsplug/bpsync.py @@ -141,8 +141,8 @@ class BPSyncPlugin(BeetsPlugin): continue # Get the Beatport album information. - album_info = self.beatport_plugin.album_for_id(album.mb_albumid) - if not album_info: + albuminfo = self.beatport_plugin.album_for_id(album.mb_albumid) + if not albuminfo: self._log.info( u'Release ID {} not found for album {}', album.mb_albumid, @@ -151,19 +151,19 @@ class BPSyncPlugin(BeetsPlugin): continue beatport_trackid_to_trackinfo = { - track.track_id: track for track in album_info.tracks + track.track_id: track for track in albuminfo.tracks } library_trackid_to_item = { int(item.mb_trackid): item for item in items } item_to_trackinfo = { - library_trackid_to_item[track_id]: track_info - for track_id, track_info in beatport_trackid_to_trackinfo.items() + item: beatport_trackid_to_trackinfo[track_id] + for track_id, item in library_trackid_to_item.items() } self._log.info(u'applying changes to {}', album) with lib.transaction(): - autotag.apply_metadata(album_info, item_to_trackinfo) + autotag.apply_metadata(albuminfo, item_to_trackinfo) changed = False # Find any changed item to apply Beatport changes to album. any_changed_item = items[0] From a7cdaac5f8da3856fa34a0cf313e53e06dcd5741 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Thu, 3 Oct 2019 22:57:28 -0700 Subject: [PATCH 252/613] Consistent naming --- beets/library.py | 1 + beetsplug/bpsync.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index d38d3ebb5..fd48a2ab8 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1653,6 +1653,7 @@ def apply_item_changes(lib, item, move, pretend, write): """ if pretend: return + # Move the item if it's in the library. if move and lib.directory in util.ancestry(item.path): item.move(with_album=False) diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py index 82867ab34..70eb9094a 100644 --- a/beetsplug/bpsync.py +++ b/beetsplug/bpsync.py @@ -94,9 +94,9 @@ class BPSyncPlugin(BeetsPlugin): continue # Apply. - track_info = self.beatport_plugin.track_for_id(item.mb_trackid) + trackinfo = self.beatport_plugin.track_for_id(item.mb_trackid) with lib.transaction(): - autotag.apply_item_metadata(item, track_info) + autotag.apply_item_metadata(item, trackinfo) library.apply_item_changes(lib, item, move, pretend, write) @staticmethod From ce90b2aae5540146df250f2c59d3ca6c04d399aa Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 15:07:20 -0700 Subject: [PATCH 253/613] Improve documentation --- beets/library.py | 14 ------------- beets/plugins.py | 46 ++++++++++++++++++++++++++++++++++++----- beetsplug/bpsync.py | 8 +++---- beetsplug/mbsync.py | 8 +++---- docs/plugins/bpsync.rst | 17 +++++++-------- 5 files changed, 54 insertions(+), 39 deletions(-) diff --git a/beets/library.py b/beets/library.py index fd48a2ab8..59791959d 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1648,20 +1648,6 @@ class DefaultTemplateFunctions(object): return falseval -def apply_item_changes(lib, item, move, pretend, write): - """Store, move and write the item according to the arguments. - """ - if pretend: - return - - # Move the item if it's in the library. - if move and lib.directory in util.ancestry(item.path): - item.move(with_album=False) - - if write: - item.try_write() - item.store() - # Get the name of tmpl_* functions in the above class. DefaultTemplateFunctions._func_names = \ [s for s in dir(DefaultTemplateFunctions) diff --git a/beets/plugins.py b/beets/plugins.py index 738b48b5e..c688b058f 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -206,7 +206,7 @@ class BeetsPlugin(object): ``descriptor`` must be an instance of ``mediafile.MediaField``. """ - # Defer impor to prevent circular dependency + # Defer import to prevent circular dependency from beets import library mediafile.MediaFile.add_field(name, descriptor) library.Item._media_fields.add(name) @@ -590,6 +590,36 @@ def get_distance(config, data_source, info): return dist +def apply_item_changes(lib, item, move, pretend, write): + """Store, move, and write the item according to the arguments. + + :param lib: beets library. + :type lib: beets.library.Library + :param item: Item whose changes to apply. + :type item: beets.library.Item + :param move: Move the item if it's in the library. + :type move: bool + :param pretend: Return without moving, writing, or storing the item's + metadata. + :type pretend: bool + :param write: Write the item's metadata to its media file. + :type write: bool + """ + if pretend: + return + + from beets import util + + # Move the item if it's in the library. + if move and lib.directory in util.ancestry(item.path): + item.move(with_album=False) + + if write: + item.try_write() + + item.store() + + @six.add_metaclass(abc.ABCMeta) class MetadataSourcePlugin(object): def __init__(self): @@ -633,12 +663,18 @@ class MetadataSourcePlugin(object): """Returns an artist string (all artists) and an artist_id (the main artist) for a list of artist object dicts. - :param artists: Iterable of artist dicts returned by API. - :type artists: list[dict] - :param id_key: Key or index corresponding to ``artist_id`` value. + For each artist, this function moves articles (such as 'a', 'an', + and 'the') to the front and strips trailing disambiguation numbers. It + returns a tuple of containing the space-separated string of all + normalized artists and the ``id`` of the main artist. + + :param artists: Iterable of artist dicts or lists returned by API. + :type artists: list[dict] or list[list] + :param id_key: Key or index corresponding to ``artist_id`` + value (the main artist). :type id_key: str or int :param name_key: Key or index corresponding to values to concatenate - for ``artist``. + for the artist string (all artists) :type name_key: str or int :return: Normalized artist string. :rtype: str diff --git a/beetsplug/bpsync.py b/beetsplug/bpsync.py index 70eb9094a..aefb1517b 100644 --- a/beetsplug/bpsync.py +++ b/beetsplug/bpsync.py @@ -17,7 +17,7 @@ """ from __future__ import division, absolute_import, print_function -from beets.plugins import BeetsPlugin +from beets.plugins import BeetsPlugin, apply_item_changes from beets import autotag, library, ui, util from .beatport import BeatportPlugin @@ -97,7 +97,7 @@ class BPSyncPlugin(BeetsPlugin): trackinfo = self.beatport_plugin.track_for_id(item.mb_trackid) with lib.transaction(): autotag.apply_item_metadata(item, trackinfo) - library.apply_item_changes(lib, item, move, pretend, write) + apply_item_changes(lib, item, move, pretend, write) @staticmethod def is_beatport_track(item): @@ -172,9 +172,7 @@ class BPSyncPlugin(BeetsPlugin): changed |= item_changed if item_changed: any_changed_item = item - library.apply_item_changes( - lib, item, move, pretend, write - ) + apply_item_changes(lib, item, move, pretend, write) if pretend or not changed: continue diff --git a/beetsplug/mbsync.py b/beetsplug/mbsync.py index 18474e6ca..a2b3bc4aa 100644 --- a/beetsplug/mbsync.py +++ b/beetsplug/mbsync.py @@ -17,7 +17,7 @@ """ from __future__ import division, absolute_import, print_function -from beets.plugins import BeetsPlugin +from beets.plugins import BeetsPlugin, apply_item_changes from beets import autotag, library, ui, util from beets.autotag import hooks from collections import defaultdict @@ -90,7 +90,7 @@ class MBSyncPlugin(BeetsPlugin): # Apply. with lib.transaction(): autotag.apply_item_metadata(item, track_info) - library.apply_item_changes(lib, item, move, pretend, write) + apply_item_changes(lib, item, move, pretend, write) def albums(self, lib, query, move, pretend, write): """Retrieve and apply info from the autotagger for albums matched by @@ -162,9 +162,7 @@ class MBSyncPlugin(BeetsPlugin): changed |= item_changed if item_changed: any_changed_item = item - library.apply_item_changes( - lib, item, move, pretend, write - ) + apply_item_changes(lib, item, move, pretend, write) if not changed: # No change to any item. diff --git a/docs/plugins/bpsync.rst b/docs/plugins/bpsync.rst index e43e33e5a..8c576e816 100644 --- a/docs/plugins/bpsync.rst +++ b/docs/plugins/bpsync.rst @@ -2,13 +2,13 @@ BPSync Plugin ============= This plugin provides the ``bpsync`` command, which lets you fetch metadata -from Beatport for albums and tracks that already have Beatport IDs. This -is useful for updating tags as they are fixed in the Beatport database, or -when you change your mind about some config options that change how tags are -written to files. If you have a music library that is already nicely tagged by -a program that also uses Beatport, this can speed up the initial import if you -just import "as-is" and then use ``bpsync`` to get up-to-date tags that are written -to the files according to your beets configuration. +from Beatport for albums and tracks that already have Beatport IDs. +This plugins works similarly to :doc:`/plugins/mbsync`. + +If you have purchased music from Beatport, this can speed +up the initial import if you just import "as-is" and then use ``bpsync`` to +get up-to-date tags that are written to the files according to your beets +configuration. Usage @@ -32,6 +32,3 @@ The command has a few command-line options: * If you have the ``import.write`` configuration option enabled, then this plugin will write new metadata to files' tags. To disable this, use the ``-W`` (``--nowrite``) option. -* To customize the output of unrecognized items, use the ``-f`` - (``--format``) option. The default output is ``format_item`` or - ``format_album`` for items and albums, respectively. From e2c63d490149356d325a0a40fccdd6bf9a8e0ca9 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 15:25:48 -0700 Subject: [PATCH 254/613] Improve docstring wording --- beets/plugins.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index c688b058f..635dd9ca2 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -666,15 +666,15 @@ class MetadataSourcePlugin(object): For each artist, this function moves articles (such as 'a', 'an', and 'the') to the front and strips trailing disambiguation numbers. It returns a tuple of containing the space-separated string of all - normalized artists and the ``id`` of the main artist. + normalized artists and the ``id`` of the main/first artist. :param artists: Iterable of artist dicts or lists returned by API. :type artists: list[dict] or list[list] - :param id_key: Key or index corresponding to ``artist_id`` - value (the main artist). + :param id_key: Key or index corresponding to the value of ``id`` for + the main/first artist. :type id_key: str or int - :param name_key: Key or index corresponding to values to concatenate - for the artist string (all artists) + :param name_key: Key or index corresponding to values of names + to concatenate for the artist string (containing all artists). :type name_key: str or int :return: Normalized artist string. :rtype: str From a07972d02a42d2280b1eae9b99d33d7ea2b0715b Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 15:53:41 -0700 Subject: [PATCH 255/613] Fix docstring wording --- beets/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 635dd9ca2..fd539934e 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -665,7 +665,7 @@ class MetadataSourcePlugin(object): For each artist, this function moves articles (such as 'a', 'an', and 'the') to the front and strips trailing disambiguation numbers. It - returns a tuple of containing the space-separated string of all + returns a tuple of containing the comma-separated string of all normalized artists and the ``id`` of the main/first artist. :param artists: Iterable of artist dicts or lists returned by API. From 19f045290ecd1636dbffb95032ad33cc8515b881 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 15:55:45 -0700 Subject: [PATCH 256/613] Fix docstring wording --- beets/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index fd539934e..f4c78d55d 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -665,7 +665,7 @@ class MetadataSourcePlugin(object): For each artist, this function moves articles (such as 'a', 'an', and 'the') to the front and strips trailing disambiguation numbers. It - returns a tuple of containing the comma-separated string of all + returns a tuple containing the comma-separated string of all normalized artists and the ``id`` of the main/first artist. :param artists: Iterable of artist dicts or lists returned by API. From 52961d71f59d53fc98273729dc828b240ebb92b0 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 16:30:24 -0700 Subject: [PATCH 257/613] Add defaults to docstring --- beets/plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index f4c78d55d..d85f53f65 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -671,10 +671,11 @@ class MetadataSourcePlugin(object): :param artists: Iterable of artist dicts or lists returned by API. :type artists: list[dict] or list[list] :param id_key: Key or index corresponding to the value of ``id`` for - the main/first artist. + the main/first artist. Defaults to 'id'. :type id_key: str or int :param name_key: Key or index corresponding to values of names to concatenate for the artist string (containing all artists). + Defaults to 'name'. :type name_key: str or int :return: Normalized artist string. :rtype: str From 46b58ea70b4e2693685dd364462760deb658d4d3 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 20:59:47 -0700 Subject: [PATCH 258/613] Fix spelling --- docs/plugins/bpsync.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/bpsync.rst b/docs/plugins/bpsync.rst index 8c576e816..29cbd08e3 100644 --- a/docs/plugins/bpsync.rst +++ b/docs/plugins/bpsync.rst @@ -3,9 +3,9 @@ BPSync Plugin This plugin provides the ``bpsync`` command, which lets you fetch metadata from Beatport for albums and tracks that already have Beatport IDs. -This plugins works similarly to :doc:`/plugins/mbsync`. +This plugin works similarly to :doc:`/plugins/mbsync`. -If you have purchased music from Beatport, this can speed +If you have downloaded music from Beatport, this can speed up the initial import if you just import "as-is" and then use ``bpsync`` to get up-to-date tags that are written to the files according to your beets configuration. From 32ea225fad4e6a09f13eb96fca5248b31fe261f5 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sat, 5 Oct 2019 23:12:53 -0700 Subject: [PATCH 259/613] Guard against "empty" albums --- beetsplug/deezer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index a4337d273..4e3fca33a 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -83,6 +83,8 @@ class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): tracks_data = requests.get( self.album_url + deezer_id + '/tracks' ).json()['data'] + if not tracks_data: + return None tracks = [] medium_totals = collections.defaultdict(int) for i, track_data in enumerate(tracks_data, start=1): From 11e00c549b390f5bd527c0b4d4beedc23c70aaa3 Mon Sep 17 00:00:00 2001 From: Rahul Ahuja Date: Sun, 6 Oct 2019 00:07:23 -0700 Subject: [PATCH 260/613] Exclude empty albums from candidates --- beets/plugins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index d85f53f65..73d85cdd3 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -733,8 +733,9 @@ class MetadataSourcePlugin(object): query_filters = {'album': album} if not va_likely: query_filters['artist'] = artist - albums = self._search_api(query_type='album', filters=query_filters) - return [self.album_for_id(album_id=a['id']) for a in albums] + results = self._search_api(query_type='album', filters=query_filters) + albums = [self.album_for_id(album_id=r['id']) for r in results] + return [a for a in albums if a is not None] def item_candidates(self, item, artist, title): """Returns a list of TrackInfo objects for Search API results From e5b43d4bf476ddf299e7ecfe4a0645b515f4b176 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Wed, 9 Oct 2019 00:52:49 -0700 Subject: [PATCH 261/613] Extended the file type export options to include not only JSON but also XML and CSV --- beetsplug/export.py | 146 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 117 insertions(+), 29 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index d783f5b93..cd91fc393 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -18,8 +18,10 @@ from __future__ import division, absolute_import, print_function import sys -import json import codecs +import json +import csv +import xml.etree.ElementTree as ET from datetime import datetime, date from beets.plugins import BeetsPlugin @@ -52,6 +54,24 @@ class ExportPlugin(BeetsPlugin): 'sort_keys': True } }, + 'csv': { + # csv module formatting options + 'formatting': { + 'ensure_ascii': False, + 'indent': 0, + 'separators': (','), + 'sort_keys': True + } + }, + 'xml': { + # xml module formatting options + 'formatting': { + 'ensure_ascii': False, + 'indent': 4, + 'separators': (','), + 'sort_keys': True + } + } # TODO: Use something like the edit plugin # 'item_fields': [] }) @@ -78,17 +98,22 @@ class ExportPlugin(BeetsPlugin): u'-o', u'--output', help=u'path for the output file. If not given, will print the data' ) + cmd.parser.add_option( + u'-f', u'--format', default='json', + help=u'specify the format of the exported data. Your options are json (deafult), csv, and xml' + ) return [cmd] def run(self, lib, opts, args): file_path = opts.output - file_format = self.config['default_format'].get(str) file_mode = 'a' if opts.append else 'w' + file_format = opts.format format_options = self.config[file_format]['formatting'].get(dict) export_format = ExportFormat.factory( - file_format, **{ + file_type=file_format, + **{ 'file_path': file_path, 'file_mode': file_mode } @@ -108,44 +133,107 @@ class ExportPlugin(BeetsPlugin): except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue - data = key_filter(data) items += [data] - export_format.export(items, **format_options) class ExportFormat(object): """The output format type""" - - @classmethod - def factory(cls, type, **kwargs): - if type == "json": - if kwargs['file_path']: - return JsonFileFormat(**kwargs) - else: - return JsonPrintFormat() - raise NotImplementedError() - - def export(self, data, **kwargs): - raise NotImplementedError() - - -class JsonPrintFormat(ExportFormat): - """Outputs to the console""" - - def export(self, data, **kwargs): - json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs) - - -class JsonFileFormat(ExportFormat): - """Saves in a json file""" - def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): self.path = file_path self.mode = file_mode self.encoding = encoding + @classmethod + def factory(cls, file_type, **kwargs): + if file_type == "json": + return JsonFormat(**kwargs) + elif file_type == "csv": + return CSVFormat(**kwargs) + elif file_type == "xml": + return XMLFormat(**kwargs) + else: + raise NotImplementedError() + def export(self, data, **kwargs): + raise NotImplementedError() + + +class JsonFormat(ExportFormat): + """Saves in a json file""" + def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): + super(JsonFormat, self).__init__(file_path, file_mode, encoding) + self.export = self.export_to_file if self.path else self.export_to_terminal + + def export_to_terminal(self, data, **kwargs): + json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs) + + def export_to_file(self, data, **kwargs): with codecs.open(self.path, self.mode, self.encoding) as f: json.dump(data, f, cls=ExportEncoder, **kwargs) + + +class CSVFormat(ExportFormat): + """Saves in a csv file""" + def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): + super(CSVFormat, self).__init__(file_path, file_mode, encoding) + self.header = [] + + def export(self, data, **kwargs): + if data and type(data) is list and len(data) > 0: + self.header = list(data[0].keys()) + if self.path: + self.export_to_file(data, **kwargs) + else: + self.export_to_terminal(data, **kwargs) + + def export_to_terminal(self, data, **kwargs): + writer = csv.DictWriter(sys.stdout, fieldnames=self.header) + writer.writeheader() + writer.writerows(data) + + def export_to_file(self, data, **kwargs): + with codecs.open(self.path, self.mode, self.encoding) as f: + writer = csv.DictWriter(f, fieldnames=self.header) + writer.writeheader() + writer.writerows(data) + + +class XMLFormat(ExportFormat): + """Saves in a xml file""" + def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): + super(XMLFormat, self).__init__(file_path, file_mode, encoding) + + def export(self, data, **kwargs): + # create the file structure + library = ET.Element('library') + tracks_key = ET.SubElement(library, 'key') + tracks_key.text = "Tracks" + tracks_dict = ET.SubElement(library, 'dict') + if data and type(data) is list \ + and len(data) > 0 and type(data[0]) is dict: + index = 1 + for item in data: + track_key = ET.SubElement(tracks_dict, 'key') + track_key.text = str(index) + track_dict = ET.SubElement(tracks_dict, 'dict') + track_details = ET.SubElement(track_dict, 'Track ID') + track_details.text = str(index) + index += 1 + for key, value in item.items(): + track_details = ET.SubElement(track_dict, key) + track_details.text = value + data = str(ET.tostring(library, encoding=self.encoding)) + #data = ET.dump(library) + if self.path: + self.export_to_file(data, **kwargs) + else: + self.export_to_terminal(data, **kwargs) + + def export_to_terminal(self, data, **kwargs): + print(data) + + def export_to_file(self, data, **kwargs): + with codecs.open(self.path, self.mode, self.encoding) as f: + f.write(data) \ No newline at end of file From 8700e271d95435ede5a98b58b29ba972f1310ed0 Mon Sep 17 00:00:00 2001 From: Logan Arens Date: Wed, 9 Oct 2019 22:47:50 -0400 Subject: [PATCH 262/613] "beet update" now confirms that the library path exists before updating. Fixes #1934. --- beets/ui/commands.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 7941be46b..e9afa2f18 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1185,8 +1185,13 @@ def update_items(lib, query, album, move, pretend, fields): def update_func(lib, opts, args): + # Verify that the library folder exists. (disk isn't unmounted, for example) + if not os.path.exists(lib.directory): + ui.print_("Library path is unavailable or does not exist.") + if not ui.input_yn("Are you sure you want to continue (y/n)?", True): + return update_items(lib, decargs(args), opts.album, ui.should_move(opts.move), - opts.pretend, opts.fields) + opts.pretend, opts.fields) update_cmd = ui.Subcommand( From 0030a2218c62679444288d8196f569671799f8c8 Mon Sep 17 00:00:00 2001 From: Logan Arens Date: Wed, 9 Oct 2019 23:17:07 -0400 Subject: [PATCH 263/613] Small cosmetic changes to meet flake8 standards --- beets/ui/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index e9afa2f18..94baced22 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1185,13 +1185,13 @@ def update_items(lib, query, album, move, pretend, fields): def update_func(lib, opts, args): - # Verify that the library folder exists. (disk isn't unmounted, for example) + # Verify that the library folder exists to prevent accidental wipes. if not os.path.exists(lib.directory): ui.print_("Library path is unavailable or does not exist.") if not ui.input_yn("Are you sure you want to continue (y/n)?", True): return update_items(lib, decargs(args), opts.album, ui.should_move(opts.move), - opts.pretend, opts.fields) + opts.pretend, opts.fields) update_cmd = ui.Subcommand( From d91da56745551c5cb0f557bbc2d23ee0952dca6e Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Thu, 10 Oct 2019 08:50:09 +0100 Subject: [PATCH 264/613] Fix various typos --- beetsplug/replaygain.py | 2 +- docs/plugins/playlist.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index e9d5cc4af..1076ac714 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -570,7 +570,7 @@ class FfmpegBackend(Backend): value = line.split(b":", 1) if len(value) < 2: raise ReplayGainError( - u"ffmpeg ouput: expected key value pair, found {0}" + u"ffmpeg output: expected key value pair, found {0}" .format(line) ) value = value[1].lstrip() diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 31609cb3c..512c782cf 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -19,7 +19,7 @@ absolute path:: $ beet ls playlist:/path/to/someplaylist.m3u The plugin also supports referencing playlists by name. The playlist is then -seached in the playlist_dir and the ".m3u" extension is appended to the +searched in the playlist_dir and the ".m3u" extension is appended to the name:: $ beet ls playlist:anotherplaylist From a4c413e64adc140d824512e183f76d609f4e29cb Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 10 Oct 2019 08:18:09 -0400 Subject: [PATCH 265/613] Try fixing Travis package list --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 455ab4ca4..ddff93b9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ addons: - bash-completion - gir1.2-gst-plugins-base-1.0 - gir1.2-gstreamer-1.0 + - libgles2 - gstreamer1.0-plugins-good - gstreamer1.0-plugins-bad - imagemagick From bd0585cb92dad02f23e4b31274bdb10573af74c1 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Thu, 10 Oct 2019 21:45:26 +0100 Subject: [PATCH 266/613] Revert "Try fixing Travis package list" This reverts commit a4c413e64adc140d824512e183f76d609f4e29cb. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ddff93b9d..455ab4ca4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,6 @@ addons: - bash-completion - gir1.2-gst-plugins-base-1.0 - gir1.2-gstreamer-1.0 - - libgles2 - gstreamer1.0-plugins-good - gstreamer1.0-plugins-bad - imagemagick From 93e10264fb31776a7bbf552d3e11f1af2047f20c Mon Sep 17 00:00:00 2001 From: Austin Marino <32184751+austinmm@users.noreply.github.com> Date: Thu, 10 Oct 2019 16:26:20 -0700 Subject: [PATCH 267/613] Updated export.rst This update reflects the code changes I made to the export plugin --- docs/plugins/export.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index d712dfc8b..4809df58b 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -2,9 +2,11 @@ Export Plugin ============= The ``export`` plugin lets you get data from the items and export the content -as `JSON`_. +as `JSON`_, `CSV`_, or `XML`_. .. _JSON: https://www.json.org +.. _CSV: https://fileinfo.com/extension/csv +.. _XML: https://fileinfo.com/extension/xml Enable the ``export`` plugin (see :ref:`using-plugins` for help). Then, type ``beet export`` followed by a :doc:`query ` to get the data from your library. For example, run this:: @@ -36,11 +38,13 @@ The ``export`` command has these command-line options: * ``--append``: Appends the data to the file instead of writing. +* ``--format`` or ``-f``: Specifies the format of the exported data. If not informed, JSON will be used. + Configuration ------------- To configure the plugin, make a ``export:`` section in your configuration -file. Under the ``json`` key, these options are available: +file. Under the ``json``, ``csv``, and ``xml`` keys, these options are available: - **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. @@ -63,3 +67,15 @@ The default options look like this:: indent: 4 separators: [',' , ': '] sort_keys: true + csv: + formatting: + ensure_ascii: False + indent: 0 + separators: [','] + sort_keys: true + xml: + formatting: + ensure_ascii: False + indent: 4 + separators: ['>'] + sort_keys: true From 4dfb6b9fae26fe1602def5fc4ac6f855dda0a93e Mon Sep 17 00:00:00 2001 From: Austin Marino <32184751+austinmm@users.noreply.github.com> Date: Thu, 10 Oct 2019 16:32:24 -0700 Subject: [PATCH 268/613] Added examples of new format option --- docs/plugins/export.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 4809df58b..d548a5e66 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -15,6 +15,7 @@ your library. For example, run this:: to print a JSON file containing information about your Beatles tracks. + Command-Line Options -------------------- @@ -38,7 +39,12 @@ The ``export`` command has these command-line options: * ``--append``: Appends the data to the file instead of writing. -* ``--format`` or ``-f``: Specifies the format of the exported data. If not informed, JSON will be used. +* ``--format`` or ``-f``: Specifies the format the data will be exported as. If not informed, JSON will be used by default. + For example:: + + $ beet export -f csv beatles + $ beet export -f json beatles + $ beet export -f xml beatles Configuration ------------- From a8a480a691c478a60e5733eeed70b8597433dbd1 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Thu, 10 Oct 2019 16:33:46 -0700 Subject: [PATCH 269/613] Updated config formats --- beetsplug/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index cd91fc393..0666747b2 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -68,7 +68,7 @@ class ExportPlugin(BeetsPlugin): 'formatting': { 'ensure_ascii': False, 'indent': 4, - 'separators': (','), + 'separators': (''), 'sort_keys': True } } From 8ff875bded5e8a9415b9c47389131d788aa98efd Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Thu, 10 Oct 2019 19:34:57 -0700 Subject: [PATCH 270/613] Addec Unit test for export plugin --- test/test_export.py | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 test/test_export.py diff --git a/test/test_export.py b/test/test_export.py new file mode 100644 index 000000000..ad00f83e7 --- /dev/null +++ b/test/test_export.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Carl Suster +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Test the beets.export utilities associated with the export plugin. +""" + +from __future__ import division, absolute_import, print_function + +import unittest +from test import helper +from test.helper import TestHelper +#from beetsplug.export import ExportPlugin, ExportFormat, JSONFormat, CSVFormat, XMLFormat +#from collections import namedtuple + + +class ExportPluginTest(unittest.TestCase, TestHelper): + def setUp(self): + self.setup_beets() + self.load_plugins('export') + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() + + def test_json_output(self): + item1, item2 = self.add_item_fixtures(count=2) + item1.album = 'talbum' + item1.artist = "tartist" + item1.track = "ttrack" + item1.write() + item1.store() + + out = self.run_with_output('export', '-f json -i "track,album" tartist') + self.assertIn('"track": "' + item1.track + '"', out) + self.assertIn('"album": "' + item1.album + '"', out) + + def test_csv_output(self): + item1, item2 = self.add_item_fixtures(count=2) + item1.album = 'talbum' + item1.artist = "tartist" + item1.track = "ttrack" + item1.write() + item1.store() + + out = self.run_with_output('export', '-f json -i "track,album" tartist') + self.assertIn(item1.track + ',' + item1.album, out) + + def test_xml_output(self): + item1, item2 = self.add_item_fixtures(count=2) + item1.album = 'talbum' + item1.artist = "tartist" + item1.track = "ttrack" + item1.write() + item1.store() + + out = self.run_with_output('export', '-f json -i "track,album" tartist') + self.assertIn("" + item1.track + "", out) + self.assertIn("" + item1.album + "", out) + + """ + def setUp(self): + Opts = namedtuple('Opts', 'output append included_keys library format') + self.args = None + self._export = ExportPlugin() + included_keys = ['title,artist,album'] + self.opts = Opts(None, False, included_keys, True, "json") + self.export_format_classes = {"json": ExportFormat, "csv": CSVFormat, "xml": XMLFormat} + + def test_run(self, _format="json"): + self.opts.format = _format + self._export.run(lib=self.lib, opts=self.opts, args=self.args) + # 1.) Test that the ExportFormat Factory class method invoked the correct class + self.assertEqual(type(self._export.export_format), self.export_format_classes[_format]) + # 2.) Test that the cmd parser options specified were processed in correctly + self.assertEqual(self._export.export_format.path, self.opts.output) + mode = 'a' if self.opts.append else 'w' + self.assertEqual(self._export.export_format.mode, mode) + """ + + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') \ No newline at end of file From c31b488e549a2efdf3819654c82f8c5d41ced5ae Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Thu, 10 Oct 2019 19:35:49 -0700 Subject: [PATCH 271/613] Updated class fields to allow for easier unit testing --- beetsplug/export.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 0666747b2..63939f7e4 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -42,6 +42,9 @@ class ExportPlugin(BeetsPlugin): def __init__(self): super(ExportPlugin, self).__init__() + # Used when testing export plugin + self.run_results = None + self.export_format = None self.config.add({ 'default_format': 'json', @@ -68,7 +71,7 @@ class ExportPlugin(BeetsPlugin): 'formatting': { 'ensure_ascii': False, 'indent': 4, - 'separators': (''), + 'separators': ('>'), 'sort_keys': True } } @@ -105,13 +108,12 @@ class ExportPlugin(BeetsPlugin): return [cmd] def run(self, lib, opts, args): - file_path = opts.output file_mode = 'a' if opts.append else 'w' file_format = opts.format format_options = self.config[file_format]['formatting'].get(dict) - export_format = ExportFormat.factory( + self.export_format = ExportFormat.factory( file_type=file_format, **{ 'file_path': file_path, @@ -135,7 +137,9 @@ class ExportPlugin(BeetsPlugin): continue data = key_filter(data) items += [data] - export_format.export(items, **format_options) + + self.run_results = items + self.export_format.export(self.run_results, **format_options) class ExportFormat(object): @@ -144,6 +148,8 @@ class ExportFormat(object): self.path = file_path self.mode = file_mode self.encoding = encoding + # Used for testing + self.results = None @classmethod def factory(cls, file_type, **kwargs): @@ -167,11 +173,13 @@ class JsonFormat(ExportFormat): self.export = self.export_to_file if self.path else self.export_to_terminal def export_to_terminal(self, data, **kwargs): - json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs) + r = json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs) + self.results = str(r) def export_to_file(self, data, **kwargs): with codecs.open(self.path, self.mode, self.encoding) as f: - json.dump(data, f, cls=ExportEncoder, **kwargs) + r = json.dump(data, f, cls=ExportEncoder, **kwargs) + self.results = str(r) class CSVFormat(ExportFormat): @@ -192,12 +200,15 @@ class CSVFormat(ExportFormat): writer = csv.DictWriter(sys.stdout, fieldnames=self.header) writer.writeheader() writer.writerows(data) + self.results = str(writer) + def export_to_file(self, data, **kwargs): with codecs.open(self.path, self.mode, self.encoding) as f: writer = csv.DictWriter(f, fieldnames=self.header) writer.writeheader() writer.writerows(data) + self.results = str(writer) class XMLFormat(ExportFormat): @@ -233,7 +244,9 @@ class XMLFormat(ExportFormat): def export_to_terminal(self, data, **kwargs): print(data) + self.results = str(data) def export_to_file(self, data, **kwargs): with codecs.open(self.path, self.mode, self.encoding) as f: - f.write(data) \ No newline at end of file + f.write(data) + self.results = str(data) \ No newline at end of file From 72fcceac99b5609f2becbd0c101f90353c46e55e Mon Sep 17 00:00:00 2001 From: Logan Arens Date: Sat, 12 Oct 2019 12:22:29 -0400 Subject: [PATCH 272/613] Added changelog entry. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 963863ae8..16d5f7d1c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -116,6 +116,11 @@ Fixes: :bug:`3242` * Fix a bug that caused a crash when tagging items with the beatport plugin. :bug:`3374` +* ``beet update`` will now confirm that the user still wants to update if + their library folder cannot be found, preventing the user from accidentally + wiping out their beets database. + Thanks to :user:`logan-arens`. + :bug:`1934` For plugin developers: From baf4953ac0ceeca7a91f9e7d6db718da58b38508 Mon Sep 17 00:00:00 2001 From: Logan Arens Date: Sat, 12 Oct 2019 12:25:18 -0400 Subject: [PATCH 273/613] Library path is printed if `update` cannot find it --- beets/ui/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 94baced22..a6d712dbf 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1188,6 +1188,7 @@ def update_func(lib, opts, args): # Verify that the library folder exists to prevent accidental wipes. if not os.path.exists(lib.directory): ui.print_("Library path is unavailable or does not exist.") + ui.print_(lib.directory) if not ui.input_yn("Are you sure you want to continue (y/n)?", True): return update_items(lib, decargs(args), opts.album, ui.should_move(opts.move), From b2ef1941aaf624ee52551a918baaadc3873c0db7 Mon Sep 17 00:00:00 2001 From: Logan Arens Date: Sat, 12 Oct 2019 14:00:44 -0400 Subject: [PATCH 274/613] Changed library check from existence to directory --- beets/ui/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index a6d712dbf..56f9ad1f5 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1186,7 +1186,7 @@ def update_items(lib, query, album, move, pretend, fields): def update_func(lib, opts, args): # Verify that the library folder exists to prevent accidental wipes. - if not os.path.exists(lib.directory): + if not os.path.isdir(lib.directory): ui.print_("Library path is unavailable or does not exist.") ui.print_(lib.directory) if not ui.input_yn("Are you sure you want to continue (y/n)?", True): From 0e2c1e0d56c857437d6230b838c7365bc29c955c Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Sat, 12 Oct 2019 14:47:44 -0700 Subject: [PATCH 275/613] Made changes to reflect comments and suggestions made by sampsyo --- beetsplug/export.py | 79 +++++++++++------------------------------ docs/plugins/export.rst | 6 +--- 2 files changed, 21 insertions(+), 64 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 63939f7e4..0288e88c6 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -42,9 +42,6 @@ class ExportPlugin(BeetsPlugin): def __init__(self): super(ExportPlugin, self).__init__() - # Used when testing export plugin - self.run_results = None - self.export_format = None self.config.add({ 'default_format': 'json', @@ -60,19 +57,18 @@ class ExportPlugin(BeetsPlugin): 'csv': { # csv module formatting options 'formatting': { - 'ensure_ascii': False, - 'indent': 0, - 'separators': (','), - 'sort_keys': True + 'delimiter': ',', # column seperator + 'dialect': 'excel', # the name of the dialect to use + 'quotechar': '|' } }, 'xml': { # xml module formatting options 'formatting': { - 'ensure_ascii': False, - 'indent': 4, - 'separators': ('>'), - 'sort_keys': True + 'encoding': 'unicode', # the output encoding + 'xml_declaration':'True', # controls if an XML declaration should be added to the file + 'method': 'xml', # either "xml", "html" or "text" (default is "xml") + 'short_empty_elements': 'True' # controls the formatting of elements that contain no content. } } # TODO: Use something like the edit plugin @@ -103,17 +99,17 @@ class ExportPlugin(BeetsPlugin): ) cmd.parser.add_option( u'-f', u'--format', default='json', - help=u'specify the format of the exported data. Your options are json (deafult), csv, and xml' + help=u"the output format: json (default), csv, or xml" ) return [cmd] def run(self, lib, opts, args): file_path = opts.output file_mode = 'a' if opts.append else 'w' - file_format = opts.format + file_format = opts.format if opts.format else self.config['default_format'].get(str) format_options = self.config[file_format]['formatting'].get(dict) - self.export_format = ExportFormat.factory( + export_format = ExportFormat.factory( file_type=file_format, **{ 'file_path': file_path, @@ -135,11 +131,11 @@ class ExportPlugin(BeetsPlugin): except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue + data = key_filter(data) items += [data] - self.run_results = items - self.export_format.export(self.run_results, **format_options) + export_format.export(items, **format_options) class ExportFormat(object): @@ -148,8 +144,8 @@ class ExportFormat(object): self.path = file_path self.mode = file_mode self.encoding = encoding - # Used for testing - self.results = None + # out_stream is assigned sys.stdout (terminal output) or the file stream for the path specified + self.out_stream = codecs.open(self.path, self.mode, self.encoding) if self.path else sys.stdout @classmethod def factory(cls, file_type, **kwargs): @@ -170,16 +166,9 @@ class JsonFormat(ExportFormat): """Saves in a json file""" def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): super(JsonFormat, self).__init__(file_path, file_mode, encoding) - self.export = self.export_to_file if self.path else self.export_to_terminal - def export_to_terminal(self, data, **kwargs): - r = json.dump(data, sys.stdout, cls=ExportEncoder, **kwargs) - self.results = str(r) - - def export_to_file(self, data, **kwargs): - with codecs.open(self.path, self.mode, self.encoding) as f: - r = json.dump(data, f, cls=ExportEncoder, **kwargs) - self.results = str(r) + def export(self, data, **kwargs): + json.dump(data, self.out_stream, cls=ExportEncoder, **kwargs) class CSVFormat(ExportFormat): @@ -189,26 +178,11 @@ class CSVFormat(ExportFormat): self.header = [] def export(self, data, **kwargs): - if data and type(data) is list and len(data) > 0: + if data and len(data) > 0: self.header = list(data[0].keys()) - if self.path: - self.export_to_file(data, **kwargs) - else: - self.export_to_terminal(data, **kwargs) - - def export_to_terminal(self, data, **kwargs): - writer = csv.DictWriter(sys.stdout, fieldnames=self.header) + writer = csv.DictWriter(self.out_stream, fieldnames=self.header, **kwargs) writer.writeheader() writer.writerows(data) - self.results = str(writer) - - - def export_to_file(self, data, **kwargs): - with codecs.open(self.path, self.mode, self.encoding) as f: - writer = csv.DictWriter(f, fieldnames=self.header) - writer.writeheader() - writer.writerows(data) - self.results = str(writer) class XMLFormat(ExportFormat): @@ -235,18 +209,5 @@ class XMLFormat(ExportFormat): for key, value in item.items(): track_details = ET.SubElement(track_dict, key) track_details.text = value - data = str(ET.tostring(library, encoding=self.encoding)) - #data = ET.dump(library) - if self.path: - self.export_to_file(data, **kwargs) - else: - self.export_to_terminal(data, **kwargs) - - def export_to_terminal(self, data, **kwargs): - print(data) - self.results = str(data) - - def export_to_file(self, data, **kwargs): - with codecs.open(self.path, self.mode, self.encoding) as f: - f.write(data) - self.results = str(data) \ No newline at end of file + tree = ET.ElementTree(library) + tree.write(self.out_stream, **kwargs) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index d548a5e66..1cd9b09d3 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -40,11 +40,7 @@ The ``export`` command has these command-line options: * ``--append``: Appends the data to the file instead of writing. * ``--format`` or ``-f``: Specifies the format the data will be exported as. If not informed, JSON will be used by default. - For example:: - - $ beet export -f csv beatles - $ beet export -f json beatles - $ beet export -f xml beatles +The format options include csv, json and xml. Configuration ------------- From ec705fae1e4ad2e665cdfb4b5f765b43034bf447 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Sat, 12 Oct 2019 15:41:06 -0700 Subject: [PATCH 276/613] Updated documents and comments to reflcet recent code changes. Cleaned up code to better follow PEP-8 conventions and just work more efficiently all around. --- beetsplug/export.py | 36 ++++++++++++++++-------------------- docs/plugins/export.rst | 16 +++++++--------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 0288e88c6..43417efea 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -46,7 +46,7 @@ class ExportPlugin(BeetsPlugin): self.config.add({ 'default_format': 'json', 'json': { - # json module formatting options + # JSON module formatting options. 'formatting': { 'ensure_ascii': False, 'indent': 4, @@ -55,20 +55,19 @@ class ExportPlugin(BeetsPlugin): } }, 'csv': { - # csv module formatting options + # CSV module formatting options. 'formatting': { - 'delimiter': ',', # column seperator - 'dialect': 'excel', # the name of the dialect to use - 'quotechar': '|' + 'delimiter': ',', # The delimiter used to seperate columns. + 'dialect': 'excel' # The type of dialect to use when formating the file output. } }, 'xml': { - # xml module formatting options + # XML module formatting options. 'formatting': { - 'encoding': 'unicode', # the output encoding - 'xml_declaration':'True', # controls if an XML declaration should be added to the file - 'method': 'xml', # either "xml", "html" or "text" (default is "xml") - 'short_empty_elements': 'True' # controls the formatting of elements that contain no content. + 'encoding': 'unicode', # The output encoding. + 'xml_declaration':True, # Controls if an XML declaration should be added to the file. + 'method': 'xml', # Can be either "xml", "html" or "text" (default is "xml"). + 'short_empty_elements': True # Controls the formatting of elements that contain no content. } } # TODO: Use something like the edit plugin @@ -123,6 +122,7 @@ class ExportPlugin(BeetsPlugin): included_keys = [] for keys in opts.included_keys: included_keys.extend(keys.split(',')) + key_filter = make_key_filter(included_keys) for data_emitter in data_collector(lib, ui.decargs(args)): @@ -144,7 +144,7 @@ class ExportFormat(object): self.path = file_path self.mode = file_mode self.encoding = encoding - # out_stream is assigned sys.stdout (terminal output) or the file stream for the path specified + # Assigned sys.stdout (terminal output) or the file stream for the path specified. self.out_stream = codecs.open(self.path, self.mode, self.encoding) if self.path else sys.stdout @classmethod @@ -175,11 +175,9 @@ class CSVFormat(ExportFormat): """Saves in a csv file""" def __init__(self, file_path, file_mode=u'w', encoding=u'utf-8'): super(CSVFormat, self).__init__(file_path, file_mode, encoding) - self.header = [] def export(self, data, **kwargs): - if data and len(data) > 0: - self.header = list(data[0].keys()) + header = list(data[0].keys()) if data else [] writer = csv.DictWriter(self.out_stream, fieldnames=self.header, **kwargs) writer.writeheader() writer.writerows(data) @@ -191,23 +189,21 @@ class XMLFormat(ExportFormat): super(XMLFormat, self).__init__(file_path, file_mode, encoding) def export(self, data, **kwargs): - # create the file structure + # Creates the XML file structure. library = ET.Element('library') tracks_key = ET.SubElement(library, 'key') tracks_key.text = "Tracks" tracks_dict = ET.SubElement(library, 'dict') - if data and type(data) is list \ - and len(data) > 0 and type(data[0]) is dict: - index = 1 - for item in data: + if data and isinstance(data[0], dict): + for index, item in enumerate(data): track_key = ET.SubElement(tracks_dict, 'key') track_key.text = str(index) track_dict = ET.SubElement(tracks_dict, 'dict') track_details = ET.SubElement(track_dict, 'Track ID') track_details.text = str(index) - index += 1 for key, value in item.items(): track_details = ET.SubElement(track_dict, key) track_details.text = value + tree = ET.ElementTree(library) tree.write(self.out_stream, **kwargs) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 1cd9b09d3..f7f9e0217 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -68,16 +68,14 @@ The default options look like this:: ensure_ascii: False indent: 4 separators: [',' , ': '] - sort_keys: true + sort_keys: True csv: formatting: - ensure_ascii: False - indent: 0 - separators: [','] - sort_keys: true + delimiter: ',' + dialect: 'excel' xml: formatting: - ensure_ascii: False - indent: 4 - separators: ['>'] - sort_keys: true + encoding: 'unicode', + xml_declaration: True, + method: 'xml' + short_empty_elements: True From c5ebbe0b783928f0461ea0b5adce6d413d7ff90a Mon Sep 17 00:00:00 2001 From: Austin Marino <32184751+austinmm@users.noreply.github.com> Date: Sat, 12 Oct 2019 15:56:16 -0700 Subject: [PATCH 277/613] Added Comments to formating configurations --- docs/plugins/export.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index f7f9e0217..ba9ae2808 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -71,11 +71,34 @@ The default options look like this:: sort_keys: True csv: formatting: + /* + Used as the separating character between fields. The default value is a comma (,). + */ delimiter: ',' - dialect: 'excel' + /* + A dialect, in the context of reading and writing CSVs, + is a construct that allows you to create, store, + and re-use various formatting parameters for your data. + */ + dialect: 'excel' xml: formatting: + /* + Use encoding="unicode" to generate a Unicode string (otherwise, a bytestring is generated). + */ encoding: 'unicode', + /* + Controls if an XML declaration should be added to the file. + Use False for never, True for always, None for only if not US-ASCII or UTF-8 or Unicode (default is None). + */ xml_declaration: True, + /* + Can be either "xml", "html" or "text" (default is "xml") + */ method: 'xml' + /* + Controls the formatting of elements that contain no content. + If True (the default), they are emitted as a single self-closed tag, + otherwise they are emitted as a pair of start/end tags. + */ short_empty_elements: True From fa2c9ba2592c408949e6fa6ca00d5eebeee8aa45 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Sun, 13 Oct 2019 11:36:33 -0700 Subject: [PATCH 278/613] Aligned export related code with flake8 standards --- beetsplug/export.py | 33 ++++++++++++++++++++++----------- test/test_export.py | 42 +++++++++--------------------------------- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 43417efea..2f6af072e 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -57,17 +57,23 @@ class ExportPlugin(BeetsPlugin): 'csv': { # CSV module formatting options. 'formatting': { - 'delimiter': ',', # The delimiter used to seperate columns. - 'dialect': 'excel' # The type of dialect to use when formating the file output. + # The delimiter used to seperate columns. + 'delimiter': ',', + # The dialect to use when formating the file output. + 'dialect': 'excel' } }, 'xml': { # XML module formatting options. 'formatting': { - 'encoding': 'unicode', # The output encoding. - 'xml_declaration':True, # Controls if an XML declaration should be added to the file. - 'method': 'xml', # Can be either "xml", "html" or "text" (default is "xml"). - 'short_empty_elements': True # Controls the formatting of elements that contain no content. + # The output encoding. + 'encoding': 'unicode', + # Controls if XML declaration should be added to the file. + 'xml_declaration': True, + # Can be either "xml", "html" or "text" (default is "xml"). + 'method': 'xml', + # Controls formatting of elements that contain no content. + 'short_empty_elements': True } } # TODO: Use something like the edit plugin @@ -105,11 +111,12 @@ class ExportPlugin(BeetsPlugin): def run(self, lib, opts, args): file_path = opts.output file_mode = 'a' if opts.append else 'w' - file_format = opts.format if opts.format else self.config['default_format'].get(str) + file_format = opts.format if opts.format else \ + self.config['default_format'].get(str) format_options = self.config[file_format]['formatting'].get(dict) export_format = ExportFormat.factory( - file_type=file_format, + file_type=file_format, **{ 'file_path': file_path, 'file_mode': file_mode @@ -144,8 +151,12 @@ class ExportFormat(object): self.path = file_path self.mode = file_mode self.encoding = encoding - # Assigned sys.stdout (terminal output) or the file stream for the path specified. - self.out_stream = codecs.open(self.path, self.mode, self.encoding) if self.path else sys.stdout + """ self.out_stream = + sys.stdout if path doesn't exit + codecs.open(..) else + """ + self.out_stream = codecs.open(self.path, self.mode, self.encoding) \ + if self.path else sys.stdout @classmethod def factory(cls, file_type, **kwargs): @@ -178,7 +189,7 @@ class CSVFormat(ExportFormat): def export(self, data, **kwargs): header = list(data[0].keys()) if data else [] - writer = csv.DictWriter(self.out_stream, fieldnames=self.header, **kwargs) + writer = csv.DictWriter(self.out_stream, fieldnames=header, **kwargs) writer.writeheader() writer.writerows(data) diff --git a/test/test_export.py b/test/test_export.py index ad00f83e7..9d7c4a457 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -19,10 +19,7 @@ from __future__ import division, absolute_import, print_function import unittest -from test import helper from test.helper import TestHelper -#from beetsplug.export import ExportPlugin, ExportFormat, JSONFormat, CSVFormat, XMLFormat -#from collections import namedtuple class ExportPluginTest(unittest.TestCase, TestHelper): @@ -41,11 +38,11 @@ class ExportPluginTest(unittest.TestCase, TestHelper): item1.track = "ttrack" item1.write() item1.store() - - out = self.run_with_output('export', '-f json -i "track,album" tartist') + options = '-f json -i "track,album" ' + item1.artist + out = self.run_with_output('export', options) self.assertIn('"track": "' + item1.track + '"', out) self.assertIn('"album": "' + item1.album + '"', out) - + def test_csv_output(self): item1, item2 = self.add_item_fixtures(count=2) item1.album = 'talbum' @@ -53,10 +50,10 @@ class ExportPluginTest(unittest.TestCase, TestHelper): item1.track = "ttrack" item1.write() item1.store() - - out = self.run_with_output('export', '-f json -i "track,album" tartist') + options = '-f csv -i "track,album" ' + item1.artist + out = self.run_with_output('export', options) self.assertIn(item1.track + ',' + item1.album, out) - + def test_xml_output(self): item1, item2 = self.add_item_fixtures(count=2) item1.album = 'talbum' @@ -64,35 +61,14 @@ class ExportPluginTest(unittest.TestCase, TestHelper): item1.track = "ttrack" item1.write() item1.store() - - out = self.run_with_output('export', '-f json -i "track,album" tartist') + options = '-f xml -i "track,album" ' + item1.artist + out = self.run_with_output('export', options) self.assertIn("" + item1.track + "", out) self.assertIn("" + item1.album + "", out) - """ - def setUp(self): - Opts = namedtuple('Opts', 'output append included_keys library format') - self.args = None - self._export = ExportPlugin() - included_keys = ['title,artist,album'] - self.opts = Opts(None, False, included_keys, True, "json") - self.export_format_classes = {"json": ExportFormat, "csv": CSVFormat, "xml": XMLFormat} - - def test_run(self, _format="json"): - self.opts.format = _format - self._export.run(lib=self.lib, opts=self.opts, args=self.args) - # 1.) Test that the ExportFormat Factory class method invoked the correct class - self.assertEqual(type(self._export.export_format), self.export_format_classes[_format]) - # 2.) Test that the cmd parser options specified were processed in correctly - self.assertEqual(self._export.export_format.path, self.opts.output) - mode = 'a' if self.opts.append else 'w' - self.assertEqual(self._export.export_format.mode, mode) - """ - - def suite(): return unittest.TestLoader().loadTestsFromName(__name__) if __name__ == '__main__': - unittest.main(defaultTest='suite') \ No newline at end of file + unittest.main(defaultTest='suite') From 294b3cdb8cf4ed63494f33cf00aa101c0dd7754e Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Sun, 13 Oct 2019 11:41:06 -0700 Subject: [PATCH 279/613] updated format descriptions --- docs/plugins/export.rst | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index ba9ae2808..ea2624094 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -56,6 +56,18 @@ file. Under the ``json``, ``csv``, and ``xml`` keys, these options are available - **sort_keys**: Sorts the keys in JSON dictionaries. +- **delimiter**: Used as the separating character between fields. The default value is a comma (,). + +- **dialect**: A dialect, in the context of reading and writing CSVs, is a construct that allows you to create, store, and re-use various formatting parameters for your data. + +- **encoding**: Use encoding="unicode" to generate a Unicode string (otherwise, a bytestring is generated). + +- **xml_declaration**: Controls if an XML declaration should be added to the file. Use False for never, True for always, None for only if not US-ASCII or UTF-8 or Unicode (default is None). + +- **method**: Can be either "xml", "html" or "text" (default is "xml") + +- **short_empty_elements**: Controls the formatting of elements that contain no content. If True (the default), they are emitted as a single self-closed tag, otherwise they are emitted as a pair of start/end tags. + These options match the options from the `Python json module`_. .. _Python json module: https://docs.python.org/2/library/json.html#basic-usage @@ -71,34 +83,11 @@ The default options look like this:: sort_keys: True csv: formatting: - /* - Used as the separating character between fields. The default value is a comma (,). - */ delimiter: ',' - /* - A dialect, in the context of reading and writing CSVs, - is a construct that allows you to create, store, - and re-use various formatting parameters for your data. - */ dialect: 'excel' xml: formatting: - /* - Use encoding="unicode" to generate a Unicode string (otherwise, a bytestring is generated). - */ - encoding: 'unicode', - /* - Controls if an XML declaration should be added to the file. - Use False for never, True for always, None for only if not US-ASCII or UTF-8 or Unicode (default is None). - */ - xml_declaration: True, - /* - Can be either "xml", "html" or "text" (default is "xml") - */ + encoding: 'unicode' + xml_declaration: True method: 'xml' - /* - Controls the formatting of elements that contain no content. - If True (the default), they are emitted as a single self-closed tag, - otherwise they are emitted as a pair of start/end tags. - */ short_empty_elements: True From 4f0a2b78a32242f50406fd26503e510ca015d1d5 Mon Sep 17 00:00:00 2001 From: Austin Marino <32184751+austinmm@users.noreply.github.com> Date: Sun, 13 Oct 2019 11:45:33 -0700 Subject: [PATCH 280/613] Updated Format description layout --- docs/plugins/export.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index ea2624094..0a27c101b 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -48,25 +48,28 @@ Configuration To configure the plugin, make a ``export:`` section in your configuration file. Under the ``json``, ``csv``, and ``xml`` keys, these options are available: -- **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. +- **JSON Formatting** + - **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. -- **indent**: The number of spaces for indentation. + - **indent**: The number of spaces for indentation. -- **separators**: A ``[item_separator, dict_separator]`` tuple. + - **separators**: A ``[item_separator, dict_separator]`` tuple. -- **sort_keys**: Sorts the keys in JSON dictionaries. + - **sort_keys**: Sorts the keys in JSON dictionaries. -- **delimiter**: Used as the separating character between fields. The default value is a comma (,). +- **CSV Formatting** + - **delimiter**: Used as the separating character between fields. The default value is a comma (,). -- **dialect**: A dialect, in the context of reading and writing CSVs, is a construct that allows you to create, store, and re-use various formatting parameters for your data. + - **dialect**: A dialect, in the context of reading and writing CSVs, is a construct that allows you to create, store, and re-use various formatting parameters for your data. -- **encoding**: Use encoding="unicode" to generate a Unicode string (otherwise, a bytestring is generated). +- **XML Formatting** + - **encoding**: Use encoding="unicode" to generate a Unicode string (otherwise, a bytestring is generated). -- **xml_declaration**: Controls if an XML declaration should be added to the file. Use False for never, True for always, None for only if not US-ASCII or UTF-8 or Unicode (default is None). + - **xml_declaration**: Controls if an XML declaration should be added to the file. Use False for never, True for always, None for only if not US-ASCII or UTF-8 or Unicode (default is None). -- **method**: Can be either "xml", "html" or "text" (default is "xml") + - **method**: Can be either "xml", "html" or "text" (default is "xml") -- **short_empty_elements**: Controls the formatting of elements that contain no content. If True (the default), they are emitted as a single self-closed tag, otherwise they are emitted as a pair of start/end tags. + - **short_empty_elements**: Controls the formatting of elements that contain no content. If True (the default), they are emitted as a single self-closed tag, otherwise they are emitted as a pair of start/end tags. These options match the options from the `Python json module`_. From db5d21620bc5b98b8b5b401e1920a04cff6bed41 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Mon, 14 Oct 2019 13:57:32 -0700 Subject: [PATCH 281/613] Updated tests --- test/test_export.py | 54 +++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/test/test_export.py b/test/test_export.py index 9d7c4a457..0fae57ca4 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -21,7 +21,6 @@ from __future__ import division, absolute_import, print_function import unittest from test.helper import TestHelper - class ExportPluginTest(unittest.TestCase, TestHelper): def setUp(self): self.setup_beets() @@ -31,40 +30,37 @@ class ExportPluginTest(unittest.TestCase, TestHelper): self.unload_plugins() self.teardown_beets() - def test_json_output(self): - item1, item2 = self.add_item_fixtures(count=2) - item1.album = 'talbum' - item1.artist = "tartist" - item1.track = "ttrack" + def execute_command(self, format_type, artist): + options = ' -f %s -i "track,album" s'.format(format_type, artist) + actual = self.run_with_output('export', options) + return actual.replace(" ", "") + + def create_item(self, album='talbum', artist='tartist', track='ttrack'): + item1, = self.add_item_fixtures() + item1.album = album + item1.artist = artist + item1.track = track item1.write() item1.store() - options = '-f json -i "track,album" ' + item1.artist - out = self.run_with_output('export', options) - self.assertIn('"track": "' + item1.track + '"', out) - self.assertIn('"album": "' + item1.album + '"', out) + return item1 + + def test_json_output(self): + item1 = self.create_item() + actual = self.execute_command(format_type='json',artist=item1.artist) + expected = '[{"track":%s,"album":%s}]'.format(item1.track,item1.album) + self.assertIn(first=expected,second=actual,msg="export in JSON format failed") def test_csv_output(self): - item1, item2 = self.add_item_fixtures(count=2) - item1.album = 'talbum' - item1.artist = "tartist" - item1.track = "ttrack" - item1.write() - item1.store() - options = '-f csv -i "track,album" ' + item1.artist - out = self.run_with_output('export', options) - self.assertIn(item1.track + ',' + item1.album, out) + item1 = self.create_item() + actual = self.execute_command(format_type='json',artist=item1.artist) + expected = 'track,album\n%s,%s'.format(item1.track,item1.album) + self.assertIn(first=expected,second=actual,msg="export in CSV format failed") def test_xml_output(self): - item1, item2 = self.add_item_fixtures(count=2) - item1.album = 'talbum' - item1.artist = "tartist" - item1.track = "ttrack" - item1.write() - item1.store() - options = '-f xml -i "track,album" ' + item1.artist - out = self.run_with_output('export', options) - self.assertIn("" + item1.track + "", out) - self.assertIn("" + item1.album + "", out) + item1 = self.create_item() + actual = self.execute_command(format_type='json',artist=item1.artist) + expected = '%s%s'.format(item1.track,item1.album) + self.assertIn(first=expected,second=actual,msg="export in XML format failed") def suite(): From 0d818eced5cfc262cf63d939bdd55b1acef3f290 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Mon, 14 Oct 2019 17:02:39 -0700 Subject: [PATCH 282/613] Ran test to ensure it works --- beetsplug/export.py | 6 ++-- test/test_export.py | 69 +++++++++++++++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 2f6af072e..f0384ce55 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -67,13 +67,11 @@ class ExportPlugin(BeetsPlugin): # XML module formatting options. 'formatting': { # The output encoding. - 'encoding': 'unicode', + 'encoding': 'utf-8', # Controls if XML declaration should be added to the file. 'xml_declaration': True, # Can be either "xml", "html" or "text" (default is "xml"). - 'method': 'xml', - # Controls formatting of elements that contain no content. - 'short_empty_elements': True + 'method': 'xml' } } # TODO: Use something like the edit plugin diff --git a/test/test_export.py b/test/test_export.py index 0fae57ca4..35ad14718 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -20,6 +20,8 @@ from __future__ import division, absolute_import, print_function import unittest from test.helper import TestHelper +import re + class ExportPluginTest(unittest.TestCase, TestHelper): def setUp(self): @@ -31,36 +33,61 @@ class ExportPluginTest(unittest.TestCase, TestHelper): self.teardown_beets() def execute_command(self, format_type, artist): - options = ' -f %s -i "track,album" s'.format(format_type, artist) - actual = self.run_with_output('export', options) - return actual.replace(" ", "") - - def create_item(self, album='talbum', artist='tartist', track='ttrack'): - item1, = self.add_item_fixtures() - item1.album = album - item1.artist = artist - item1.track = track - item1.write() - item1.store() - return item1 + actual = self.run_with_output( + 'export', + '-f', format_type, + '-i', 'album,title', + artist + ) + return re.sub("\\s+", '', actual) + + def create_item(self): + item, = self.add_item_fixtures() + item.artist = 'xartist' + item.title = 'xtitle' + item.album = 'xalbum' + item.write() + item.store() + return item def test_json_output(self): item1 = self.create_item() - actual = self.execute_command(format_type='json',artist=item1.artist) - expected = '[{"track":%s,"album":%s}]'.format(item1.track,item1.album) - self.assertIn(first=expected,second=actual,msg="export in JSON format failed") + actual = self.execute_command( + format_type='json', + artist=item1.artist + ) + expected = u'[{"album":"%s","title":"%s"}]'\ + % (item1.album, item1.title) + self.assertIn( + expected, + actual + ) def test_csv_output(self): item1 = self.create_item() - actual = self.execute_command(format_type='json',artist=item1.artist) - expected = 'track,album\n%s,%s'.format(item1.track,item1.album) - self.assertIn(first=expected,second=actual,msg="export in CSV format failed") + actual = self.execute_command( + format_type='csv', + artist=item1.artist + ) + expected = u'album,title%s,%s'\ + % (item1.album, item1.title) + self.assertIn( + expected, + actual + ) def test_xml_output(self): item1 = self.create_item() - actual = self.execute_command(format_type='json',artist=item1.artist) - expected = '%s%s'.format(item1.track,item1.album) - self.assertIn(first=expected,second=actual,msg="export in XML format failed") + actual = self.execute_command( + format_type='xml', + artist=item1.artist + ) + expected = u'%s%s'\ + % (item1.album, item1.title) + self.assertIn( + expected, + actual + ) def suite(): From 5193f1b19acb8ed0df919223601c73795da63093 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Mon, 14 Oct 2019 17:31:59 -0700 Subject: [PATCH 283/613] Updated export doc --- docs/plugins/export.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 0a27c101b..16f5e8ac1 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -69,8 +69,6 @@ file. Under the ``json``, ``csv``, and ``xml`` keys, these options are available - **method**: Can be either "xml", "html" or "text" (default is "xml") - - **short_empty_elements**: Controls the formatting of elements that contain no content. If True (the default), they are emitted as a single self-closed tag, otherwise they are emitted as a pair of start/end tags. - These options match the options from the `Python json module`_. .. _Python json module: https://docs.python.org/2/library/json.html#basic-usage @@ -92,5 +90,4 @@ The default options look like this:: formatting: encoding: 'unicode' xml_declaration: True - method: 'xml' - short_empty_elements: True + method: 'xml' \ No newline at end of file From a9440ada2b47bb8902e030add91cf50daad78022 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Mon, 14 Oct 2019 18:17:12 -0700 Subject: [PATCH 284/613] Updated test structure for export --- test/test_export.py | 46 +++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/test/test_export.py b/test/test_export.py index 35ad14718..2ebc6cf95 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -21,12 +21,16 @@ from __future__ import division, absolute_import, print_function import unittest from test.helper import TestHelper import re +import beets +import beets.plugins class ExportPluginTest(unittest.TestCase, TestHelper): + def setUp(self): self.setup_beets() self.load_plugins('export') + self.test_values = {'title': 'xtitle', 'album': 'xalbum'} def tearDown(self): self.unload_plugins() @@ -44,8 +48,8 @@ class ExportPluginTest(unittest.TestCase, TestHelper): def create_item(self): item, = self.add_item_fixtures() item.artist = 'xartist' - item.title = 'xtitle' - item.album = 'xalbum' + item.title = self.test_values['title'] + item.album = self.test_values['album'] item.write() item.store() return item @@ -56,12 +60,13 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='json', artist=item1.artist ) - expected = u'[{"album":"%s","title":"%s"}]'\ - % (item1.album, item1.title) - self.assertIn( - expected, - actual - ) + for key, val in self.test_values.items(): + self.check_assertIn( + actual=actual, + str_format='"{0}":"{1}"', + key=key, + val=val + ) def test_csv_output(self): item1 = self.create_item() @@ -69,12 +74,13 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='csv', artist=item1.artist ) - expected = u'album,title%s,%s'\ - % (item1.album, item1.title) - self.assertIn( - expected, - actual - ) + for key, val in self.test_values.items(): + self.check_assertIn( + actual=actual, + str_format='{0}{1}', + key='', + val=val + ) def test_xml_output(self): item1 = self.create_item() @@ -82,8 +88,16 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='xml', artist=item1.artist ) - expected = u'%s%s'\ - % (item1.album, item1.title) + for key, val in self.test_values.items(): + self.check_assertIn( + actual=actual, + str_format='<{0}>{1}', + key=key, + val=val + ) + + def check_assertIn(self, actual, str_format, key, val): + expected = str_format.format(key, val) self.assertIn( expected, actual From 07138f86dc1095d3d16ac40d2598e754cba66622 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Mon, 14 Oct 2019 18:41:06 -0700 Subject: [PATCH 285/613] Fixed styling and test to pass cli --- docs/plugins/export.rst | 5 ++--- test/test_export.py | 11 ++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 16f5e8ac1..460d4d41c 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -39,8 +39,7 @@ The ``export`` command has these command-line options: * ``--append``: Appends the data to the file instead of writing. -* ``--format`` or ``-f``: Specifies the format the data will be exported as. If not informed, JSON will be used by default. -The format options include csv, json and xml. +* ``--format`` or ``-f``: Specifies the format the data will be exported as. If not informed, JSON will be used by default. The format options include csv, json and xml. Configuration ------------- @@ -90,4 +89,4 @@ The default options look like this:: formatting: encoding: 'unicode' xml_declaration: True - method: 'xml' \ No newline at end of file + method: 'xml' diff --git a/test/test_export.py b/test/test_export.py index 2ebc6cf95..c48a151f5 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -21,12 +21,9 @@ from __future__ import division, absolute_import, print_function import unittest from test.helper import TestHelper import re -import beets -import beets.plugins class ExportPluginTest(unittest.TestCase, TestHelper): - def setUp(self): self.setup_beets() self.load_plugins('export') @@ -61,7 +58,7 @@ class ExportPluginTest(unittest.TestCase, TestHelper): artist=item1.artist ) for key, val in self.test_values.items(): - self.check_assertIn( + self.check_assertin( actual=actual, str_format='"{0}":"{1}"', key=key, @@ -75,7 +72,7 @@ class ExportPluginTest(unittest.TestCase, TestHelper): artist=item1.artist ) for key, val in self.test_values.items(): - self.check_assertIn( + self.check_assertin( actual=actual, str_format='{0}{1}', key='', @@ -89,14 +86,14 @@ class ExportPluginTest(unittest.TestCase, TestHelper): artist=item1.artist ) for key, val in self.test_values.items(): - self.check_assertIn( + self.check_assertin( actual=actual, str_format='<{0}>{1}', key=key, val=val ) - def check_assertIn(self, actual, str_format, key, val): + def check_assertin(self, actual, str_format, key, val): expected = str_format.format(key, val) self.assertIn( expected, From 4251ff70dcb814c9158df684fedac269331970bf Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 00:15:45 -0700 Subject: [PATCH 286/613] updated the way in which xml is outputted --- beetsplug/export.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index f0384ce55..98b8f724b 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -67,9 +67,9 @@ class ExportPlugin(BeetsPlugin): # XML module formatting options. 'formatting': { # The output encoding. - 'encoding': 'utf-8', + 'encoding': 'unicode', # Controls if XML declaration should be added to the file. - 'xml_declaration': True, + # 'xml_declaration': True, # Can be either "xml", "html" or "text" (default is "xml"). 'method': 'xml' } @@ -199,20 +199,25 @@ class XMLFormat(ExportFormat): def export(self, data, **kwargs): # Creates the XML file structure. - library = ET.Element('library') - tracks_key = ET.SubElement(library, 'key') - tracks_key.text = "Tracks" - tracks_dict = ET.SubElement(library, 'dict') + library = ET.Element(u'library') + tracks_key = ET.SubElement(library, u'key') + tracks_key.text = u'tracks' + tracks_dict = ET.SubElement(library, u'dict') if data and isinstance(data[0], dict): for index, item in enumerate(data): - track_key = ET.SubElement(tracks_dict, 'key') + track_key = ET.SubElement(tracks_dict, u'key') track_key.text = str(index) - track_dict = ET.SubElement(tracks_dict, 'dict') - track_details = ET.SubElement(track_dict, 'Track ID') + track_dict = ET.SubElement(tracks_dict, u'dict') + track_details = ET.SubElement(track_dict, u'Track ID') track_details.text = str(index) for key, value in item.items(): track_details = ET.SubElement(track_dict, key) track_details.text = value - tree = ET.ElementTree(library) - tree.write(self.out_stream, **kwargs) + # tree = ET.ElementTree(element=library) + try: + data = ET.tostring(library, **kwargs) + except LookupError: + data = ET.tostring(library, encoding='utf-8', method='xml') + + self.out_stream.write(data) From b6588edac5c5f67d535f4b2f770c0882f6a03505 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Tue, 15 Oct 2019 12:04:04 +0200 Subject: [PATCH 287/613] unimported files plugin --- beetsplug/unimported.py | 36 ++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + docs/plugins/index.rst | 1 + docs/plugins/unimported.rst | 18 ++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 beetsplug/unimported.py create mode 100644 docs/plugins/unimported.rst diff --git a/beetsplug/unimported.py b/beetsplug/unimported.py new file mode 100644 index 000000000..5a9f4d82b --- /dev/null +++ b/beetsplug/unimported.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import os + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand + + +class Unimported(BeetsPlugin): + + def __init__(self): + super(Unimported, self).__init__() + self.config.add( + { + 'ignore_extensions': '[]' + } + ) + + def commands(self): + def print_unimported(lib, opts, args): + in_library = set( + (os.path.join(r, file) for r, d, f in os.walk(lib.directory) + for file in f if not any( + [file.endswith(extension.encode()) for extension in + self.config['ignore_extensions'].get()]))) + test = set((x.path for x in lib.items())) + for f in in_library - test: + print(f.decode('utf-8')) + + unimported = Subcommand( + 'unimported', + help='list files in library which have not been imported') + unimported.func = print_unimported + return [unimported] diff --git a/docs/changelog.rst b/docs/changelog.rst index 963863ae8..122e88499 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* :doc:`/plugins/unimported`: Added `unimported` plugin. * We now fetch information about `works`_ from MusicBrainz. MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` (the MBID), and ``work_disambig`` (the disambiguation string). diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 56864b2e0..066d6ec22 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -119,6 +119,7 @@ following to your configuration:: the thumbnails types + unimported web zero diff --git a/docs/plugins/unimported.rst b/docs/plugins/unimported.rst new file mode 100644 index 000000000..908636a30 --- /dev/null +++ b/docs/plugins/unimported.rst @@ -0,0 +1,18 @@ +Play Plugin +=========== + +The ``unimported`` plugin allows to list all files in the library folder which are not imported. + +Command Line Usage +------------------ + +To use the ``unimported`` plugin, enable it in your configuration (see +:ref:`using-plugins`). Then use it by invoking the ``beet unimported`` command. +The command will list all files in the library folder which are not imported. You can +exclude file extensions using the configuration file:: + + unimported: + ignore_extensions: [jpg,png] # default [] + + + From c7e81d8b81ff41a1ebb9907ca22066c0be59b9c5 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Tue, 15 Oct 2019 15:32:50 +0200 Subject: [PATCH 288/613] improvements from review --- beetsplug/unimported.py | 21 +++++++++++---------- docs/plugins/unimported.rst | 10 +++++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/beetsplug/unimported.py b/beetsplug/unimported.py index 5a9f4d82b..c31ab5702 100644 --- a/beetsplug/unimported.py +++ b/beetsplug/unimported.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import os +from beets import util from beets.plugins import BeetsPlugin from beets.ui import Subcommand @@ -14,23 +13,25 @@ class Unimported(BeetsPlugin): super(Unimported, self).__init__() self.config.add( { - 'ignore_extensions': '[]' + 'ignore_extensions': [] } ) def commands(self): def print_unimported(lib, opts, args): - in_library = set( + exts_to_ignore = self.config['ignore_extensions'].as_str_seq() + in_folder = set( (os.path.join(r, file) for r, d, f in os.walk(lib.directory) for file in f if not any( [file.endswith(extension.encode()) for extension in - self.config['ignore_extensions'].get()]))) - test = set((x.path for x in lib.items())) - for f in in_library - test: - print(f.decode('utf-8')) + exts_to_ignore]))) + in_library = set(x.path for x in lib.items()) + for f in in_folder - in_library: + print(util.displayable_path(f)) unimported = Subcommand( 'unimported', - help='list files in library which have not been imported') + help='list all files in the library folder which are not listed' + ' in the beets library database') unimported.func = print_unimported return [unimported] diff --git a/docs/plugins/unimported.rst b/docs/plugins/unimported.rst index 908636a30..6fe23eb16 100644 --- a/docs/plugins/unimported.rst +++ b/docs/plugins/unimported.rst @@ -1,7 +1,7 @@ -Play Plugin +Unimported Plugin =========== -The ``unimported`` plugin allows to list all files in the library folder which are not imported. +The ``unimported`` plugin allows to list all files in the library folder which are not listed in the beets library database. Command Line Usage ------------------ @@ -12,7 +12,7 @@ The command will list all files in the library folder which are not imported. Yo exclude file extensions using the configuration file:: unimported: - ignore_extensions: [jpg,png] # default [] - - + ignore_extensions: jpg png +The default configuration moves all English articles to the end of the string, +but you can override these defaults to make more complex changes. \ No newline at end of file From 712b4b8bdf4dcf38c23a696209f179cf76037ac0 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Tue, 15 Oct 2019 15:49:44 +0200 Subject: [PATCH 289/613] corrected to short underline in documentation --- docs/plugins/unimported.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/unimported.rst b/docs/plugins/unimported.rst index 6fe23eb16..2f26ecb67 100644 --- a/docs/plugins/unimported.rst +++ b/docs/plugins/unimported.rst @@ -1,5 +1,5 @@ Unimported Plugin -=========== +================= The ``unimported`` plugin allows to list all files in the library folder which are not listed in the beets library database. From 6be6ded02202e1163baea0e74b0d1701f9764455 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Tue, 15 Oct 2019 16:03:12 +0200 Subject: [PATCH 290/613] don't list album art paths as unimported --- beetsplug/unimported.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/unimported.py b/beetsplug/unimported.py index c31ab5702..584e41b5c 100644 --- a/beetsplug/unimported.py +++ b/beetsplug/unimported.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function import os -from beets import util +from beets import util, config from beets.plugins import BeetsPlugin from beets.ui import Subcommand @@ -26,7 +26,8 @@ class Unimported(BeetsPlugin): [file.endswith(extension.encode()) for extension in exts_to_ignore]))) in_library = set(x.path for x in lib.items()) - for f in in_folder - in_library: + art_files = set(x.artpath for x in lib.albums()) + for f in in_folder - in_library - art_files: print(util.displayable_path(f)) unimported = Subcommand( From 3d15eaff2e0261d8446ff85450bb4f74487723a5 Mon Sep 17 00:00:00 2001 From: Joris Date: Tue, 15 Oct 2019 16:35:21 +0200 Subject: [PATCH 291/613] Removed unused import --- beetsplug/unimported.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/unimported.py b/beetsplug/unimported.py index 584e41b5c..e9b7d0186 100644 --- a/beetsplug/unimported.py +++ b/beetsplug/unimported.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function import os -from beets import util, config +from beets import util from beets.plugins import BeetsPlugin from beets.ui import Subcommand From eb6055eeca5671fae99387b07e29875a7706aedb Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 11:45:01 -0700 Subject: [PATCH 292/613] Cleaned up comments and code --- beetsplug/export.py | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 98b8f724b..9a570c4fb 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -66,10 +66,6 @@ class ExportPlugin(BeetsPlugin): 'xml': { # XML module formatting options. 'formatting': { - # The output encoding. - 'encoding': 'unicode', - # Controls if XML declaration should be added to the file. - # 'xml_declaration': True, # Can be either "xml", "html" or "text" (default is "xml"). 'method': 'xml' } @@ -109,8 +105,7 @@ class ExportPlugin(BeetsPlugin): def run(self, lib, opts, args): file_path = opts.output file_mode = 'a' if opts.append else 'w' - file_format = opts.format if opts.format else \ - self.config['default_format'].get(str) + file_format = opts.format or self.config['default_format'].get(str) format_options = self.config[file_format]['formatting'].get(dict) export_format = ExportFormat.factory( @@ -149,10 +144,7 @@ class ExportFormat(object): self.path = file_path self.mode = file_mode self.encoding = encoding - """ self.out_stream = - sys.stdout if path doesn't exit - codecs.open(..) else - """ + # creates a file object to write/append or sets to stdout self.out_stream = codecs.open(self.path, self.mode, self.encoding) \ if self.path else sys.stdout @@ -200,24 +192,17 @@ class XMLFormat(ExportFormat): def export(self, data, **kwargs): # Creates the XML file structure. library = ET.Element(u'library') - tracks_key = ET.SubElement(library, u'key') - tracks_key.text = u'tracks' - tracks_dict = ET.SubElement(library, u'dict') + tracks = ET.SubElement(library, u'tracks') if data and isinstance(data[0], dict): for index, item in enumerate(data): - track_key = ET.SubElement(tracks_dict, u'key') - track_key.text = str(index) - track_dict = ET.SubElement(tracks_dict, u'dict') - track_details = ET.SubElement(track_dict, u'Track ID') - track_details.text = str(index) + track = ET.SubElement(tracks, u'track') for key, value in item.items(): - track_details = ET.SubElement(track_dict, key) + track_details = ET.SubElement(track, key) track_details.text = value - - # tree = ET.ElementTree(element=library) + # Depending on the version of python the encoding needs to change try: - data = ET.tostring(library, **kwargs) + data = ET.tostring(library, encoding='unicode', **kwargs) except LookupError: - data = ET.tostring(library, encoding='utf-8', method='xml') + data = ET.tostring(library, encoding='utf-8', **kwargs) self.out_stream.write(data) From 21d809180eb4f9034e8afbabac5d6a0c87c39213 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 11:45:38 -0700 Subject: [PATCH 293/613] Updated Test structure --- test/test_export.py | 97 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 23 deletions(-) diff --git a/test/test_export.py b/test/test_export.py index c48a151f5..5112ce9c7 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -20,7 +20,10 @@ from __future__ import division, absolute_import, print_function import unittest from test.helper import TestHelper -import re +import re # used to test csv format +import json +from xml.etree.ElementTree import Element +import xml.etree.ElementTree as ET class ExportPluginTest(unittest.TestCase, TestHelper): @@ -34,10 +37,11 @@ class ExportPluginTest(unittest.TestCase, TestHelper): self.teardown_beets() def execute_command(self, format_type, artist): + query = ','.format(list(self.test_values.keys())) actual = self.run_with_output( 'export', '-f', format_type, - '-i', 'album,title', + '-i', query, artist ) return re.sub("\\s+", '', actual) @@ -53,18 +57,64 @@ class ExportPluginTest(unittest.TestCase, TestHelper): def test_json_output(self): item1 = self.create_item() - actual = self.execute_command( - format_type='json', - artist=item1.artist + out = self.run_with_output( + 'export', + '-f', 'json', + '-i', 'album,title', + item1.artist ) + json_data = json.loads(out)[0] for key, val in self.test_values.items(): - self.check_assertin( - actual=actual, - str_format='"{0}":"{1}"', - key=key, - val=val - ) + self.assertTrue(key in json_data) + self.assertEqual(val, json_data[key]) + def test_csv_output(self): + item1 = self.create_item() + out = self.run_with_output( + 'export', + '-f', 'csv', + '-i', 'album,title', + item1.artist + ) + csv_list = re.split('\r', re.sub('\n', '', out)) + head = re.split(',', csv_list[0]) + vals = re.split(',|\r', csv_list[1]) + for index, column in enumerate(head): + self.assertTrue(self.test_values.get(column, None) is not None) + self.assertEqual(vals[index], self.test_values[column]) + + def test_xml_output(self): + item1 = self.create_item() + out = self.run_with_output( + 'export', + '-f', 'xml', + '-i', 'album,title', + item1.artist + ) + library = ET.fromstring(out) + self.assertIsInstance(library, Element) + for track in library[0]: + for details in track: + tag = details.tag + txt = details.text + self.assertTrue(tag in self.test_values, msg=tag) + self.assertEqual(self.test_values[tag], txt, msg=txt) + + def check_assertin(self, actual, str_format, key, val): + expected = str_format.format(key, val) + self.assertIn( + expected, + actual + ) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') + +""" def test_csv_output(self): item1 = self.create_item() actual = self.execute_command( @@ -93,16 +143,17 @@ class ExportPluginTest(unittest.TestCase, TestHelper): val=val ) - def check_assertin(self, actual, str_format, key, val): - expected = str_format.format(key, val) - self.assertIn( - expected, - actual + def test_json_output(self): + item1 = self.create_item() + actual = self.execute_command( + format_type='json', + artist=item1.artist ) - - -def suite(): - return unittest.TestLoader().loadTestsFromName(__name__) - -if __name__ == '__main__': - unittest.main(defaultTest='suite') + for key, val in self.test_values.items(): + self.check_assertin( + actual=actual, + str_format='"{0}":"{1}"', + key=key, + val=val + ) +""" From 623f553c92d18faed0a94f339decb9278217dd49 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 11:51:45 -0700 Subject: [PATCH 294/613] Updated Test structure --- test/test_export.py | 81 +++++++-------------------------------------- 1 file changed, 12 insertions(+), 69 deletions(-) diff --git a/test/test_export.py b/test/test_export.py index 5112ce9c7..757212a38 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -37,14 +37,14 @@ class ExportPluginTest(unittest.TestCase, TestHelper): self.teardown_beets() def execute_command(self, format_type, artist): - query = ','.format(list(self.test_values.keys())) - actual = self.run_with_output( + query = ','.join(self.test_values.keys()) + out = self.run_with_output( 'export', '-f', format_type, '-i', query, artist ) - return re.sub("\\s+", '', actual) + return out def create_item(self): item, = self.add_item_fixtures() @@ -57,11 +57,9 @@ class ExportPluginTest(unittest.TestCase, TestHelper): def test_json_output(self): item1 = self.create_item() - out = self.run_with_output( - 'export', - '-f', 'json', - '-i', 'album,title', - item1.artist + out = self.execute_command( + format_type='json', + artist=item1.artist ) json_data = json.loads(out)[0] for key, val in self.test_values.items(): @@ -70,11 +68,9 @@ class ExportPluginTest(unittest.TestCase, TestHelper): def test_csv_output(self): item1 = self.create_item() - out = self.run_with_output( - 'export', - '-f', 'csv', - '-i', 'album,title', - item1.artist + out = self.execute_command( + format_type='csv', + artist=item1.artist ) csv_list = re.split('\r', re.sub('\n', '', out)) head = re.split(',', csv_list[0]) @@ -85,11 +81,9 @@ class ExportPluginTest(unittest.TestCase, TestHelper): def test_xml_output(self): item1 = self.create_item() - out = self.run_with_output( - 'export', - '-f', 'xml', - '-i', 'album,title', - item1.artist + out = self.execute_command( + format_type='xml', + artist=item1.artist ) library = ET.fromstring(out) self.assertIsInstance(library, Element) @@ -100,60 +94,9 @@ class ExportPluginTest(unittest.TestCase, TestHelper): self.assertTrue(tag in self.test_values, msg=tag) self.assertEqual(self.test_values[tag], txt, msg=txt) - def check_assertin(self, actual, str_format, key, val): - expected = str_format.format(key, val) - self.assertIn( - expected, - actual - ) - def suite(): return unittest.TestLoader().loadTestsFromName(__name__) if __name__ == '__main__': unittest.main(defaultTest='suite') - -""" - def test_csv_output(self): - item1 = self.create_item() - actual = self.execute_command( - format_type='csv', - artist=item1.artist - ) - for key, val in self.test_values.items(): - self.check_assertin( - actual=actual, - str_format='{0}{1}', - key='', - val=val - ) - - def test_xml_output(self): - item1 = self.create_item() - actual = self.execute_command( - format_type='xml', - artist=item1.artist - ) - for key, val in self.test_values.items(): - self.check_assertin( - actual=actual, - str_format='<{0}>{1}', - key=key, - val=val - ) - - def test_json_output(self): - item1 = self.create_item() - actual = self.execute_command( - format_type='json', - artist=item1.artist - ) - for key, val in self.test_values.items(): - self.check_assertin( - actual=actual, - str_format='"{0}":"{1}"', - key=key, - val=val - ) -""" From d11e14b1a08cdfadc1eb2a4249ac0a93cd03efec Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Tue, 15 Oct 2019 21:54:35 +0200 Subject: [PATCH 295/613] improvements from review #2 --- beetsplug/unimported.py | 32 +++++++++++++++++++++++++++----- docs/changelog.rst | 2 +- docs/plugins/unimported.rst | 5 ++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/beetsplug/unimported.py b/beetsplug/unimported.py index e9b7d0186..544e9de46 100644 --- a/beetsplug/unimported.py +++ b/beetsplug/unimported.py @@ -1,10 +1,31 @@ # -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Joris Jensen +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +""" +List all files in the library folder which are not listed in the + beets library database, including art files +""" + from __future__ import absolute_import, division, print_function import os from beets import util from beets.plugins import BeetsPlugin -from beets.ui import Subcommand +from beets.ui import Subcommand, print_ + +__author__ = 'https://github.com/MrNuggelz' class Unimported(BeetsPlugin): @@ -19,16 +40,17 @@ class Unimported(BeetsPlugin): def commands(self): def print_unimported(lib, opts, args): - exts_to_ignore = self.config['ignore_extensions'].as_str_seq() + ignore_exts = [('.' + x).encode() for x + in self.config['ignore_extensions'].as_str_seq()] in_folder = set( (os.path.join(r, file) for r, d, f in os.walk(lib.directory) for file in f if not any( - [file.endswith(extension.encode()) for extension in - exts_to_ignore]))) + [file.endswith(extension) for extension in + ignore_exts]))) in_library = set(x.path for x in lib.items()) art_files = set(x.artpath for x in lib.albums()) for f in in_folder - in_library - art_files: - print(util.displayable_path(f)) + print_(util.displayable_path(f)) unimported = Subcommand( 'unimported', diff --git a/docs/changelog.rst b/docs/changelog.rst index 122e88499..29ad9b6eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog New features: -* :doc:`/plugins/unimported`: Added `unimported` plugin. +* :doc:`/plugins/unimported`: lets you find untracked files in your library directory. * We now fetch information about `works`_ from MusicBrainz. MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` (the MBID), and ``work_disambig`` (the disambiguation string). diff --git a/docs/plugins/unimported.rst b/docs/plugins/unimported.rst index 2f26ecb67..447c4ec8c 100644 --- a/docs/plugins/unimported.rst +++ b/docs/plugins/unimported.rst @@ -1,7 +1,7 @@ Unimported Plugin ================= -The ``unimported`` plugin allows to list all files in the library folder which are not listed in the beets library database. +The ``unimported`` plugin allows to list all files in the library folder which are not listed in the beets library database, including art files. Command Line Usage ------------------ @@ -14,5 +14,4 @@ exclude file extensions using the configuration file:: unimported: ignore_extensions: jpg png -The default configuration moves all English articles to the end of the string, -but you can override these defaults to make more complex changes. \ No newline at end of file +The default configuration list all unimported files, ignoring no extensions. \ No newline at end of file From d86e31d3706b0dcfdb4744c4701e37684c8eac55 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 13:10:47 -0700 Subject: [PATCH 296/613] Updated to reflect code changes and updated styling/format --- docs/plugins/export.rst | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 460d4d41c..8afc740cd 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -56,22 +56,26 @@ file. Under the ``json``, ``csv``, and ``xml`` keys, these options are available - **sort_keys**: Sorts the keys in JSON dictionaries. -- **CSV Formatting** - - **delimiter**: Used as the separating character between fields. The default value is a comma (,). - - - **dialect**: A dialect, in the context of reading and writing CSVs, is a construct that allows you to create, store, and re-use various formatting parameters for your data. - -- **XML Formatting** - - **encoding**: Use encoding="unicode" to generate a Unicode string (otherwise, a bytestring is generated). - - - **xml_declaration**: Controls if an XML declaration should be added to the file. Use False for never, True for always, None for only if not US-ASCII or UTF-8 or Unicode (default is None). - - - **method**: Can be either "xml", "html" or "text" (default is "xml") - These options match the options from the `Python json module`_. .. _Python json module: https://docs.python.org/2/library/json.html#basic-usage +- **CSV Formatting** + - **delimiter**: Used as the separating character between fields. The default value is a comma (,). + + - **dialect**: A dialect is a construct that allows you to create, store, and re-use various formatting parameters for your data. + +These options match the options from the `Python csv module`_. + +.. _Python csv module: https://docs.python.org/3/library/csv.html#csv-fmt-params + +- **XML Formatting** + - **method**: Can be either "xml", "html" or "text" (default is "xml") + +These options match the options from the `Python xml module`_. + +.. _Python xml module: https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.tostring + The default options look like this:: export: @@ -87,6 +91,4 @@ The default options look like this:: dialect: 'excel' xml: formatting: - encoding: 'unicode' - xml_declaration: True method: 'xml' From 7f6630c006406a488dae4d60e8a14d07d2d48765 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 14:16:23 -0700 Subject: [PATCH 297/613] removed xml configs from doc and code --- beetsplug/export.py | 5 +---- docs/plugins/export.rst | 12 +----------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 9a570c4fb..f7e84a570 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -65,10 +65,7 @@ class ExportPlugin(BeetsPlugin): }, 'xml': { # XML module formatting options. - 'formatting': { - # Can be either "xml", "html" or "text" (default is "xml"). - 'method': 'xml' - } + 'formatting': {} } # TODO: Use something like the edit plugin # 'item_fields': [] diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 8afc740cd..a88925765 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -69,13 +69,6 @@ These options match the options from the `Python csv module`_. .. _Python csv module: https://docs.python.org/3/library/csv.html#csv-fmt-params -- **XML Formatting** - - **method**: Can be either "xml", "html" or "text" (default is "xml") - -These options match the options from the `Python xml module`_. - -.. _Python xml module: https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.tostring - The default options look like this:: export: @@ -88,7 +81,4 @@ The default options look like this:: csv: formatting: delimiter: ',' - dialect: 'excel' - xml: - formatting: - method: 'xml' + dialect: 'excel' From d7b0e9347afd148510415a9a3c72a632cf5b5a71 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 15:32:03 -0700 Subject: [PATCH 298/613] Updated changelog to reflect export plugin changes --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cf14ae974..2826c0d94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,10 @@ Changelog New features: +* :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; + which allows for the ability to export in json, csv and xml. + Thanks to :user:`austinmm`. + :bug:`3402` * We now fetch information about `works`_ from MusicBrainz. MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` (the MBID), and ``work_disambig`` (the disambiguation string). From 5d7c937d41cd425b37c3c44e2e80e2234d0a9673 Mon Sep 17 00:00:00 2001 From: Austin Marino Date: Tue, 15 Oct 2019 15:34:59 -0700 Subject: [PATCH 299/613] fixed conflicting files issues with changelog --- docs/changelog.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2826c0d94..6f4fdc6d3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ New features: which allows for the ability to export in json, csv and xml. Thanks to :user:`austinmm`. :bug:`3402` +* :doc:`/plugins/unimported`: lets you find untracked files in your library directory. * We now fetch information about `works`_ from MusicBrainz. MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` (the MBID), and ``work_disambig`` (the disambiguation string). @@ -74,6 +75,16 @@ New features: you can now match tracks and albums using the `Deezer`_ database. Thanks to :user:`rhlahuja`. :bug:`3355` +* :doc:`/plugins/beatport`: The plugin now gets the musical key, BPM and the + genre for each track. + :bug:`2080` +* :doc:`/plugins/beatport`: Fix default assignment of the musical key. + :bug:`3377` +* :doc:`/plugins/bpsync`: Add `bpsync` plugin to sync metadata changes + from the Beatport database. +* :doc:`/plugins/beatport`: Fix assignment of `genre` and rename `musical_key` + to `initial_key`. + :bug:`3387` Fixes: @@ -108,6 +119,13 @@ Fixes: * ``none_rec_action`` does not import automatically when ``timid`` is enabled. Thanks to :user:`RollingStar`. :bug:`3242` +* Fix a bug that caused a crash when tagging items with the beatport plugin. + :bug:`3374` +* ``beet update`` will now confirm that the user still wants to update if + their library folder cannot be found, preventing the user from accidentally + wiping out their beets database. + Thanks to :user:`logan-arens`. + :bug:`1934` For plugin developers: @@ -140,6 +158,8 @@ For plugin developers: APIs to provide metadata matches for the importer. Refer to the Spotify and Deezer plugins for examples of using this template class. :bug:`3355` +* The autotag hooks have been modified such that they now take 'bpm', + 'musical_key' and a per-track based 'genre' as attributes. For packagers: From d45b8bb03e17348b2220f90318ae32fa3bb42a12 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 16 Oct 2019 14:27:06 -0400 Subject: [PATCH 300/613] Docs fixes from my code review --- docs/plugins/export.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index a88925765..6be20bec7 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -63,7 +63,7 @@ These options match the options from the `Python json module`_. - **CSV Formatting** - **delimiter**: Used as the separating character between fields. The default value is a comma (,). - - **dialect**: A dialect is a construct that allows you to create, store, and re-use various formatting parameters for your data. + - **dialect**: The kind of CSV file to produce. The default is `excel`. These options match the options from the `Python csv module`_. @@ -77,8 +77,8 @@ The default options look like this:: ensure_ascii: False indent: 4 separators: [',' , ': '] - sort_keys: True + sort_keys: true csv: formatting: delimiter: ',' - dialect: 'excel' + dialect: excel From 229177857565735dc2dfed471dff881765eca42e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 16 Oct 2019 14:29:32 -0400 Subject: [PATCH 301/613] Docs simplifications for #3400 --- docs/plugins/export.rst | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 6be20bec7..f3756718c 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -45,28 +45,24 @@ Configuration ------------- To configure the plugin, make a ``export:`` section in your configuration -file. Under the ``json``, ``csv``, and ``xml`` keys, these options are available: +file. +For JSON export, these options are available under the ``json`` key: -- **JSON Formatting** - - **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. +- **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. +- **indent**: The number of spaces for indentation. +- **separators**: A ``[item_separator, dict_separator]`` tuple. +- **sort_keys**: Sorts the keys in JSON dictionaries. - - **indent**: The number of spaces for indentation. +Those options match the options from the `Python json module`_. +Similarly, these options are available for the CSV format under the ``csv`` +key: - - **separators**: A ``[item_separator, dict_separator]`` tuple. - - - **sort_keys**: Sorts the keys in JSON dictionaries. - -These options match the options from the `Python json module`_. - -.. _Python json module: https://docs.python.org/2/library/json.html#basic-usage - -- **CSV Formatting** - - **delimiter**: Used as the separating character between fields. The default value is a comma (,). - - - **dialect**: The kind of CSV file to produce. The default is `excel`. +- **delimiter**: Used as the separating character between fields. The default value is a comma (,). +- **dialect**: The kind of CSV file to produce. The default is `excel`. These options match the options from the `Python csv module`_. +.. _Python json module: https://docs.python.org/2/library/json.html#basic-usage .. _Python csv module: https://docs.python.org/3/library/csv.html#csv-fmt-params The default options look like this:: @@ -74,7 +70,7 @@ The default options look like this:: export: json: formatting: - ensure_ascii: False + ensure_ascii: false indent: 4 separators: [',' , ': '] sort_keys: true From 430eab2cf0a974e74f89ddceecd8f1006463487b Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Wed, 16 Oct 2019 21:52:39 +0100 Subject: [PATCH 302/613] Switch to using check_call for hooks --- beetsplug/hook.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/hook.py b/beetsplug/hook.py index ac0c4acad..ff3968a6a 100644 --- a/beetsplug/hook.py +++ b/beetsplug/hook.py @@ -105,7 +105,10 @@ class HookPlugin(BeetsPlugin): u' '.join(command_pieces), event) try: - subprocess.Popen(command_pieces).wait() + subprocess.check_call(command_pieces) + except subprocess.CalledProcessError as exc: + self._log.error(u'hook for {0} exited with status {1}', + event, exc.returncode) except OSError as exc: self._log.error(u'hook for {0} failed: {1}', event, exc) From cf7a04ea3624f18c3567b022efb892e4753e4d79 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Wed, 16 Oct 2019 21:55:46 +0100 Subject: [PATCH 303/613] Add tests for hook errors --- test/test_hook.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/test/test_hook.py b/test/test_hook.py index 81363c73c..2a48a72b1 100644 --- a/test/test_hook.py +++ b/test/test_hook.py @@ -20,7 +20,7 @@ import tempfile import unittest from test import _common -from test.helper import TestHelper +from test.helper import TestHelper, capture_log from beets import config from beets import plugins @@ -37,7 +37,7 @@ class HookTest(_common.TestCase, TestHelper): TEST_HOOK_COUNT = 5 def setUp(self): - self.setup_beets() # Converter is threaded + self.setup_beets() def tearDown(self): self.unload_plugins() @@ -54,6 +54,38 @@ class HookTest(_common.TestCase, TestHelper): config['hook']['hooks'] = hooks + def test_hook_empty_command(self): + self._add_hook('test_event', '') + + self.load_plugins('hook') + + with capture_log('beets.hook') as logs: + plugins.send('test_event') + + self.assertIn('hook: invalid command ""', logs) + + def test_hook_non_zero_exit(self): + self._add_hook('test_event', 'sh -c "exit 1"') + + self.load_plugins('hook') + + with capture_log('beets.hook') as logs: + plugins.send('test_event') + + self.assertIn('hook: hook for test_event exited with status 1', logs) + + def test_hook_non_existent_command(self): + self._add_hook('test_event', 'non-existent-command') + + self.load_plugins('hook') + + with capture_log('beets.hook') as logs: + plugins.send('test_event') + + self.assertTrue(any( + message.startswith("hook: hook for test_event failed: ") + for message in logs)) + def test_hook_no_arguments(self): temporary_paths = [ get_temporary_path() for i in range(self.TEST_HOOK_COUNT) From c5dab1859197902abb1e09bbd583520a927900c5 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Thu, 17 Oct 2019 13:21:26 +0100 Subject: [PATCH 304/613] Document hook error handling changes --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ab752b264..9d451b899 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -110,6 +110,8 @@ Changes: * :doc:`/plugins/export` now also exports ``path`` field if user explicitly specifies it with ``-i`` parameter. Only works when exporting library fields. :bug:`3084` +* :doc:`/plugins/hook` now treats non-zero exit codes as errors. + :bug:`3409` .. _Groups: https://groups.google.com/forum/#!searchin/beets-users/mbsync|sort:date/beets-users/iwCF6bNdh9A/i1xl4Gx8BQAJ From 1f665b02903bc647ae5f4a1edce67cec9f5029d6 Mon Sep 17 00:00:00 2001 From: msil Date: Thu, 24 Oct 2019 10:13:37 +0000 Subject: [PATCH 305/613] adding `discogs_labelid` and `discogs_artistid` fields --- beets/autotag/__init__.py | 2 ++ beets/autotag/hooks.py | 8 ++++++-- beets/library.py | 6 ++++++ beetsplug/discogs.py | 6 ++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index ee1bf051c..f9e38413e 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -170,6 +170,8 @@ def apply_metadata(album_info, mapping): 'style', 'genre', 'discogs_albumid', + 'discogs_artistid', + 'discogs_labelid', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index c0f0ace7b..030f371ba 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -84,7 +84,8 @@ class AlbumInfo(object): releasegroupdisambig=None, artist_credit=None, original_year=None, original_month=None, original_day=None, data_source=None, data_url=None, - discogs_albumid=None): + discogs_albumid=None, discogs_labelid=None, + discogs_artistid=None): self.album = album self.album_id = album_id self.artist = artist @@ -117,6 +118,8 @@ class AlbumInfo(object): self.data_source = data_source self.data_url = data_url self.discogs_albumid = discogs_albumid + self.discogs_labelid = discogs_labelid + self.discogs_artistid = discogs_artistid # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -129,7 +132,8 @@ class AlbumInfo(object): 'catalognum', 'script', 'language', 'country', 'style', 'genre', 'albumstatus', 'albumdisambig', 'releasegroupdisambig', 'artist_credit', - 'media', 'discogs_albumid']: + 'media', 'discogs_albumid', 'discogs_labelid', + 'discogs_artistid']: value = getattr(self, fld) if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) diff --git a/beets/library.py b/beets/library.py index 59791959d..5d90ae43b 100644 --- a/beets/library.py +++ b/beets/library.py @@ -450,6 +450,8 @@ class Item(LibModel): 'genre': types.STRING, 'style': types.STRING, 'discogs_albumid': types.INTEGER, + 'discogs_artistid': types.INTEGER, + 'discogs_labelid': types.INTEGER, 'lyricist': types.STRING, 'composer': types.STRING, 'composer_sort': types.STRING, @@ -934,6 +936,8 @@ class Album(LibModel): 'genre': types.STRING, 'style': types.STRING, 'discogs_albumid': types.INTEGER, + 'discogs_artistid': types.INTEGER, + 'discogs_labelid': types.INTEGER, 'year': types.PaddedInt(4), 'month': types.PaddedInt(2), 'day': types.PaddedInt(2), @@ -981,6 +985,8 @@ class Album(LibModel): 'genre', 'style', 'discogs_albumid', + 'discogs_artistid', + 'discogs_labelid', 'year', 'month', 'day', diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index bccf1f7e2..1437c970e 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -325,7 +325,7 @@ class DiscogsPlugin(BeetsPlugin): # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. - albumtype = media = label = catalogno = None + albumtype = media = label = catalogno = labelid = None if result.data.get('formats'): albumtype = ', '.join( result.data['formats'][0].get('descriptions', [])) or None @@ -333,6 +333,7 @@ class DiscogsPlugin(BeetsPlugin): if result.data.get('labels'): label = result.data['labels'][0].get('name') catalogno = result.data['labels'][0].get('catno') + labelid = result.data['labels'][0].get('id') # Additional cleanups (various artists name, catalog number, media). if va: @@ -365,7 +366,8 @@ class DiscogsPlugin(BeetsPlugin): original_year=original_year, original_month=None, original_day=None, data_source='Discogs', data_url=data_url, - discogs_albumid=discogs_albumid) + discogs_albumid=discogs_albumid, + discogs_labelid=labelid, discogs_artistid=artist_id) def format(self, classification): if classification: From 700b9a64ee2f267fd2cedc5fb285d0822d59a0b3 Mon Sep 17 00:00:00 2001 From: msil Date: Thu, 24 Oct 2019 15:24:49 +0100 Subject: [PATCH 306/613] update changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 15958a0ea..169da71ba 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog New features: +* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and + `discogs_artistid` + :bug: `3413` * :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; which allows for the ability to export in json, csv and xml. Thanks to :user:`austinmm`. From 25e55a88a3acf2c6324aac28395b7d5782fa5219 Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:23:18 -0400 Subject: [PATCH 307/613] Clarify getitem docstring. Log when falling back. I figure, let someone change the docstring again if the function gets a special case besides artist / album artist. --- beets/library.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 5d90ae43b..d5fb5fa75 100644 --- a/beets/library.py +++ b/beets/library.py @@ -410,7 +410,8 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): raise KeyError(key) def __getitem__(self, key): - """Get the value for a key. Certain unset values are remapped. + """Get the value for a key. `artist` and `albumartist` + are fallback values for each other when unmapped. """ value = self._get(key) @@ -418,8 +419,10 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: + log.debug('No artist, using album artist {0}'.format(value)) return self._get('albumartist') elif key == 'albumartist' and not value: + log.debug('No albumartist, using artist {0}'.format(value)) return self._get('artist') else: return value From 59dec3f4da17d107fc15a7108e02592cf09df8ba Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:29:32 -0400 Subject: [PATCH 308/613] Log the new value --- beets/library.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/beets/library.py b/beets/library.py index d5fb5fa75..257197149 100644 --- a/beets/library.py +++ b/beets/library.py @@ -419,11 +419,13 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - log.debug('No artist, using album artist {0}'.format(value)) - return self._get('albumartist') + new_value = self._get('albumartist') + log.debug('No artist, using album artist {0}'.format(new_value)) + return new_value elif key == 'albumartist' and not value: - log.debug('No albumartist, using artist {0}'.format(value)) - return self._get('artist') + new_value = self._get('artist') + log.debug('No albumartist, using artist {0}'.format(new_value)) + return new_value else: return value From 918d833e35c6230e74465773ab75399811af8d5a Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:30:25 -0400 Subject: [PATCH 309/613] fix(?) tabs --- beets/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index 257197149..d9afac677 100644 --- a/beets/library.py +++ b/beets/library.py @@ -419,11 +419,11 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - new_value = self._get('albumartist') + new_value = self._get('albumartist') log.debug('No artist, using album artist {0}'.format(new_value)) return new_value elif key == 'albumartist' and not value: - new_value = self._get('artist') + new_value = self._get('artist') log.debug('No albumartist, using artist {0}'.format(new_value)) return new_value else: From 73ede2642db21b5a3f3e929cc2d6901f580605d6 Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:33:21 -0400 Subject: [PATCH 310/613] Better docstring; fix tabs again --- beets/library.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/library.py b/beets/library.py index d9afac677..3ad875991 100644 --- a/beets/library.py +++ b/beets/library.py @@ -411,7 +411,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def __getitem__(self, key): """Get the value for a key. `artist` and `albumartist` - are fallback values for each other when unmapped. + are fallback values for each other when not set. """ value = self._get(key) @@ -419,11 +419,11 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - new_value = self._get('albumartist') + new_value = self._get('albumartist') log.debug('No artist, using album artist {0}'.format(new_value)) return new_value elif key == 'albumartist' and not value: - new_value = self._get('artist') + new_value = self._get('artist') log.debug('No albumartist, using artist {0}'.format(new_value)) return new_value else: From b489f94b0dc76bef2815bd93d9111e935a000676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sat, 2 Nov 2019 23:06:17 +0100 Subject: [PATCH 311/613] Replace the clinical "obsessive-compulsive" term This is a follow-up on that conversation on gitter: https://gitter.im/beetbox/beets?at=5d75ac34b3e2fc579379fe25 --- README.rst | 3 +-- docs/index.rst | 2 +- test/rsrc/lyrics/examplecom/beetssong.txt | 2 +- test/rsrc/lyricstext.yaml | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index f9be39c52..0f653ac02 100644 --- a/README.rst +++ b/README.rst @@ -14,8 +14,7 @@ beets ===== -Beets is the media library management system for obsessive-compulsive music -geeks. +Beets is the media library management system for obsessive music geeks. The purpose of beets is to get your music collection right once and for all. It catalogs your collection, automatically improving its metadata as it goes. diff --git a/docs/index.rst b/docs/index.rst index 4919147ce..62c87461b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ beets: the music geek's media organizer ======================================= Welcome to the documentation for `beets`_, the media library management system -for obsessive-compulsive music geeks. +for obsessive music geeks. If you're new to beets, begin with the :doc:`guides/main` guide. That guide walks you through installing beets, setting it up how you like it, and starting diff --git a/test/rsrc/lyrics/examplecom/beetssong.txt b/test/rsrc/lyrics/examplecom/beetssong.txt index 3bba9f702..c546dd602 100644 --- a/test/rsrc/lyrics/examplecom/beetssong.txt +++ b/test/rsrc/lyrics/examplecom/beetssong.txt @@ -220,7 +220,7 @@ e9.size = "120x600, 160x600";

John Doe
beets song lyrics

-Ringtones left icon Send "beets song" Ringtone to your Cell Ringtones right icon

Beets is the media library management system for obsessive-compulsive music geeks.
+Ringtones left icon Send "beets song" Ringtone to your Cell Ringtones right icon

Beets is the media library management system for obsessive music geeks.
The purpose of beets is to get your music collection right once and for all. It catalogs your collection, automatically improving its metadata as it goes. It then provides a bouquet of tools for manipulating and accessing your music.
Here's an example of beets' brainy tag corrector doing its thing: diff --git a/test/rsrc/lyricstext.yaml b/test/rsrc/lyricstext.yaml index 7ae1a70e7..af6b09877 100644 --- a/test/rsrc/lyricstext.yaml +++ b/test/rsrc/lyricstext.yaml @@ -1,7 +1,7 @@ # Song used by LyricsGooglePluginMachineryTest Beets_song: | - beets is the media library management system for obsessive-compulsive music geeks the purpose of + beets is the media library management system for obsessive music geeks the purpose of beets is to get your music collection right once and for all it catalogs your collection automatically improving its metadata as it goes it then provides a bouquet of tools for manipulating and accessing your music here's an example of beets' brainy tag corrector doing its From 41cfe8bbb337ecbdb8e956fa2f5666c0656d36fe Mon Sep 17 00:00:00 2001 From: regagain Date: Sun, 3 Nov 2019 19:07:10 +0100 Subject: [PATCH 312/613] Fix wrong section name (smartplaylist => playlist) See https://discourse.beets.io/t/playlist-vs-smartplaylist-plugins/975/2 --- docs/plugins/playlist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index 512c782cf..f65e2e7a5 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -31,7 +31,7 @@ configuration option. Configuration ------------- -To configure the plugin, make a ``smartplaylist:`` section in your +To configure the plugin, make a ``playlist:`` section in your configuration file. In addition to the ``playlists`` described above, the other configuration options are: From 0191794285422eb4173421c04065f704f14a2b34 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 10 Nov 2019 21:59:58 +0100 Subject: [PATCH 313/613] subsonicplaylist plugin --- beetsplug/subsonicplaylist.py | 126 ++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + docs/plugins/index.rst | 1 + docs/plugins/subsonicplaylist.rst | 24 ++++++ 4 files changed, 152 insertions(+) create mode 100644 beetsplug/subsonicplaylist.py create mode 100644 docs/plugins/subsonicplaylist.rst diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py new file mode 100644 index 000000000..5b7ed95bc --- /dev/null +++ b/beetsplug/subsonicplaylist.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Joris Jensen +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import absolute_import, division, print_function + +import os +from hashlib import md5 +import xml.etree.ElementTree as ET +import random +import string +import requests +from beets.util import normpath, bytestring_path, mkdirall, syspath, \ + path_as_posix, sanitize_path + +from beets.ui import Subcommand + +from beets.plugins import BeetsPlugin + +__author__ = 'https://github.com/MrNuggelz' + + +class SubsonicPlaylistPlugin(BeetsPlugin): + + def __init__(self): + super(SubsonicPlaylistPlugin, self).__init__() + self.config.add( + { + 'relative_to': None, + 'playlist_dir': '.', + 'forward_slash': False, + 'playlist_ids': [], + 'playlist_names': [], + 'username': '', + 'password': '' + } + ) + self.config['password'].redact = True + + def create_playlist(self, xml, lib): + relative_to = self.config['relative_to'].get() + if relative_to: + relative_to = normpath(relative_to) + + playlist = ET.fromstring(xml)[0] + if playlist.attrib.get('code', '200') != '200': + alt_error = 'error getting playlist, but no error message found' + self._log.warn(playlist.attrib.get('message', alt_error)) + return + name = '{}.m3u'.format() + tracks = [(t.attrib['artist'], t.attrib['album'], t.attrib['title']) + for t in playlist] + track_paths = [] + for t in tracks: + query = 'artist:"{}" album:"{}" title:"{}"'.format(*t) + items = lib.items(query) + if len(items) > 0: + item_path = items[0].path + if relative_to: + item_path = os.path.relpath(items[0].path, relative_to) + track_paths.append(item_path) + else: + self._log.warn(u"{} | track not found ({})", name, query) + # write playlist + playlist_dir = self.config['playlist_dir'].as_filename() + playlist_dir = bytestring_path(playlist_dir) + m3u_path = normpath(os.path.join(playlist_dir, bytestring_path(name))) + mkdirall(m3u_path) + with open(syspath(m3u_path), 'wb') as f: + for path in track_paths: + if self.config['forward_slash'].get(): + path = path_as_posix(path) + f.write(path + b'\n') + + def get_playlist_by_id(self, playlist_id, lib): + xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text + self.create_playlist(xml, lib) + + def commands(self): + def build_playlist(lib, opts, args): + + if len(self.config['playlist_ids'].as_str_seq()) > 0: + for playlist_id in self.config['playlist_ids'].as_str_seq(): + self.get_playlist_by_id(playlist_id, lib) + if len(self.config['playlist_names'].as_str_seq()) > 0: + playlists = ET.fromstring(self.send('getPlaylists').text)[0] + if playlists.attrib.get('code', '200') != '200': + alt_error = 'error getting playlists,' \ + ' but no erro message found' + self._log.warn(playlists.attrib.get('message', alt_error)) + return + for name in self.config['playlist_names'].as_str_seq(): + for playlist in playlists: + if name == playlist.attrib['name']: + self.get_playlist_by_id(playlist.attrib['id'], lib) + + subsonicplaylist_cmds = Subcommand( + 'subsonicplaylist', help=u'import a subsonic playlist' + ) + subsonicplaylist_cmds.func = build_playlist + return [subsonicplaylist_cmds] + + def generate_token(self): + salt = ''.join(random.choices(string.ascii_lowercase + string.digits)) + return md5( + (self.config['password'].get() + salt).encode()).hexdigest(), salt + + def send(self, endpoint, params=''): + url = '{}/rest/{}?u={}&t={}&s={}&v=1.12.0&c=beets'.format( + self.config['base_url'].get(), + endpoint, + self.config['username'], + *self.generate_token()) + resp = requests.get(url + params) + return resp diff --git a/docs/changelog.rst b/docs/changelog.rst index 169da71ba..9a8d1ae5b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. * :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and `discogs_artistid` :bug: `3413` diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 066d6ec22..9cf2cc5a9 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -115,6 +115,7 @@ following to your configuration:: smartplaylist sonosupdate spotify + subsonicplaylist subsonicupdate the thumbnails diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst new file mode 100644 index 000000000..d6d64f60b --- /dev/null +++ b/docs/plugins/subsonicplaylist.rst @@ -0,0 +1,24 @@ +Subsonic Playlist Plugin +======================== + +The ``subsonicplaylist`` plugin allows to import playlist from a subsonic server + +Command Line Usage +------------------ + +To use the ``subsonicplaylist`` plugin, enable it in your configuration (see +:ref:`using-plugins`). Then use it by invoking the ``subsonicplaylist`` command. +The command will search the playlist on the subsonic server and create a playlist +using the beets library. Options to be defined in your config with their default value:: + + subsonicplaylist: + base_url: "https://your.subsonic.server" + 'relative_to': None, + 'playlist_dir': '.', + 'forward_slash': False, + 'playlist_ids': [], + 'playlist_names': [], + 'username': '', + 'password': '' + +Parameters `base_url`, `username` and `password` must be defined! \ No newline at end of file From ec696f5e58aa83240b578642fad9714455a3828e Mon Sep 17 00:00:00 2001 From: Joris Date: Mon, 11 Nov 2019 10:48:29 +0100 Subject: [PATCH 314/613] Removed unused sanitize_path import --- beetsplug/subsonicplaylist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 5b7ed95bc..9d3ee7cf6 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -22,7 +22,7 @@ import random import string import requests from beets.util import normpath, bytestring_path, mkdirall, syspath, \ - path_as_posix, sanitize_path + path_as_posix from beets.ui import Subcommand From 33c0755aed6ee2d21089f0586e6d361c040fa432 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 12 Nov 2019 08:07:37 +1100 Subject: [PATCH 315/613] Fix simple typo: speicifying -> specifying --- docs/plugins/playlist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/playlist.rst b/docs/plugins/playlist.rst index f65e2e7a5..81fc60beb 100644 --- a/docs/plugins/playlist.rst +++ b/docs/plugins/playlist.rst @@ -13,7 +13,7 @@ Then configure your playlists like this:: playlist_dir: ~/.mpd/playlists forward_slash: no -It is possible to query the library based on a playlist by speicifying its +It is possible to query the library based on a playlist by specifying its absolute path:: $ beet ls playlist:/path/to/someplaylist.m3u From bb305b17e14ee36bc46594418913d43c6128e02b Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 16 Nov 2019 14:36:48 +0100 Subject: [PATCH 316/613] set tags instead of creating m3u --- beetsplug/subsonicplaylist.py | 74 +++++++++++++++-------------------- 1 file changed, 31 insertions(+), 43 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 9d3ee7cf6..1dd73f939 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -15,14 +15,11 @@ from __future__ import absolute_import, division, print_function -import os from hashlib import md5 import xml.etree.ElementTree as ET import random import string import requests -from beets.util import normpath, bytestring_path, mkdirall, syspath, \ - path_as_posix from beets.ui import Subcommand @@ -37,9 +34,6 @@ class SubsonicPlaylistPlugin(BeetsPlugin): super(SubsonicPlaylistPlugin, self).__init__() self.config.add( { - 'relative_to': None, - 'playlist_dir': '.', - 'forward_slash': False, 'playlist_ids': [], 'playlist_names': [], 'username': '', @@ -48,62 +42,56 @@ class SubsonicPlaylistPlugin(BeetsPlugin): ) self.config['password'].redact = True - def create_playlist(self, xml, lib): - relative_to = self.config['relative_to'].get() - if relative_to: - relative_to = normpath(relative_to) + def update_tags(self, playlist_dict, lib): + for query, playlist_tag in playlist_dict.items(): + query = 'artist:"{}" album:"{}" title:"{}"'.format(*query) + items = lib.items(query) + if len(items) <= 0: + self._log.warn(u"{} | track not found ({})", playlist_tag, + query) + continue + for item in items: + item.update({'subsonic_playlist': playlist_tag}) + with lib.transaction(): + item.try_sync(write=True, move=False) + def get_playlist(self, playlist_id): + xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text playlist = ET.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' self._log.warn(playlist.attrib.get('message', alt_error)) return - name = '{}.m3u'.format() + + name = playlist.attrib.get('name', 'undefined') tracks = [(t.attrib['artist'], t.attrib['album'], t.attrib['title']) for t in playlist] - track_paths = [] - for t in tracks: - query = 'artist:"{}" album:"{}" title:"{}"'.format(*t) - items = lib.items(query) - if len(items) > 0: - item_path = items[0].path - if relative_to: - item_path = os.path.relpath(items[0].path, relative_to) - track_paths.append(item_path) - else: - self._log.warn(u"{} | track not found ({})", name, query) - # write playlist - playlist_dir = self.config['playlist_dir'].as_filename() - playlist_dir = bytestring_path(playlist_dir) - m3u_path = normpath(os.path.join(playlist_dir, bytestring_path(name))) - mkdirall(m3u_path) - with open(syspath(m3u_path), 'wb') as f: - for path in track_paths: - if self.config['forward_slash'].get(): - path = path_as_posix(path) - f.write(path + b'\n') - - def get_playlist_by_id(self, playlist_id, lib): - xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text - self.create_playlist(xml, lib) + return name, tracks def commands(self): def build_playlist(lib, opts, args): - - if len(self.config['playlist_ids'].as_str_seq()) > 0: - for playlist_id in self.config['playlist_ids'].as_str_seq(): - self.get_playlist_by_id(playlist_id, lib) + ids = self.config['playlist_ids'].as_str_seq() if len(self.config['playlist_names'].as_str_seq()) > 0: - playlists = ET.fromstring(self.send('getPlaylists').text)[0] + playlists = ET.fromstring(self.send('getPlaylists').text)[ + 0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ ' but no erro message found' - self._log.warn(playlists.attrib.get('message', alt_error)) + self._log.warn( + playlists.attrib.get('message', alt_error)) return for name in self.config['playlist_names'].as_str_seq(): for playlist in playlists: if name == playlist.attrib['name']: - self.get_playlist_by_id(playlist.attrib['id'], lib) + ids.append(playlist.attrib['id']) + playlist_dict = dict() + for playlist_id in ids: + name, tracks = self.get_playlist(playlist_id) + for track in tracks: + if track not in playlist_dict: + playlist_dict[track] = ';' + playlist_dict[track] += name + ';' + self.update_tags(playlist_dict, lib) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' From e9dee5dca83a02c9985d3fff61f18d9a83aea8c6 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 16 Nov 2019 20:30:53 +0100 Subject: [PATCH 317/613] added flag to delete old tags --- beetsplug/subsonicplaylist.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 1dd73f939..ef100c6da 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -34,6 +34,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): super(SubsonicPlaylistPlugin, self).__init__() self.config.add( { + 'delete': False, 'playlist_ids': [], 'playlist_names': [], 'username': '', @@ -70,6 +71,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def commands(self): def build_playlist(lib, opts, args): + self._parse_opts(opts) ids = self.config['playlist_ids'].as_str_seq() if len(self.config['playlist_names'].as_str_seq()) > 0: playlists = ET.fromstring(self.send('getPlaylists').text)[ @@ -91,14 +93,35 @@ class SubsonicPlaylistPlugin(BeetsPlugin): if track not in playlist_dict: playlist_dict[track] = ';' playlist_dict[track] += name + ';' + # delete old tags + if self.config['delete']: + for item in lib.items('subsonic_playlist:";"'): + item.update({'subsonic_playlist': ''}) + with lib.transaction(): + item.try_sync(write=True, move=False) + self.update_tags(playlist_dict, lib) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' ) + subsonicplaylist_cmds.parser.add_option( + u'-d', + u'--delete', + action='store_true', + help=u'delete tag from items not in any playlist anymore', + ) subsonicplaylist_cmds.func = build_playlist return [subsonicplaylist_cmds] + def _parse_opts(self, opts): + + if opts.delete: + self.config['delete'].set(True) + + self.opts = opts + return True + def generate_token(self): salt = ''.join(random.choices(string.ascii_lowercase + string.digits)) return md5( From 00330da623be15eb7fd8f4ad19be65d7222dd945 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 16 Nov 2019 21:10:41 +0100 Subject: [PATCH 318/613] updated documentation --- docs/plugins/subsonicplaylist.rst | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index d6d64f60b..22fe16816 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -1,21 +1,30 @@ Subsonic Playlist Plugin ======================== -The ``subsonicplaylist`` plugin allows to import playlist from a subsonic server +The ``subsonicplaylist`` plugin allows to import playlists from a subsonic server. +This is done by retrieving the track infos from the subsonic server, searching +them in the beets library and adding the playlist names to the +`subsonic_playlist` tag of the found items. The content of the tag has the format: + + subsonic_playlist: ";first playlist;second playlist" + +To get all items in a playlist use the query `;playlist name;`. Command Line Usage ------------------ To use the ``subsonicplaylist`` plugin, enable it in your configuration (see :ref:`using-plugins`). Then use it by invoking the ``subsonicplaylist`` command. -The command will search the playlist on the subsonic server and create a playlist -using the beets library. Options to be defined in your config with their default value:: +By default only the tags of the items found for playlists will be updated. +This means that, if one imported a playlist, then delete one song from it and +imported the playlist again, the deleted song will still have the playlist set +in its `subsonic_playlist` tag. To solve this problem one can use the `-d/--delete` +flag. This resets all `subsonic_playlist` tag before importing playlists. +Options to be defined in your config with their default value:: subsonicplaylist: - base_url: "https://your.subsonic.server" - 'relative_to': None, - 'playlist_dir': '.', - 'forward_slash': False, + 'base_url': "https://your.subsonic.server" + 'delete': False, 'playlist_ids': [], 'playlist_names': [], 'username': '', From 9c479659b24d88ae1048f1bbe70260f0d1b35624 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Sun, 24 Nov 2019 07:20:43 -0800 Subject: [PATCH 319/613] Increment playlist_version when a track is consumed. This fixes the playlist not updating when in consume mode, at least in ncmpcpp. --- beetsplug/bpd/__init__.py | 2 ++ docs/changelog.rst | 1 + 2 files changed, 3 insertions(+) diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 045bce035..dad864b8b 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -639,6 +639,8 @@ class BaseServer(object): self.playlist.pop(old_index) if self.current_index > old_index: self.current_index -= 1 + self.playlist_version += 1 + self._send_event("playlist") if self.current_index >= len(self.playlist): # Fallen off the end. Move to stopped state or loop. if self.repeat: diff --git a/docs/changelog.rst b/docs/changelog.rst index 169da71ba..44337e9b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -131,6 +131,7 @@ Fixes: wiping out their beets database. Thanks to :user:`logan-arens`. :bug:`1934` +* :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. For plugin developers: From 60bba370c0ecc58b4057d78867273e5936d22dee Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 25 Nov 2019 08:45:30 -0500 Subject: [PATCH 320/613] Credit & bug link for #3437 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 44337e9b6..c6b805b30 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -132,6 +132,8 @@ Fixes: Thanks to :user:`logan-arens`. :bug:`1934` * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. + Thanks to :user:`aereaux`. + :bug:`3437` For plugin developers: From d43cf35ad233739bf397dfdc484cfedb779fe90a Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Thu, 5 Dec 2019 20:06:46 -0700 Subject: [PATCH 321/613] Strip and lowercase Genius lyrics artist comparison --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 16699d9d3..1345018f0 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -395,7 +395,7 @@ class Genius(Backend): song_info = None for hit in json["response"]["hits"]: - if hit["result"]["primary_artist"]["name"] == artist: + if hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() == artist.lower(): song_info = hit break From c8e8e587f8ffd8319e941a3e201dfd6a516d368e Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Thu, 5 Dec 2019 20:06:49 -0700 Subject: [PATCH 322/613] Add debug logger for Genius lyrics no-match --- beetsplug/lyrics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 1345018f0..b10f8dd02 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -402,6 +402,8 @@ class Genius(Backend): if song_info: song_api_path = song_info["result"]["api_path"] return self.lyrics_from_song_api_path(song_api_path) + else: + self._log.debug(u'Genius did not return a matching artist entry') class LyricsWiki(SymbolsReplaced): From 1690c08e77092257659dda0bfe4124fc47d3f795 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 5 Dec 2019 22:19:09 -0500 Subject: [PATCH 323/613] Added URL to config and deprecated old configuration --- beetsplug/subsonicupdate.py | 107 +++++++++++++++++++++----------- docs/plugins/subsonicupdate.rst | 14 +++++ 2 files changed, 84 insertions(+), 37 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index bb9e8a952..9cb8758d9 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -25,16 +25,68 @@ a "subsonic" section like the following: """ from __future__ import division, absolute_import, print_function -from beets.plugins import BeetsPlugin -from beets import config -import requests -import string import hashlib import random +import string + +import requests + +from beets import config +from beets.plugins import BeetsPlugin __author__ = 'https://github.com/maffo999' +def build_payload(): + """ To avoid sending plaintext passwords, authentication will be + performed via username, a token, and a 6 random + letters/numbers sequence. + The token is the concatenation of your password and the 6 random + letters/numbers (the salt) which is hashed with MD5. + """ + + user = config['subsonic']['user'].as_str() + password = config['subsonic']['pass'].as_str() + + # Pick the random sequence and salt the password + r = string.ascii_letters + string.digits + salt = "".join([random.choice(r) for n in range(6)]) + t = password + salt + token = hashlib.md5() + token.update(t.encode('utf-8')) + + # Put together the payload of the request to the server and the URL + return { + 'u': user, + 't': token.hexdigest(), + 's': salt, + 'v': '1.15.0', # Subsonic 6.1 and newer. + 'c': 'beets' + } + + +def formal_url(): + """ Formats URL to send request to Subsonic + DEPRECATED schema, host, port, contextpath; use ${url} + """ + + host = config['subsonic']['host'].as_str() + port = config['subsonic']['port'].get(int) + + context_path = config['subsonic']['contextpath'].as_str() + if context_path == '/': + context_path = '' + + url = config['subsonic']['url'].as_str() + if url and url.endsWith('/'): + url = url[:-1] + + if not url: + url = "http://{}:{}{}".format(host, port, context_path) + + return url + '/rest/startScan' + + class SubsonicUpdate(BeetsPlugin): def __init__(self): super(SubsonicUpdate, self).__init__() @@ -46,42 +98,23 @@ class SubsonicUpdate(BeetsPlugin): 'user': 'admin', 'pass': 'admin', 'contextpath': '/', + 'url': 'http://localhost:4040' }) + config['subsonic']['pass'].redact = True - self.register_listener('import', self.loaded) + self.register_listener('import', self.start_scan) - def loaded(self): - host = config['subsonic']['host'].as_str() - port = config['subsonic']['port'].get(int) - user = config['subsonic']['user'].as_str() - passw = config['subsonic']['pass'].as_str() - contextpath = config['subsonic']['contextpath'].as_str() + def start_scan(self): + url = formal_url() + payload = build_payload() - # To avoid sending plaintext passwords, authentication will be - # performed via username, a token, and a 6 random - # letters/numbers sequence. - # The token is the concatenation of your password and the 6 random - # letters/numbers (the salt) which is hashed with MD5. - - # Pick the random sequence and salt the password - r = string.ascii_letters + string.digits - salt = "".join([random.choice(r) for n in range(6)]) - t = passw + salt - token = hashlib.md5() - token.update(t.encode('utf-8')) - - # Put together the payload of the request to the server and the URL - payload = { - 'u': user, - 't': token.hexdigest(), - 's': salt, - 'v': '1.15.0', # Subsonic 6.1 and newer. - 'c': 'beets' - } - if contextpath == '/': - contextpath = '' - url = "http://{}:{}{}/rest/startScan".format(host, port, contextpath) response = requests.post(url, params=payload) - if response.status_code != 200: - self._log.error(u'Generic error, please try again later.') + if response.status_code == 403: + self._log.error(u'Server authentication failed') + elif response.status_code == 200: + self._log.debug(u'Updating Subsonic') + else: + self._log.error( + u'Generic error, please try again later [Status Code: {}]' + .format(response.status_code)) diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index 2d9331b7c..667bceb3e 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -12,6 +12,12 @@ Then, you'll probably want to configure the specifics of your Subsonic server. You can do that using a ``subsonic:`` section in your ``config.yaml``, which looks like this:: + subsonic: + url: https://mydomain.com:443/subsonic + user: username + pass: password + + # DEPRECATED subsonic: host: X.X.X.X port: 4040 @@ -30,6 +36,14 @@ Configuration The available options under the ``subsonic:`` section are: +- **url**: The Subsonic server resource. Default: ``http://localhost:4040`` + +Example: ``https://mydomain.com:443/subsonic`` + +\* Note: context path is optional + +DEPRECATED: + - **host**: The Subsonic server name/IP. Default: ``localhost`` - **port**: The Subsonic server port. Default: ``4040`` - **user**: The Subsonic user. Default: ``admin`` From 5a38e1b35c98c93f0c952929ab7de4fc7deb7325 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 6 Dec 2019 18:30:52 -0500 Subject: [PATCH 324/613] Refactored token generation and updated comments based on suggestions. Also updated documentation to note the password options. --- beetsplug/subsonicupdate.py | 64 ++++++++++++++++----------------- docs/plugins/subsonicupdate.rst | 22 ++---------- 2 files changed, 35 insertions(+), 51 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 9cb8758d9..24d430839 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -37,51 +37,42 @@ from beets.plugins import BeetsPlugin __author__ = 'https://github.com/maffo999' -def build_payload(): - """ To avoid sending plaintext passwords, authentication will be - performed via username, a token, and a 6 random - letters/numbers sequence. - The token is the concatenation of your password and the 6 random - letters/numbers (the salt) which is hashed with MD5. - """ +def create_token(): + """ Creates salt and token from given password. - user = config['subsonic']['user'].as_str() + :return: The generated salt and hashed token + """ password = config['subsonic']['pass'].as_str() # Pick the random sequence and salt the password r = string.ascii_letters + string.digits - salt = "".join([random.choice(r) for n in range(6)]) - t = password + salt - token = hashlib.md5() - token.update(t.encode('utf-8')) + salt = "".join([random.choice(r) for _ in range(6)]) + salted_password = password + salt + token = hashlib.md5().update(salted_password.encode('utf-8')).hexdigest() # Put together the payload of the request to the server and the URL - return { - 'u': user, - 't': token.hexdigest(), - 's': salt, - 'v': '1.15.0', # Subsonic 6.1 and newer. - 'c': 'beets' - } + return salt, token -def formal_url(): - """ Formats URL to send request to Subsonic - DEPRECATED schema, host, port, contextpath; use ${url} +def format_url(): + """ Get the Subsonic URL to trigger a scan. Uses either the url + config option or the deprecated host, port, and context_path config + options together. + + :return: Endpoint for updating Subsonic """ - host = config['subsonic']['host'].as_str() - port = config['subsonic']['port'].get(int) - - context_path = config['subsonic']['contextpath'].as_str() - if context_path == '/': - context_path = '' - url = config['subsonic']['url'].as_str() if url and url.endsWith('/'): url = url[:-1] + # @deprecated("Use url config option instead") if not url: + host = config['subsonic']['host'].as_str() + port = config['subsonic']['port'].get(int) + context_path = config['subsonic']['contextpath'].as_str() + if context_path == '/': + context_path = '' url = "http://{}:{}{}".format(host, port, context_path) return url + '/rest/startScan' @@ -105,8 +96,17 @@ class SubsonicUpdate(BeetsPlugin): self.register_listener('import', self.start_scan) def start_scan(self): - url = formal_url() - payload = build_payload() + user = config['subsonic']['user'].as_str() + url = format_url() + salt, token = create_token() + + payload = { + 'u': user, + 't': token, + 's': salt, + 'v': '1.15.0', # Subsonic 6.1 and newer. + 'c': 'beets' + } response = requests.post(url, params=payload) @@ -117,4 +117,4 @@ class SubsonicUpdate(BeetsPlugin): else: self._log.error( u'Generic error, please try again later [Status Code: {}]' - .format(response.status_code)) + .format(response.status_code)) diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index 667bceb3e..68496c4e3 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -13,17 +13,11 @@ You can do that using a ``subsonic:`` section in your ``config.yaml``, which looks like this:: subsonic: - url: https://mydomain.com:443/subsonic + url: https://example.com:443/subsonic user: username pass: password - # DEPRECATED - subsonic: - host: X.X.X.X - port: 4040 - user: username - pass: password - contextpath: /subsonic +\* NOTE: The pass config option can either be clear text or hex-encoded with a "enc:" prefix. With that all in place, beets will send a Rest API to your Subsonic server every time you import new music. @@ -38,14 +32,4 @@ The available options under the ``subsonic:`` section are: - **url**: The Subsonic server resource. Default: ``http://localhost:4040`` -Example: ``https://mydomain.com:443/subsonic`` - -\* Note: context path is optional - -DEPRECATED: - -- **host**: The Subsonic server name/IP. Default: ``localhost`` -- **port**: The Subsonic server port. Default: ``4040`` -- **user**: The Subsonic user. Default: ``admin`` -- **pass**: The Subsonic user password. Default: ``admin`` -- **contextpath**: The Subsonic context path. Default: ``/`` +Example: ``https://mydomain.com:443/subsonic`` \ No newline at end of file From e18b91da2673619548daff275787ee2c9b3120ac Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 6 Dec 2019 18:32:35 -0500 Subject: [PATCH 325/613] Remove example --- docs/plugins/subsonicupdate.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index 68496c4e3..5508e103c 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -30,6 +30,4 @@ Configuration The available options under the ``subsonic:`` section are: -- **url**: The Subsonic server resource. Default: ``http://localhost:4040`` - -Example: ``https://mydomain.com:443/subsonic`` \ No newline at end of file +- **url**: The Subsonic server resource. Default: ``http://localhost:4040`` \ No newline at end of file From 89f21d960198884488ead88805a1f09bd1f048cb Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 9 Dec 2019 07:13:25 -0500 Subject: [PATCH 326/613] Updated documentation --- beetsplug/subsonicupdate.py | 6 +++--- docs/plugins/subsonicupdate.rst | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 24d430839..b3d05e245 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -38,7 +38,7 @@ __author__ = 'https://github.com/maffo999' def create_token(): - """ Creates salt and token from given password. + """Creates salt and token from given password. :return: The generated salt and hashed token """ @@ -55,7 +55,7 @@ def create_token(): def format_url(): - """ Get the Subsonic URL to trigger a scan. Uses either the url + """Get the Subsonic URL to trigger a scan. Uses either the url config option or the deprecated host, port, and context_path config options together. @@ -89,7 +89,7 @@ class SubsonicUpdate(BeetsPlugin): 'user': 'admin', 'pass': 'admin', 'contextpath': '/', - 'url': 'http://localhost:4040' + 'url': 'http://localhost:4040', }) config['subsonic']['pass'].redact = True diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index 5508e103c..bdd33593b 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -17,8 +17,6 @@ which looks like this:: user: username pass: password -\* NOTE: The pass config option can either be clear text or hex-encoded with a "enc:" prefix. - With that all in place, beets will send a Rest API to your Subsonic server every time you import new music. Due to a current limitation of the API, all libraries visible to that user will be scanned. @@ -30,4 +28,8 @@ Configuration The available options under the ``subsonic:`` section are: -- **url**: The Subsonic server resource. Default: ``http://localhost:4040`` \ No newline at end of file +- **url**: The Subsonic server resource. Default: ``http://localhost:4040`` +- **user**: The Subsonic user. Default: ``admin`` +- **pass**: The Subsonic user password. Default: ``admin`` + +\* NOTE: The pass config option can either be clear text or hex-encoded with a "enc:" prefix. \ No newline at end of file From 9739e9be9bf023dbde4e9058a90a6404beef109e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 9 Dec 2019 15:55:31 -0500 Subject: [PATCH 327/613] Docs tweaks for #3449 --- beetsplug/subsonicupdate.py | 2 +- docs/plugins/index.rst | 3 +++ docs/plugins/subsonicupdate.rst | 5 ++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index b3d05e245..469a9d142 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -38,7 +38,7 @@ __author__ = 'https://github.com/maffo999' def create_token(): - """Creates salt and token from given password. + """Create salt and token from given password. :return: The generated salt and hashed token """ diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 066d6ec22..7dc152bd1 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -198,12 +198,15 @@ Interoperability * :doc:`sonosupdate`: Automatically notifies `Sonos`_ whenever the beets library changes. * :doc:`thumbnails`: Get thumbnails with the cover art on your album folders. +* :doc:`subsonicupdate`: Automatically notifies `Subsonic`_ whenever the beets + library changes. .. _Emby: https://emby.media .. _Plex: https://plex.tv .. _Kodi: https://kodi.tv .. _Sonos: https://sonos.com +.. _Subsonic: http://www.subsonic.org/ Miscellaneous ------------- diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index bdd33593b..3549be091 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -30,6 +30,5 @@ The available options under the ``subsonic:`` section are: - **url**: The Subsonic server resource. Default: ``http://localhost:4040`` - **user**: The Subsonic user. Default: ``admin`` -- **pass**: The Subsonic user password. Default: ``admin`` - -\* NOTE: The pass config option can either be clear text or hex-encoded with a "enc:" prefix. \ No newline at end of file +- **pass**: The Subsonic user password. (This may either be a clear-text + password or hex-encoded with the prefix ``enc:``.) Default: ``admin`` From 2e24887f539302f44e37dcf0119a8cb1f26a3150 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 9 Dec 2019 15:57:41 -0500 Subject: [PATCH 328/613] Changelog for #3449 --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c6b805b30..23a898e64 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -90,6 +90,12 @@ New features: :bug:`3387` * :doc:`/plugins/hook` now treats non-zero exit codes as errors. :bug:`3409` +* :doc:`/plugins/subsonicupdate`: A new ``url`` configuration replaces the + older (and now deprecated) separate ``host``, ``port``, and ``contextpath`` + config options. As a consequence, the plugin can now talk to Subsonic over + HTTPS. + Thanks to :user:`jef`. + :bug:`3449` Fixes: From bc9e9664fdec77262a9a6047659c8967d6c25a93 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 10 Dec 2019 13:03:53 -0500 Subject: [PATCH 329/613] Drop Python 3.4; add Travis builds for 3.8 --- .travis.yml | 14 +++++++------- docs/changelog.rst | 1 + setup.py | 1 - 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 455ab4ca4..b889f698d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,10 @@ matrix: env: {TOX_ENV: py27-cov, COVERAGE: 1} - python: 2.7.13 env: {TOX_ENV: py27-test} - - python: 3.4 - env: {TOX_ENV: py34-test} - - python: 3.4_with_system_site_packages - env: {TOX_ENV: py34-test} + # - python: 3.4 + # env: {TOX_ENV: py34-test} + # - python: 3.4_with_system_site_packages + # env: {TOX_ENV: py34-test} - python: 3.5 env: {TOX_ENV: py35-test} - python: 3.6 @@ -24,9 +24,9 @@ matrix: - python: 3.7 env: {TOX_ENV: py37-test} dist: xenial - # - python: 3.8-dev - # env: {TOX_ENV: py38-test} - # dist: xenial + - python: 3.8 + env: {TOX_ENV: py38-test} + dist: xenial # - python: pypy # - env: {TOX_ENV: pypy-test} - python: 3.6 diff --git a/docs/changelog.rst b/docs/changelog.rst index 23a898e64..b5b6c3901 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -189,6 +189,7 @@ For packagers: * We attempted to fix an unreliable test, so a patch to `skip `_ or `repair `_ the test may no longer be necessary. +* This version drops support for Python 3.4. .. _MediaFile: https://github.com/beetbox/mediafile .. _Confuse: https://github.com/beetbox/confuse diff --git a/setup.py b/setup.py index cfcffdbf5..544721937 100755 --- a/setup.py +++ b/setup.py @@ -172,7 +172,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', From a1af5d280e56d51f2970997339582eae0305758b Mon Sep 17 00:00:00 2001 From: Tyler Faulk Date: Tue, 10 Dec 2019 22:54:31 -0500 Subject: [PATCH 330/613] added merge_environment_settings call in fetchart plugin to handle connections with proxy servers --- beetsplug/fetchart.py | 6 +++++- docs/changelog.rst | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index a815d4d9b..ca93d685e 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -164,13 +164,17 @@ def _logged_get(log, *args, **kwargs): message = 'getting URL' req = requests.Request('GET', *args, **req_kwargs) + with requests.Session() as s: s.headers = {'User-Agent': 'beets'} prepped = s.prepare_request(req) + settings = s.merge_environment_settings( + prepped.url, {}, None, None, None + ) + send_kwargs.update(settings) log.debug('{}: {}', message, prepped.url) return s.send(prepped, **send_kwargs) - class RequestMixin(object): """Adds a Requests wrapper to the class that uses the logger, which must be named `self._log`. diff --git a/docs/changelog.rst b/docs/changelog.rst index b5b6c3901..e24bb976e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -99,6 +99,9 @@ New features: Fixes: +* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take + environment variables such as proxy servers into account when making requests + :bug:`3450` * :doc:`/plugins/inline`: In function-style field definitions that refer to flexible attributes, values could stick around from one function invocation to the next. This meant that, when displaying a list of objects, later From e760fc6947bbd5125da7aea5346fe9a84362b265 Mon Sep 17 00:00:00 2001 From: Mat Date: Mon, 16 Dec 2019 21:47:13 +0000 Subject: [PATCH 331/613] Wording/grammar --- docs/reference/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index e17d5b42f..b66920d96 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -71,7 +71,7 @@ box. To extract `rar` files, install the `rarfile`_ package and the Optional command flags: -* By default, the command copies files your the library directory and +* By default, the command copies files to your library directory and updates the ID3 tags on your music. In order to move the files, instead of copying, use the ``-m`` (move) option. If you'd like to leave your music files untouched, try the ``-C`` (don't copy) and ``-W`` (don't write tags) From 6889b9ffdc588aae5f1e793aa0844c53257d2d31 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 13:43:53 -0500 Subject: [PATCH 332/613] Add `index_tracks' configuration option --- beetsplug/discogs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 1437c970e..905cc12da 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -57,7 +57,8 @@ class DiscogsPlugin(BeetsPlugin): 'tokenfile': 'discogs_token.json', 'source_weight': 0.5, 'user_token': '', - 'separator': u', ' + 'separator': u', ', + 'index_tracks': False }) self.config['apikey'].redact = True self.config['apisecret'].redact = True From e31695b606b570218dac7b2bf55fd28a2af97544 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 15:27:26 -0500 Subject: [PATCH 333/613] Trace hierarchy of index tracks --- beetsplug/discogs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 905cc12da..eedfde739 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -398,14 +398,23 @@ class DiscogsPlugin(BeetsPlugin): tracks = [] index_tracks = {} index = 0 + divisions, next_divisions = [], [] for track in clean_tracklist: # Only real tracks have `position`. Otherwise, it's an index track. if track['position']: index += 1 + if next_divisions: + divisions += next_divisions + next_divisions.clear() track_info = self.get_track_info(track, index) track_info.track_alt = track['position'] tracks.append(track_info) else: + next_divisions.append(track['title']) + try: + divisions.pop() + except IndexError: + pass index_tracks[index + 1] = track['title'] # Fix up medium and medium_index for each track. Discogs position is From 02e03be93d23a5b3c24a683043daa10d8024fe5d Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 15:38:54 -0500 Subject: [PATCH 334/613] Incorporate divisions into track titles --- beetsplug/discogs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index eedfde739..ea3239c21 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -406,7 +406,7 @@ class DiscogsPlugin(BeetsPlugin): if next_divisions: divisions += next_divisions next_divisions.clear() - track_info = self.get_track_info(track, index) + track_info = self.get_track_info(track, index, divisions) track_info.track_alt = track['position'] tracks.append(track_info) else: @@ -549,10 +549,13 @@ class DiscogsPlugin(BeetsPlugin): return tracklist - def get_track_info(self, track, index): + def get_track_info(self, track, index, divisions): """Returns a TrackInfo object for a discogs track. """ title = track['title'] + if self.config['index_tracks']: + prefix = ', '.join(divisions) + title = ': '.join(prefix, title) track_id = None medium, medium_index, _ = self.get_track_index(track['position']) artist, artist_id = MetadataSourcePlugin.get_artist( From 8805ba28fdc25ad90f29bab4da1f7a72305faa35 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 16:33:41 -0500 Subject: [PATCH 335/613] Add comments --- beetsplug/discogs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index ea3239c21..7e398072b 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -398,12 +398,14 @@ class DiscogsPlugin(BeetsPlugin): tracks = [] index_tracks = {} index = 0 + # Distinct works and intra-work divisions, as defined by index tracks. divisions, next_divisions = [], [] for track in clean_tracklist: # Only real tracks have `position`. Otherwise, it's an index track. if track['position']: index += 1 if next_divisions: + # End of a block of index tracks: update the current divisions. divisions += next_divisions next_divisions.clear() track_info = self.get_track_info(track, index, divisions) @@ -411,6 +413,8 @@ class DiscogsPlugin(BeetsPlugin): tracks.append(track_info) else: next_divisions.append(track['title']) + # We expect new levels of division at the beginning of the tracklist + # (and possibly elsewhere). try: divisions.pop() except IndexError: From 90fb79f40834d219323dcf07284c3177bfae38e6 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 16:54:56 -0500 Subject: [PATCH 336/613] Document `index_tracks' option --- docs/plugins/discogs.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 6c16083ee..80be4c8bb 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -48,6 +48,33 @@ Configuration This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`. +There is one additional option in the ``discogs:`` section, ``index_tracks``. +Index tracks (see the `Discogs guidelines +`_), +along with headers, mark divisions between distinct works on the same release +or within works. When ``index_tracks`` is enabled,:: + + discogs: + index_tracks: yes + +beets will incorporate the names of the divisions containing each track into +the imported track's title. For example, importing +`this album +`_ +would result in track names like:: + + Messiah, Part I: No.1: Sinfony + Messiah, Part II: No.22: Chorus- Behold The Lamb Of God + Athalia, Act I, Scene I: Sinfonia + +whereas with ``index_track`` disabled you'd get:: + + No.1: Sinfony + No.22: Chorus- Behold The Lamb Of God + Sinfonia + +This option is useful when importing classical music. + Troubleshooting --------------- From 67e402bbae1586d035623e818efc895771e27687 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 16:57:24 -0500 Subject: [PATCH 337/613] Fix typo in new documentation --- docs/plugins/discogs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 80be4c8bb..2a7e90439 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -67,7 +67,7 @@ would result in track names like:: Messiah, Part II: No.22: Chorus- Behold The Lamb Of God Athalia, Act I, Scene I: Sinfonia -whereas with ``index_track`` disabled you'd get:: +whereas with ``index_tracks`` disabled you'd get:: No.1: Sinfony No.22: Chorus- Behold The Lamb Of God From 5f74edf2d9e482ea413a43202e219829e7e6acd7 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 17:01:34 -0500 Subject: [PATCH 338/613] One more documentation typo --- docs/plugins/discogs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 2a7e90439..c199ccf49 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -52,7 +52,7 @@ There is one additional option in the ``discogs:`` section, ``index_tracks``. Index tracks (see the `Discogs guidelines `_), along with headers, mark divisions between distinct works on the same release -or within works. When ``index_tracks`` is enabled,:: +or within works. When ``index_tracks`` is enabled:: discogs: index_tracks: yes From 3ccafa24953cde2ec37603ff4fc1a8fff48eb6ee Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 17:40:16 -0500 Subject: [PATCH 339/613] Fix `str.join' usage --- beetsplug/discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 7e398072b..cc2bd67ff 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -559,7 +559,7 @@ class DiscogsPlugin(BeetsPlugin): title = track['title'] if self.config['index_tracks']: prefix = ', '.join(divisions) - title = ': '.join(prefix, title) + title = ': '.join([prefix, title]) track_id = None medium, medium_index, _ = self.get_track_index(track['position']) artist, artist_id = MetadataSourcePlugin.get_artist( From 614289af5f519dfdc8bfb87aaa8d4637b6320762 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 17:46:43 -0500 Subject: [PATCH 340/613] Remove use of `list.clear()' for compatibility --- beetsplug/discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index cc2bd67ff..6171f8f56 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -407,7 +407,7 @@ class DiscogsPlugin(BeetsPlugin): if next_divisions: # End of a block of index tracks: update the current divisions. divisions += next_divisions - next_divisions.clear() + del next_divisions[:] track_info = self.get_track_info(track, index, divisions) track_info.track_alt = track['position'] tracks.append(track_info) From 6c948764fe684848aca5365781bc7771644d6fcf Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 17 Dec 2019 17:56:56 -0500 Subject: [PATCH 341/613] Wrap comment lines --- beetsplug/discogs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 6171f8f56..7abffdf07 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -405,7 +405,8 @@ class DiscogsPlugin(BeetsPlugin): if track['position']: index += 1 if next_divisions: - # End of a block of index tracks: update the current divisions. + # End of a block of index tracks: update the current + # divisions. divisions += next_divisions del next_divisions[:] track_info = self.get_track_info(track, index, divisions) @@ -413,8 +414,8 @@ class DiscogsPlugin(BeetsPlugin): tracks.append(track_info) else: next_divisions.append(track['title']) - # We expect new levels of division at the beginning of the tracklist - # (and possibly elsewhere). + # We expect new levels of division at the beginning of the + # tracklist (and possibly elsewhere). try: divisions.pop() except IndexError: From e945ed894dada7ecef24f4158cf2672aa5c773f3 Mon Sep 17 00:00:00 2001 From: Cole Miller <53574922+cole-miller@users.noreply.github.com> Date: Wed, 18 Dec 2019 14:31:59 -0500 Subject: [PATCH 342/613] Add trailing comma Co-Authored-By: Adrian Sampson --- beetsplug/discogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 7abffdf07..0ba27d7dd 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -58,7 +58,7 @@ class DiscogsPlugin(BeetsPlugin): 'source_weight': 0.5, 'user_token': '', 'separator': u', ', - 'index_tracks': False + 'index_tracks': False, }) self.config['apikey'].redact = True self.config['apisecret'].redact = True From 49d9f474a53dc2afceb071dc03e2e4821b887f34 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Wed, 18 Dec 2019 14:44:57 -0500 Subject: [PATCH 343/613] Update changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b5b6c3901..0608fb06c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -96,6 +96,11 @@ New features: HTTPS. Thanks to :user:`jef`. :bug:`3449` +* :doc:`/plugins/discogs`: The new ``index_tracks`` option enables + incorporation of work names and intra-work divisions into imported track + titles. + Thanks to :user:`cole-miller`. + :bug:`3459` Fixes: From 3570f5cd56425a8c347e0efe1c21f314327ae113 Mon Sep 17 00:00:00 2001 From: Mat Date: Thu, 26 Dec 2019 17:36:56 +0000 Subject: [PATCH 344/613] New high_resolution config option in fetchart --- beetsplug/fetchart.py | 16 ++++++++++++++-- docs/changelog.rst | 6 +++++- docs/plugins/fetchart.rst | 3 +++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index a815d4d9b..9cb564eaf 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -470,6 +470,14 @@ class ITunesStore(RemoteArtSource): NAME = u"iTunes Store" API_URL = u'https://itunes.apple.com/search' + def __init__(self, *args, **kwargs): + super(ITunesStore, self).__init__(*args, **kwargs) + high_resolution = self._config['high_resolution'].get() + if high_resolution: + self.image_suffix = '100000x100000-999' + else: + self.image_suffix = '1200x1200bb' + def get(self, album, plugin, paths): """Return art URL from iTunes Store given an album title. """ @@ -510,7 +518,8 @@ class ITunesStore(RemoteArtSource): if (c['artistName'] == album.albumartist and c['collectionName'] == album.album): art_url = c['artworkUrl100'] - art_url = art_url.replace('100x100', '1200x1200') + art_url = art_url.replace('100x100bb', + self.image_suffix) yield self._candidate(url=art_url, match=Candidate.MATCH_EXACT) except KeyError as e: @@ -520,7 +529,8 @@ class ITunesStore(RemoteArtSource): try: fallback_art_url = candidates[0]['artworkUrl100'] - fallback_art_url = fallback_art_url.replace('100x100', '1200x1200') + fallback_art_url = fallback_art_url.replace('100x100bb', + self.image_suffix) yield self._candidate(url=fallback_art_url, match=Candidate.MATCH_FALLBACK) except KeyError as e: @@ -774,6 +784,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'google_engine': u'001442825323518660753:hrh5ch1gjzm', 'fanarttv_key': None, 'store_source': False, + 'high_resolution': False, }) self.config['google_key'].redact = True self.config['fanarttv_key'].redact = True @@ -802,6 +813,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.cover_names = list(map(util.bytestring_path, cover_names)) self.cautious = self.config['cautious'].get(bool) self.store_source = self.config['store_source'].get(bool) + self.high_resolution = self.config['high_resolution'].get(bool) self.src_removed = (config['import']['delete'].get(bool) or config['import']['move'].get(bool)) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0608fb06c..0017b98a2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,11 @@ Changelog New features: -* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and +* :doc:`plugins/fetchart`: Added new ``high_resolution`` config option to + allow downloading of higher resolution iTunes artwork (at the expense of + file size) + :bug: `3391` +* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and `discogs_artistid` :bug: `3413` * :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index f23fec765..c4ce189c7 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -67,6 +67,9 @@ file. The available options are: - **store_source**: If enabled, fetchart stores the artwork's source in a flexible tag named ``art_source``. See below for the rationale behind this. Default: ``no``. +- **high_resolution**: If enabled, fetchart retrieves artwork in the highest + resolution it can find (Warning: image files can sometimes reach >20MB) + Default: ``no``. Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ or `Pillow`_. From 7ad3f7f72869a34d7b762e8cf7c441407099631e Mon Sep 17 00:00:00 2001 From: Mat Date: Thu, 26 Dec 2019 21:09:26 +0000 Subject: [PATCH 345/613] Apply suggestions from code review Fix wording in docs Co-Authored-By: Adrian Sampson --- docs/changelog.rst | 4 ++-- docs/plugins/fetchart.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0017b98a2..e8a2717c5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,9 +6,9 @@ Changelog New features: -* :doc:`plugins/fetchart`: Added new ``high_resolution`` config option to +* :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to allow downloading of higher resolution iTunes artwork (at the expense of - file size) + file size). :bug: `3391` * :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and `discogs_artistid` diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index c4ce189c7..68212a582 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -68,7 +68,7 @@ file. The available options are: flexible tag named ``art_source``. See below for the rationale behind this. Default: ``no``. - **high_resolution**: If enabled, fetchart retrieves artwork in the highest - resolution it can find (Warning: image files can sometimes reach >20MB) + resolution it can find (warning: image files can sometimes reach >20MB). Default: ``no``. Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ From 2593a5be3405e9e0c9c49027d80e33e08f820a98 Mon Sep 17 00:00:00 2001 From: Mat Date: Thu, 26 Dec 2019 21:55:48 +0000 Subject: [PATCH 346/613] Use a local var to use high resolution option --- beetsplug/fetchart.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 9cb564eaf..fa43025de 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -470,14 +470,6 @@ class ITunesStore(RemoteArtSource): NAME = u"iTunes Store" API_URL = u'https://itunes.apple.com/search' - def __init__(self, *args, **kwargs): - super(ITunesStore, self).__init__(*args, **kwargs) - high_resolution = self._config['high_resolution'].get() - if high_resolution: - self.image_suffix = '100000x100000-999' - else: - self.image_suffix = '1200x1200bb' - def get(self, album, plugin, paths): """Return art URL from iTunes Store given an album title. """ @@ -513,13 +505,18 @@ class ITunesStore(RemoteArtSource): payload['term']) return + if self._config['high_resolution'].get(): + image_suffix = '100000x100000-999' + else: + image_suffix = '1200x1200bb' + for c in candidates: try: if (c['artistName'] == album.albumartist and c['collectionName'] == album.album): art_url = c['artworkUrl100'] art_url = art_url.replace('100x100bb', - self.image_suffix) + image_suffix) yield self._candidate(url=art_url, match=Candidate.MATCH_EXACT) except KeyError as e: @@ -530,7 +527,7 @@ class ITunesStore(RemoteArtSource): try: fallback_art_url = candidates[0]['artworkUrl100'] fallback_art_url = fallback_art_url.replace('100x100bb', - self.image_suffix) + image_suffix) yield self._candidate(url=fallback_art_url, match=Candidate.MATCH_FALLBACK) except KeyError as e: @@ -813,7 +810,6 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.cover_names = list(map(util.bytestring_path, cover_names)) self.cautious = self.config['cautious'].get(bool) self.store_source = self.config['store_source'].get(bool) - self.high_resolution = self.config['high_resolution'].get(bool) self.src_removed = (config['import']['delete'].get(bool) or config['import']['move'].get(bool)) From a08f2315ea0c13c60d46f2947984e835898d083c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 26 Dec 2019 20:44:14 -0500 Subject: [PATCH 347/613] Simplify Confuse usage (#3463) --- beetsplug/fetchart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index fa43025de..6b9fa375e 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -505,7 +505,7 @@ class ITunesStore(RemoteArtSource): payload['term']) return - if self._config['high_resolution'].get(): + if self._config['high_resolution']: image_suffix = '100000x100000-999' else: image_suffix = '1200x1200bb' From 6d69d01016adaa780d660ee3c16199ad2afb96f5 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Mon, 13 Jan 2020 15:42:55 +0100 Subject: [PATCH 348/613] added database changed event to subsonicplaylist --- beetsplug/subsonicplaylist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index ef100c6da..fbbcd8ac3 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -20,6 +20,7 @@ import xml.etree.ElementTree as ET import random import string import requests +from beets import plugins from beets.ui import Subcommand @@ -101,6 +102,8 @@ class SubsonicPlaylistPlugin(BeetsPlugin): item.try_sync(write=True, move=False) self.update_tags(playlist_dict, lib) + ## notify plugins + plugins.send('database_change', lib=lib, model=self) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' From f11831468250c62be6c1897cbe6ad9f99f79ca65 Mon Sep 17 00:00:00 2001 From: BrainDamage Date: Sun, 12 Jan 2020 20:09:13 +0100 Subject: [PATCH 349/613] support for keyfinder-cli, which doesn't want the -f flag for paths --- beetsplug/keyfinder.py | 11 ++++++++--- docs/changelog.rst | 3 +++ docs/plugins/keyfinder.rst | 15 +++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/beetsplug/keyfinder.py b/beetsplug/keyfinder.py index ea928ef43..a75b8d972 100644 --- a/beetsplug/keyfinder.py +++ b/beetsplug/keyfinder.py @@ -18,6 +18,7 @@ from __future__ import division, absolute_import, print_function +import os.path import subprocess from beets import ui @@ -52,15 +53,19 @@ class KeyFinderPlugin(BeetsPlugin): def find_key(self, items, write=False): overwrite = self.config['overwrite'].get(bool) - bin = self.config['bin'].as_str() + command = [self.config['bin'].as_str()] + # The KeyFinder GUI program needs the -f flag before the path. + # keyfinder-cli is similar, but just wants the path with no flag. + if 'keyfinder-cli' not in os.path.basename(command[0]).lower(): + command.append('-f') for item in items: if item['initial_key'] and not overwrite: continue try: - output = util.command_output([bin, '-f', - util.syspath(item.path)]).stdout + output = util.command_output(command + [util.syspath( + item.path)]).stdout except (subprocess.CalledProcessError, OSError) as exc: self._log.error(u'execution failed: {0}', exc) continue diff --git a/docs/changelog.rst b/docs/changelog.rst index e8a2717c5..95fde6888 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog New features: +* :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ + Thanks to :user:`BrainDamage`. * :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to allow downloading of higher resolution iTunes artwork (at the expense of file size). @@ -204,6 +206,7 @@ For packagers: .. _Confuse: https://github.com/beetbox/confuse .. _works: https://musicbrainz.org/doc/Work .. _Deezer: https://www.deezer.com +.. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli 1.4.9 (May 30, 2019) diff --git a/docs/plugins/keyfinder.rst b/docs/plugins/keyfinder.rst index 878830f29..2ed2c1cec 100644 --- a/docs/plugins/keyfinder.rst +++ b/docs/plugins/keyfinder.rst @@ -1,9 +1,9 @@ Key Finder Plugin ================= -The `keyfinder` plugin uses the `KeyFinder`_ program to detect the -musical key of track from its audio data and store it in the -`initial_key` field of your database. It does so +The `keyfinder` plugin uses either the `KeyFinder`_ or `keyfinder-cli`_ +program to detect the musical key of a track from its audio data and store +it in the `initial_key` field of your database. It does so automatically when importing music or through the ``beet keyfinder [QUERY]`` command. @@ -20,13 +20,16 @@ configuration file. The available options are: import. Otherwise, you need to use the ``beet keyfinder`` command explicitly. Default: ``yes`` -- **bin**: The name of the `KeyFinder`_ program on your system or - a path to the binary. If you installed the KeyFinder GUI on a Mac, for - example, you want something like +- **bin**: The name of the program use for key analysis. You can use either + `KeyFinder`_ or `keyfinder-cli`_. + If you installed the KeyFinder GUI on a Mac, for example, you want + something like ``/Applications/KeyFinder.app/Contents/MacOS/KeyFinder``. + If using `keyfinder-cli`_, the binary must be named ``keyfinder-cli``. Default: ``KeyFinder`` (i.e., search for the program in your ``$PATH``).. - **overwrite**: Calculate a key even for files that already have an `initial_key` value. Default: ``no``. .. _KeyFinder: https://www.ibrahimshaath.co.uk/keyfinder/ +.. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli/ From a4a0a4bd28ca72205bfef7062606eb916261ea35 Mon Sep 17 00:00:00 2001 From: ybnd Date: Tue, 28 Jan 2020 09:45:20 +0100 Subject: [PATCH 350/613] Remove --replaygain flag when checking bs1770gain availability bs1770gain exits with error 1 when called without data, interpreted as unavailable --- test/test_replaygain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index fe0515bee..437b1426a 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain', ['--replaygain']): +if has_program('bs1770gain'): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -58,7 +58,6 @@ def reset_replaygain(item): class ReplayGainCliTestBase(TestHelper): - def setUp(self): self.setup_beets() self.config['replaygain']['backend'] = self.backend From 53820c0a9843ce613feeb2d9896739f0d3cbf369 Mon Sep 17 00:00:00 2001 From: ybnd Date: Tue, 28 Jan 2020 09:45:20 +0100 Subject: [PATCH 351/613] Handle bs1770gain v0.6.0 XML output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove `0%\x08\x08` from output (backspace code doesn't resolve; progress percentages get spliced in) * Handle changed attributes/fields: * `sample-peak` attribute `factor` is called `amplitude` instead * Album summary is not included in a `summary` tag now, but in two separate `integrated` and `sample-peak` tags * Handle `lu` attribute * Get bs1770gain version * If v0.6.0 or later, add `--unit=ebu` flag to convert `db` attributes to LUFS * May be useful later on ### Output examples Track: ``` ``` Album: ``` ``` --- beetsplug/replaygain.py | 32 ++++++++++++++++++++++++++++++-- test/test_replaygain.py | 3 +-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 1076ac714..3e284ce62 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -22,6 +22,7 @@ import math import sys import warnings import enum +import re import xml.parsers.expat from six.moves import zip @@ -135,6 +136,11 @@ class Bs1770gainBackend(Backend): -18: "replaygain", } + version = re.search( + 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', + call(['bs1770gain', '--version']).stdout.decode('utf-8') + ).group(1) + def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) config.add({ @@ -252,6 +258,8 @@ class Bs1770gainBackend(Backend): cmd = [self.command] cmd += ["--" + method] cmd += ['--xml', '-p'] + if self.version >= '0.6.0': + cmd += ['--unit=ebu'] # set units to LU # Workaround for Windows: the underlying tool fails on paths # with the \\?\ prefix, so we don't use it here. This @@ -286,6 +294,7 @@ class Bs1770gainBackend(Backend): album_gain = {} # mutable variable so it can be set from handlers parser = xml.parsers.expat.ParserCreate(encoding='utf-8') state = {'file': None, 'gain': None, 'peak': None} + album_state = {'gain': None, 'peak': None} def start_element_handler(name, attrs): if name == u'track': @@ -294,9 +303,13 @@ class Bs1770gainBackend(Backend): raise ReplayGainError( u'duplicate filename in bs1770gain output') elif name == u'integrated': - state['gain'] = float(attrs[u'lu']) + if 'lu' in attrs: + state['gain'] = float(attrs[u'lu']) elif name == u'sample-peak': - state['peak'] = float(attrs[u'factor']) + if 'factor' in attrs: + state['peak'] = float(attrs[u'factor']) + elif 'amplitude' in attrs: + state['peak'] = float(attrs[u'amplitude']) def end_element_handler(name): if name == u'track': @@ -312,10 +325,25 @@ class Bs1770gainBackend(Backend): 'the output of bs1770gain') album_gain["album"] = Gain(state['gain'], state['peak']) state['gain'] = state['peak'] = None + elif len(per_file_gain) == len(path_list): + if state['gain'] is not None: + album_state['gain'] = state['gain'] + if state['peak'] is not None: + album_state['peak'] = state['peak'] + if album_state['gain'] is not None \ + and album_state['peak'] is not None: + album_gain["album"] = Gain( + album_state['gain'], album_state['peak']) + state['gain'] = state['peak'] = None + parser.StartElementHandler = start_element_handler parser.EndElementHandler = end_element_handler try: + if type(text) == bytes: + text = text.decode('utf-8') + while '\x08' in text: + text = re.sub('[^\x08]\x08', '', text) parser.Parse(text, True) except xml.parsers.expat.ExpatError: raise ReplayGainError( diff --git a/test/test_replaygain.py b/test/test_replaygain.py index fe0515bee..437b1426a 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain', ['--replaygain']): +if has_program('bs1770gain'): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -58,7 +58,6 @@ def reset_replaygain(item): class ReplayGainCliTestBase(TestHelper): - def setUp(self): self.setup_beets() self.config['replaygain']['backend'] = self.backend From c78afb1a97bb41788197ec1f7f3965e2e6a4f61b Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 16:37:56 +0100 Subject: [PATCH 352/613] Don't call bs1770gain outside of try statement --- beetsplug/replaygain.py | 12 +++++++----- test/test_replaygain.py | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3e284ce62..5b715191e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -136,11 +136,6 @@ class Bs1770gainBackend(Backend): -18: "replaygain", } - version = re.search( - 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', - call(['bs1770gain', '--version']).stdout.decode('utf-8') - ).group(1) - def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) config.add({ @@ -155,6 +150,13 @@ class Bs1770gainBackend(Backend): try: call([cmd, "--help"]) self.command = cmd + try: + self.version = re.search( + '([0-9]+.[0-9]+.[0-9]+), ', + call([cmd, '--version']).stdout.decode('utf-8') + ).group(1) + except AttributeError: + self.version = '0.0.0' except OSError: raise FatalReplayGainError( u'Is bs1770gain installed?' diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 437b1426a..fe0515bee 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain'): +if has_program('bs1770gain', ['--replaygain']): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -58,6 +58,7 @@ def reset_replaygain(item): class ReplayGainCliTestBase(TestHelper): + def setUp(self): self.setup_beets() self.config['replaygain']['backend'] = self.backend From c1cb78c908b39d506aa6ea9384c1eb98938217cd Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 17:59:57 +0100 Subject: [PATCH 353/613] Small fixes in `replaygain.Bs1770gainBackend` and test_replaygain.py * Fix unspecified `gain_adjustment` when method defined in config * Fix difference between dB and LUFS values in case of mismatched `target_level`/`method`: ``` db_to_lufs( target_level ) - lufs_to_dB( -23 ) ``` * Ignore single assertion in case of bs1770gain (cherry picked from commit 2395bf224032c44f1ea5d28e0c63af96a92b96df) --- beetsplug/replaygain.py | 7 +++++-- test/test_replaygain.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 5b715191e..b86ef4b1e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -249,12 +249,15 @@ class Bs1770gainBackend(Backend): if self.__method != "": # backward compatibility to `method` option method = self.__method + gain_adjustment = target_level \ + - [k for k, v in self.methods.items() if v == method][0] elif target_level in self.methods: method = self.methods[target_level] gain_adjustment = 0 else: - method = self.methods[-23] - gain_adjustment = target_level - lufs_to_db(-23) + lufs_target = -23 + method = self.methods[lufs_target] + gain_adjustment = target_level - lufs_target # Construct shell command. cmd = [self.command] diff --git a/test/test_replaygain.py b/test/test_replaygain.py index fe0515bee..3f317aeb3 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -151,7 +151,9 @@ class ReplayGainCliTestBase(TestHelper): self.assertEqual(max(gains), min(gains)) self.assertNotEqual(max(gains), 0.0) - self.assertNotEqual(max(peaks), 0.0) + if not self.backend == "bs1770gain": + # Actually produces peaks == 0.0 ~ self.add_album_fixture + self.assertNotEqual(max(peaks), 0.0) def test_cli_writes_only_r128_tags(self): if self.backend == "command": From c3817a4c06ea2e029a9d6a07a5f0e1facd6779b1 Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 18:13:59 +0100 Subject: [PATCH 354/613] Implement review comments * safer version comparison * regex bytes directly * handle b'\x08 ...' case * test_replaygain.py: injected command output should match the type of the actual output --- beetsplug/replaygain.py | 37 ++++++++++++++++++++++++------------- test/test_replaygain.py | 4 ++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index b86ef4b1e..661fcc2e3 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -25,6 +25,7 @@ import enum import re import xml.parsers.expat from six.moves import zip +from packaging import version from beets import ui from beets.plugins import BeetsPlugin @@ -148,19 +149,18 @@ class Bs1770gainBackend(Backend): cmd = 'bs1770gain' try: - call([cmd, "--help"]) + version_out = call([cmd, '--version']) self.command = cmd - try: - self.version = re.search( - '([0-9]+.[0-9]+.[0-9]+), ', - call([cmd, '--version']).stdout.decode('utf-8') - ).group(1) - except AttributeError: - self.version = '0.0.0' + self.version = re.search( + '([0-9]+.[0-9]+.[0-9]+), ', + version_out.stdout.decode('utf-8') + ).group(1) except OSError: raise FatalReplayGainError( u'Is bs1770gain installed?' ) + except AttributeError: + self.version = '0.0.0' if not self.command: raise FatalReplayGainError( u'no replaygain command found: install bs1770gain' @@ -263,7 +263,7 @@ class Bs1770gainBackend(Backend): cmd = [self.command] cmd += ["--" + method] cmd += ['--xml', '-p'] - if self.version >= '0.6.0': + if version.parse(self.version) >= version.parse('0.6.0'): cmd += ['--unit=ebu'] # set units to LU # Workaround for Windows: the underlying tool fails on paths @@ -345,10 +345,21 @@ class Bs1770gainBackend(Backend): parser.EndElementHandler = end_element_handler try: - if type(text) == bytes: - text = text.decode('utf-8') - while '\x08' in text: - text = re.sub('[^\x08]\x08', '', text) + # Sometimes, the XML out put of `bs1770gain` gets spliced with + # some progress percentages: b'9%\x08\x0810%\x08\x08' + # that are supposed to be canceled out by appending + # a b'\x08' backspace characters for every character, + # + # For some reason, these backspace characters don't get + # resolved, resulting in mangled XML. + + # While there are backspace characters in the output + while b'\x08' in text: + text = re.sub(b'[^\x08]\x08|^\x08', b'', text) + # Replace every occurence of a non-backspace character + # followed by a backspace character or a backspace character + # at the beginning of the string by an empty byte string b'' + parser.Parse(text, True) except xml.parsers.expat.ExpatError: raise ReplayGainError( diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 3f317aeb3..3ac19f8f9 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain', ['--replaygain']): +if has_program('bs1770gain'): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -252,7 +252,7 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): @patch('beetsplug.replaygain.call') def test_malformed_output(self, call_patch): # Return malformed XML (the ampersand should be &) - call_patch.return_value = CommandOutput(stdout=""" + call_patch.return_value = CommandOutput(stdout=b""" From 506be0259789c552f346d618d1774847e57ce0cf Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 20:11:09 +0100 Subject: [PATCH 355/613] Remove `packaging` dependency --- beetsplug/replaygain.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 661fcc2e3..950969547 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -25,7 +25,6 @@ import enum import re import xml.parsers.expat from six.moves import zip -from packaging import version from beets import ui from beets.plugins import BeetsPlugin @@ -68,6 +67,11 @@ def call(args, **kwargs): raise ReplayGainError(u"argument encoding failed") +def after_version(version_a, version_b): + return tuple(int(s) for s in version_a.split('.')) \ + >= tuple(int(s) for s in version_b.split('.')) + + def db_to_lufs(db): """Convert db to LUFS. @@ -263,7 +267,7 @@ class Bs1770gainBackend(Backend): cmd = [self.command] cmd += ["--" + method] cmd += ['--xml', '-p'] - if version.parse(self.version) >= version.parse('0.6.0'): + if after_version(self.version, '0.6.0'): cmd += ['--unit=ebu'] # set units to LU # Workaround for Windows: the underlying tool fails on paths From bef473c8e85b375a2dc5a40feb82bc971286e550 Mon Sep 17 00:00:00 2001 From: ybnd Date: Fri, 31 Jan 2020 07:42:50 +0100 Subject: [PATCH 356/613] Remove spliced progress regex and add --suppress-progress flag --- beetsplug/replaygain.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 950969547..3c652c7bd 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -268,7 +268,8 @@ class Bs1770gainBackend(Backend): cmd += ["--" + method] cmd += ['--xml', '-p'] if after_version(self.version, '0.6.0'): - cmd += ['--unit=ebu'] # set units to LU + cmd += ['--unit=ebu'] # set units to LU + cmd += ['--suppress-progress'] # don't print % to XML output # Workaround for Windows: the underlying tool fails on paths # with the \\?\ prefix, so we don't use it here. This @@ -349,21 +350,6 @@ class Bs1770gainBackend(Backend): parser.EndElementHandler = end_element_handler try: - # Sometimes, the XML out put of `bs1770gain` gets spliced with - # some progress percentages: b'9%\x08\x0810%\x08\x08' - # that are supposed to be canceled out by appending - # a b'\x08' backspace characters for every character, - # - # For some reason, these backspace characters don't get - # resolved, resulting in mangled XML. - - # While there are backspace characters in the output - while b'\x08' in text: - text = re.sub(b'[^\x08]\x08|^\x08', b'', text) - # Replace every occurence of a non-backspace character - # followed by a backspace character or a backspace character - # at the beginning of the string by an empty byte string b'' - parser.Parse(text, True) except xml.parsers.expat.ExpatError: raise ReplayGainError( From 9f43408f1b25910f7ebe98e0a30aab6ff0a33611 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sun, 2 Feb 2020 15:57:43 -0700 Subject: [PATCH 357/613] Changelog and cleanup --- beetsplug/lyrics.py | 8 ++++++-- docs/changelog.rst | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index b10f8dd02..20e39548c 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -395,15 +395,19 @@ class Genius(Backend): song_info = None for hit in json["response"]["hits"]: - if hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() == artist.lower(): + # Genius uses zero-width characters to denote lowercase artist names + hit_artist = hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() + + if hit_artist == artist.lower(): song_info = hit break if song_info: + self._log.debug(u'fetched: {0}', song_info["result"]["url"]) song_api_path = song_info["result"]["api_path"] return self.lyrics_from_song_api_path(song_api_path) else: - self._log.debug(u'Genius did not return a matching artist entry') + self._log.debug(u'genius: no matching artist') class LyricsWiki(SymbolsReplaced): diff --git a/docs/changelog.rst b/docs/changelog.rst index c6b805b30..545bf7a84 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -134,6 +134,8 @@ Fixes: * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. Thanks to :user:`aereaux`. :bug:`3437` +* :doc:`/plugins/lyrics`: Fix a corner-case with Genius lowercase artist names + :bug:`3446` For plugin developers: From 95e0f54d7cea6c338efa47d16bc96c148970219f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 2 Feb 2020 21:04:55 -0500 Subject: [PATCH 358/613] Fix too-long lines (#3448) --- beetsplug/lyrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 20e39548c..0e797d5a3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -395,8 +395,10 @@ class Genius(Backend): song_info = None for hit in json["response"]["hits"]: - # Genius uses zero-width characters to denote lowercase artist names - hit_artist = hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() + # Genius uses zero-width characters to denote lowercase + # artist names. + hit_artist = hit["result"]["primary_artist"]["name"]. \ + strip(u'\u200b').lower() if hit_artist == artist.lower(): song_info = hit From 9465e933ba4d24dd409fe4de894da71808113179 Mon Sep 17 00:00:00 2001 From: ybnd Date: Wed, 5 Feb 2020 08:36:44 +0100 Subject: [PATCH 359/613] Add to changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 95fde6888..ffbcf3b97 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -151,6 +151,8 @@ Fixes: * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. Thanks to :user:`aereaux`. :bug:`3437` +* :doc:`/plugins/replaygain`: Support ``bs1770gain`` v0.6.0 and up + :bug:`3480` For plugin developers: From 7005691410549c65c680c66bd852a38ad725c981 Mon Sep 17 00:00:00 2001 From: ybnd Date: Wed, 5 Feb 2020 08:52:50 +0100 Subject: [PATCH 360/613] Add comment to clarify unexpected AttributeError handling --- beetsplug/replaygain.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3c652c7bd..f5a4a2c55 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -164,6 +164,9 @@ class Bs1770gainBackend(Backend): u'Is bs1770gain installed?' ) except AttributeError: + # Raised by ReplayGainLdnsCliMalformedTest.test_malformed_output + # in test_replaygain.py; bs1770gain backend runs even though + # the bs1770gain command is not found test.helper.has_program self.version = '0.0.0' if not self.command: raise FatalReplayGainError( From 63ea17365a5dc75a9decbba3899bb4975df3989c Mon Sep 17 00:00:00 2001 From: ybnd Date: Wed, 5 Feb 2020 09:03:45 +0100 Subject: [PATCH 361/613] Modify patched stdout in test_malformed_output --- beetsplug/replaygain.py | 7 +------ test/test_replaygain.py | 4 +++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index f5a4a2c55..646e3acce 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -156,18 +156,13 @@ class Bs1770gainBackend(Backend): version_out = call([cmd, '--version']) self.command = cmd self.version = re.search( - '([0-9]+.[0-9]+.[0-9]+), ', + 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', version_out.stdout.decode('utf-8') ).group(1) except OSError: raise FatalReplayGainError( u'Is bs1770gain installed?' ) - except AttributeError: - # Raised by ReplayGainLdnsCliMalformedTest.test_malformed_output - # in test_replaygain.py; bs1770gain backend runs even though - # the bs1770gain command is not found test.helper.has_program - self.version = '0.0.0' if not self.command: raise FatalReplayGainError( u'no replaygain command found: install bs1770gain' diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 3ac19f8f9..969f5c230 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -230,7 +230,9 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): # Patch call to return nothing, bypassing the bs1770gain installation # check. - call_patch.return_value = CommandOutput(stdout=b"", stderr=b"") + call_patch.return_value = CommandOutput( + stdout=b'bs1770gain 0.0.0, ', stderr=b'' + ) try: self.load_plugins('replaygain') except Exception: From 964a6c2e63e56a9d04f1c34cc8b21511e0a11ba0 Mon Sep 17 00:00:00 2001 From: Tyler Faulk Date: Thu, 6 Feb 2020 12:10:38 -0500 Subject: [PATCH 362/613] restored whitespace to please style checker --- beetsplug/fetchart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ca93d685e..f78e6116a 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -175,6 +175,7 @@ def _logged_get(log, *args, **kwargs): log.debug('{}: {}', message, prepped.url) return s.send(prepped, **send_kwargs) + class RequestMixin(object): """Adds a Requests wrapper to the class that uses the logger, which must be named `self._log`. From 91be732bf467f9bc33a2455a6a2c5aff12db810d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 6 Feb 2020 22:18:15 -0500 Subject: [PATCH 363/613] Fix whitespace (#3453) --- beetsplug/fetchart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 408b947ca..ad96ece23 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -169,7 +169,7 @@ def _logged_get(log, *args, **kwargs): s.headers = {'User-Agent': 'beets'} prepped = s.prepare_request(req) settings = s.merge_environment_settings( - prepped.url, {}, None, None, None + prepped.url, {}, None, None, None ) send_kwargs.update(settings) log.debug('{}: {}', message, prepped.url) From d43d54e21cde97f57f19486925ab56b419254cc8 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 6 Feb 2020 22:22:54 -0500 Subject: [PATCH 364/613] Try to work around a Werkzeug change? --- beetsplug/web/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index f53fb3a95..21ff5d94e 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -169,7 +169,7 @@ class IdListConverter(BaseConverter): return ids def to_url(self, value): - return ','.join(value) + return ','.join(str(v) for v in value) class QueryConverter(PathConverter): From 9426ad73455657c616a116761e65debf898f8384 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 17 Feb 2020 09:40:30 +0100 Subject: [PATCH 365/613] Added beets-bpmanalyser to the list of 'other plugins' --- docs/plugins/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 7dc152bd1..955cc1405 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -299,6 +299,8 @@ Here are a few of the plugins written by the beets community: * `beets-mosaic`_ generates a montage of a mosiac from cover art. +* `beets-bpmanalyser`_ analyses songs and calculates tempo(bpm). + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -321,3 +323,4 @@ Here are a few of the plugins written by the beets community: .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic +.. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser From ddfb715e322c38cdf22019dacb96d81951e96ea7 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 17 Feb 2020 06:49:26 -0800 Subject: [PATCH 366/613] Style fixes for #3491 --- docs/plugins/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 955cc1405..125bdd934 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -299,7 +299,7 @@ Here are a few of the plugins written by the beets community: * `beets-mosaic`_ generates a montage of a mosiac from cover art. -* `beets-bpmanalyser`_ analyses songs and calculates tempo(bpm). +* `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check From 535af4bdb2ef7e9470798830e87a9ba4bad9217e Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Mon, 17 Feb 2020 07:40:38 -0800 Subject: [PATCH 367/613] parentwork: Only call store when the metadata has changed. Otherwise, this rewrites all your files every time. --- beetsplug/parentwork.py | 11 ++++++----- docs/changelog.rst | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index eaa8abb30..d40254696 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -89,10 +89,11 @@ class ParentWorkPlugin(BeetsPlugin): write = ui.should_write() for item in lib.items(ui.decargs(args)): - self.find_work(item, force_parent) - item.store() - if write: - item.try_write() + changed = self.find_work(item, force_parent) + if changed: + item.store() + if write: + item.try_write() command = ui.Subcommand( 'parentwork', help=u'fetche parent works, composers and dates') @@ -198,7 +199,7 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) if work_date: item['work_date'] = work_date - ui.show_model_changes( + return ui.show_model_changes( item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date']) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c91b422c..97cc7ca12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -158,6 +158,8 @@ Fixes: :bug:`3446` * :doc:`/plugins/replaygain`: Support ``bs1770gain`` v0.6.0 and up :bug:`3480` +* :doc:`/plugins/parentwork`: Don't save tracks when nothing has changed. + :bug:`3492` For plugin developers: From 86946ad4b7d03cd78d308456eec193b068078628 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sun, 15 Dec 2019 16:37:21 -0500 Subject: [PATCH 368/613] Allow the quality to be set for embedded/fetched cover art --- beets/art.py | 14 +++++++------- beets/util/artresizer.py | 11 ++++++----- beetsplug/convert.py | 2 +- beetsplug/embedart.py | 12 +++++++----- beetsplug/fetchart.py | 2 +- beetsplug/thumbnails.py | 2 +- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/beets/art.py b/beets/art.py index e7a087a05..e48be4a57 100644 --- a/beets/art.py +++ b/beets/art.py @@ -50,7 +50,7 @@ def get_art(log, item): return mf.art -def embed_item(log, item, imagepath, maxwidth=None, itempath=None, +def embed_item(log, item, imagepath, maxwidth=None, quality=75, itempath=None, compare_threshold=0, ifempty=False, as_album=False, id3v23=None): """Embed an image into the item's media file. @@ -64,7 +64,7 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, log.info(u'media file already contained art') return if maxwidth and not as_album: - imagepath = resize_image(log, imagepath, maxwidth) + imagepath = resize_image(log, imagepath, maxwidth, quality) # Get the `Image` object from the file. try: @@ -84,7 +84,7 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23) -def embed_album(log, album, maxwidth=None, quiet=False, +def embed_album(log, album, maxwidth=None, quality=75, quiet=False, compare_threshold=0, ifempty=False): """Embed album art into all of the album's items. """ @@ -97,20 +97,20 @@ def embed_album(log, album, maxwidth=None, quiet=False, displayable_path(imagepath), album) return if maxwidth: - imagepath = resize_image(log, imagepath, maxwidth) + imagepath = resize_image(log, imagepath, maxwidth, quality) log.info(u'Embedding album art into {0}', album) for item in album.items(): - embed_item(log, item, imagepath, maxwidth, None, + embed_item(log, item, imagepath, maxwidth, quality, None, compare_threshold, ifempty, as_album=True) -def resize_image(log, imagepath, maxwidth): +def resize_image(log, imagepath, maxwidth, quality): """Returns path to an image resized to maxwidth. """ log.debug(u'Resizing album art to {0} pixels wide', maxwidth) - imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath)) + imagepath = ArtResizer.shared.resize(maxwidth, quality, syspath(imagepath)) return imagepath diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 99e28c0cc..3a6f58752 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -59,7 +59,7 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize(maxwidth, path_in, path_out=None): +def pil_resize(maxwidth, quality, path_in, path_out=None): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -72,7 +72,7 @@ def pil_resize(maxwidth, path_in, path_out=None): im = Image.open(util.syspath(path_in)) size = maxwidth, maxwidth im.thumbnail(size, Image.ANTIALIAS) - im.save(util.py3_path(path_out)) + im.save(util.py3_path(path_out), quality=quality) return path_out except IOError: log.error(u"PIL cannot create thumbnail for '{0}'", @@ -80,7 +80,7 @@ def pil_resize(maxwidth, path_in, path_out=None): return path_in -def im_resize(maxwidth, path_in, path_out=None): +def im_resize(maxwidth, quality, path_in, path_out=None): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -96,6 +96,7 @@ def im_resize(maxwidth, path_in, path_out=None): cmd = ArtResizer.shared.im_convert_cmd + \ [util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), + '-quality', '{0}x'.format(quality), util.syspath(path_out, prefix=False)] try: @@ -190,14 +191,14 @@ class ArtResizer(six.with_metaclass(Shareable, object)): self.im_convert_cmd = ['magick'] self.im_identify_cmd = ['magick', 'identify'] - def resize(self, maxwidth, path_in, path_out=None): + def resize(self, maxwidth, quality, path_in, path_out=None): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a temporary file. For WEBPROXY, returns `path_in` unmodified. """ if self.local: func = BACKEND_FUNCS[self.method[0]] - return func(maxwidth, path_in, path_out) + return func(maxwidth, quality, path_in, path_out) else: return path_in diff --git a/beetsplug/convert.py b/beetsplug/convert.py index e7ac4f3ac..ab86800f6 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -422,7 +422,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(album.artpath), util.displayable_path(dest)) if not pretend: - ArtResizer.shared.resize(maxwidth, album.artpath, dest) + ArtResizer.shared.resize(maxwidth, 75, album.artpath, dest) else: if pretend: msg = 'ln' if hardlink else ('ln -s' if link else 'cp') diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 71681f024..e581a2ddf 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -59,7 +59,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): 'auto': True, 'compare_threshold': 0, 'ifempty': False, - 'remove_art_file': False + 'remove_art_file': False, + 'quality': 95, }) if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: @@ -86,6 +87,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): u"-y", u"--yes", action="store_true", help=u"skip confirmation" ) maxwidth = self.config['maxwidth'].get(int) + quality = self.config['quality'].get(int) compare_threshold = self.config['compare_threshold'].get(int) ifempty = self.config['ifempty'].get(bool) @@ -104,8 +106,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): return for item in items: - art.embed_item(self._log, item, imagepath, maxwidth, None, - compare_threshold, ifempty) + art.embed_item(self._log, item, imagepath, maxwidth, + quality, None, compare_threshold, ifempty) else: albums = lib.albums(decargs(args)) @@ -114,8 +116,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): return for album in albums: - art.embed_album(self._log, album, maxwidth, False, - compare_threshold, ifempty) + art.embed_album(self._log, album, maxwidth, quality, + False, compare_threshold, ifempty) self.remove_artfile(album) embed_cmd.func = embed_func diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad96ece23..99c592991 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -135,7 +135,7 @@ class Candidate(object): def resize(self, plugin): if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path) + self.path = ArtResizer.shared.resize(plugin.maxwidth, 75, self.path) def _logged_get(log, *args, **kwargs): diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index fe36fbd13..de949ea32 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -152,7 +152,7 @@ class ThumbnailsPlugin(BeetsPlugin): self._log.debug(u"{1}x{1} thumbnail for {0} exists and is " u"recent enough", album, size) return False - resized = ArtResizer.shared.resize(size, album.artpath, + resized = ArtResizer.shared.resize(size, 75, album.artpath, util.syspath(target)) self.add_tags(album, util.syspath(resized)) shutil.move(resized, target) From a30c90e6152ed13293267e5246119398e676fcad Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Mon, 17 Feb 2020 22:22:47 -0500 Subject: [PATCH 369/613] Fix one of the failing tests --- test/test_thumbnails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index dc03f06f7..0cf22d4a1 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -154,7 +154,7 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): any_order=True) resize = mock_artresizer.shared.resize - resize.assert_called_once_with(12345, path_to_art, md5_file) + resize.assert_called_once_with(12345, 75, path_to_art, md5_file) plugin.add_tags.assert_called_once_with(album, resize.return_value) mock_shutils.move.assert_called_once_with(resize.return_value, md5_file) From 036202e1c5a0cae19397affa6eeace1cfed5ea6c Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Tue, 18 Feb 2020 14:50:57 -0500 Subject: [PATCH 370/613] Default quality to 0 which means don't specify From the ImageMagick docs: "The default is to use the estimated quality of your input image if it can be determined, otherwise 92." In order to get the original behaviour we need to conditional add the quality parameter to the `magick` call. The quality range can be anything from 1 to 100, which gives us the convenience of using 0 to specify no specific quality level. --- beets/art.py | 17 +++++++++-------- beets/util/artresizer.py | 22 +++++++++++++--------- beetsplug/convert.py | 2 +- beetsplug/embedart.py | 10 ++++++---- test/test_thumbnails.py | 2 +- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/beets/art.py b/beets/art.py index e48be4a57..b8b172594 100644 --- a/beets/art.py +++ b/beets/art.py @@ -50,9 +50,9 @@ def get_art(log, item): return mf.art -def embed_item(log, item, imagepath, maxwidth=None, quality=75, itempath=None, - compare_threshold=0, ifempty=False, as_album=False, - id3v23=None): +def embed_item(log, item, imagepath, maxwidth=None, itempath=None, + compare_threshold=0, ifempty=False, as_album=False, id3v23=None, + quality=0): """Embed an image into the item's media file. """ # Conditions and filters. @@ -84,8 +84,8 @@ def embed_item(log, item, imagepath, maxwidth=None, quality=75, itempath=None, item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23) -def embed_album(log, album, maxwidth=None, quality=75, quiet=False, - compare_threshold=0, ifempty=False): +def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0, + ifempty=False, quality=0): """Embed album art into all of the album's items. """ imagepath = album.artpath @@ -102,15 +102,16 @@ def embed_album(log, album, maxwidth=None, quality=75, quiet=False, log.info(u'Embedding album art into {0}', album) for item in album.items(): - embed_item(log, item, imagepath, maxwidth, quality, None, - compare_threshold, ifempty, as_album=True) + embed_item(log, item, imagepath, maxwidth, None, compare_threshold, + ifempty, as_album=True, quality=quality) def resize_image(log, imagepath, maxwidth, quality): """Returns path to an image resized to maxwidth. """ log.debug(u'Resizing album art to {0} pixels wide', maxwidth) - imagepath = ArtResizer.shared.resize(maxwidth, quality, syspath(imagepath)) + imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath), + quality=quality) return imagepath diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 3a6f58752..09d138322 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -59,7 +59,7 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize(maxwidth, quality, path_in, path_out=None): +def pil_resize(maxwidth, path_in, path_out=None, quality=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -80,7 +80,7 @@ def pil_resize(maxwidth, quality, path_in, path_out=None): return path_in -def im_resize(maxwidth, quality, path_in, path_out=None): +def im_resize(maxwidth, path_in, path_out=None, quality=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -93,11 +93,15 @@ def im_resize(maxwidth, quality, path_in, path_out=None): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. - cmd = ArtResizer.shared.im_convert_cmd + \ - [util.syspath(path_in, prefix=False), - '-resize', '{0}x>'.format(maxwidth), - '-quality', '{0}x'.format(quality), - util.syspath(path_out, prefix=False)] + cmd = ArtResizer.shared.im_convert_cmd + [ + util.syspath(path_in, prefix=False), + '-resize', '{0}x>'.format(maxwidth), + ] + + if quality > 0: + cmd += ['-quality', '{0}'.format(quality)] + + cmd.append(util.syspath(path_out, prefix=False)) try: util.command_output(cmd) @@ -191,14 +195,14 @@ class ArtResizer(six.with_metaclass(Shareable, object)): self.im_convert_cmd = ['magick'] self.im_identify_cmd = ['magick', 'identify'] - def resize(self, maxwidth, quality, path_in, path_out=None): + def resize(self, maxwidth, path_in, path_out=None, quality=0): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a temporary file. For WEBPROXY, returns `path_in` unmodified. """ if self.local: func = BACKEND_FUNCS[self.method[0]] - return func(maxwidth, quality, path_in, path_out) + return func(maxwidth, path_in, path_out, quality=quality) else: return path_in diff --git a/beetsplug/convert.py b/beetsplug/convert.py index ab86800f6..e7ac4f3ac 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -422,7 +422,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(album.artpath), util.displayable_path(dest)) if not pretend: - ArtResizer.shared.resize(maxwidth, 75, album.artpath, dest) + ArtResizer.shared.resize(maxwidth, album.artpath, dest) else: if pretend: msg = 'ln' if hardlink else ('ln -s' if link else 'cp') diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index e581a2ddf..61a4d798f 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -60,7 +60,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): 'compare_threshold': 0, 'ifempty': False, 'remove_art_file': False, - 'quality': 95, + 'quality': 0, }) if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: @@ -107,7 +107,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): for item in items: art.embed_item(self._log, item, imagepath, maxwidth, - quality, None, compare_threshold, ifempty) + None, compare_threshold, ifempty, + quality=quality) else: albums = lib.albums(decargs(args)) @@ -116,8 +117,9 @@ class EmbedCoverArtPlugin(BeetsPlugin): return for album in albums: - art.embed_album(self._log, album, maxwidth, quality, - False, compare_threshold, ifempty) + art.embed_album(self._log, album, maxwidth, + False, compare_threshold, ifempty, + quality=quality) self.remove_artfile(album) embed_cmd.func = embed_func diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index 0cf22d4a1..dc03f06f7 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -154,7 +154,7 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): any_order=True) resize = mock_artresizer.shared.resize - resize.assert_called_once_with(12345, 75, path_to_art, md5_file) + resize.assert_called_once_with(12345, path_to_art, md5_file) plugin.add_tags.assert_called_once_with(album, resize.return_value) mock_shutils.move.assert_called_once_with(resize.return_value, md5_file) From 96b0e8a33ef0e9f38a4d6264cb4355434e7b17e5 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Tue, 18 Feb 2020 16:44:45 -0500 Subject: [PATCH 371/613] No longer need to pass a default quality here --- beetsplug/fetchart.py | 2 +- beetsplug/thumbnails.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 99c592991..ad96ece23 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -135,7 +135,7 @@ class Candidate(object): def resize(self, plugin): if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, 75, self.path) + self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path) def _logged_get(log, *args, **kwargs): diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index de949ea32..fe36fbd13 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -152,7 +152,7 @@ class ThumbnailsPlugin(BeetsPlugin): self._log.debug(u"{1}x{1} thumbnail for {0} exists and is " u"recent enough", album, size) return False - resized = ArtResizer.shared.resize(size, 75, album.artpath, + resized = ArtResizer.shared.resize(size, album.artpath, util.syspath(target)) self.add_tags(album, util.syspath(resized)) shutil.move(resized, target) From c9e6f910308a4fa46b74755cb0ba2e373e117df2 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Thu, 20 Feb 2020 18:55:56 +0100 Subject: [PATCH 372/613] Added beets-goingrunning to the list of 'other plugins' --- docs/plugins/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 125bdd934..5d6f4795e 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -301,6 +301,8 @@ Here are a few of the plugins written by the beets community: * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). +* `beets-goingrunning`_ copies songs to external device based on their tempo (BPM) and length to go with your running session. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -324,3 +326,4 @@ Here are a few of the plugins written by the beets community: .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser +.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ \ No newline at end of file From 37f3e226b0fd8b044a8fff73a848a40107595e62 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Thu, 20 Feb 2020 18:58:11 +0100 Subject: [PATCH 373/613] beets-goingrunning - shortened description --- docs/plugins/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 5d6f4795e..383466a68 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -301,7 +301,7 @@ Here are a few of the plugins written by the beets community: * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). -* `beets-goingrunning`_ copies songs to external device based on their tempo (BPM) and length to go with your running session. +* `beets-goingrunning`_ copies songs to external device to go with your running session. .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check From bc53695f24d06bcc33df2d0a4e6e6a739cab27fb Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sat, 22 Feb 2020 01:56:57 +0100 Subject: [PATCH 374/613] added missing test requirement --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 544721937..c50e65bf5 100755 --- a/setup.py +++ b/setup.py @@ -118,7 +118,8 @@ setup( 'responses', 'pyxdg', 'python-mpd2', - 'discogs-client' + 'discogs-client', + 'requests_oauthlib' ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] From 15ad525e0d904dab48d2575f5c2438b3f7d1276f Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:02:10 -0500 Subject: [PATCH 375/613] Mention quality level in docstrings --- beets/art.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/art.py b/beets/art.py index b8b172594..0cb9cdf1b 100644 --- a/beets/art.py +++ b/beets/art.py @@ -107,9 +107,11 @@ def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0, def resize_image(log, imagepath, maxwidth, quality): - """Returns path to an image resized to maxwidth. + """Returns path to an image resized to maxwidth and encoded with the + specified quality level. """ - log.debug(u'Resizing album art to {0} pixels wide', maxwidth) + log.debug(u'Resizing album art to {0} pixels wide and encoding at quality + level {1}', maxwidth, quality) imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath), quality=quality) return imagepath From d82573f8f3231863cb892d222250bd603a0fe382 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:02:20 -0500 Subject: [PATCH 376/613] Document quality setting --- docs/plugins/embedart.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index cc2fe6fc8..bd87abed6 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -58,6 +58,11 @@ file. The available options are: the aspect ratio is preserved. See also :ref:`image-resizing` for further caveats about image resizing. Default: 0 (disabled). +- **quality**: The quality level to use when encoding the image file when + downscaling to ``maxwidth``. The default behaviour depends on the method used + to scale the images. ImageMagick tries to estimate the input image quality and + uses 92 if it cannot be determined. Pillow defaults to 75. + Default: 0 (disabled) - **remove_art_file**: Automatically remove the album art file for the album after it has been embedded. This option is best used alongside the :doc:`FetchArt ` plugin to download art with the purpose of From 4e7fd47a3574e567482bc2fb7e9a3bb51215fc12 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:05:02 -0500 Subject: [PATCH 377/613] Add quality to art resizer docstring --- beets/util/artresizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 09d138322..cf457e797 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -198,7 +198,8 @@ class ArtResizer(six.with_metaclass(Shareable, object)): def resize(self, maxwidth, path_in, path_out=None, quality=0): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a - temporary file. For WEBPROXY, returns `path_in` unmodified. + temporary file and encodes with the specified quality level. + For WEBPROXY, returns `path_in` unmodified. """ if self.local: func = BACKEND_FUNCS[self.method[0]] From a729bd872959232dad22591bfa497b40f0974872 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:23:02 -0500 Subject: [PATCH 378/613] Send quality parameter to images.weserv.nl --- beets/util/artresizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index cf457e797..aee06a5b6 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -40,14 +40,19 @@ else: log = logging.getLogger('beets') -def resize_url(url, maxwidth): +def resize_url(url, maxwidth, quality=0): """Return a proxied image URL that resizes the original image to maxwidth (preserving aspect ratio). """ - return '{0}?{1}'.format(PROXY_URL, urlencode({ + params = { 'url': url.replace('http://', ''), 'w': maxwidth, - })) + } + + if quality > 0: + params['q'] = quality + + return '{0}?{1}'.format(PROXY_URL, urlencode(params)) def temp_file_for(path): @@ -215,7 +220,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)): if self.local: return url else: - return resize_url(url, maxwidth) + return resize_url(url, maxwidth, quality) @property def local(self): From 6a84949020d0b05be86c0da56a13c67222c6f3ee Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:43:22 -0500 Subject: [PATCH 379/613] Woops, this needs to be explicitly multiline --- beets/art.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/art.py b/beets/art.py index 0cb9cdf1b..20b0e96d2 100644 --- a/beets/art.py +++ b/beets/art.py @@ -110,7 +110,7 @@ def resize_image(log, imagepath, maxwidth, quality): """Returns path to an image resized to maxwidth and encoded with the specified quality level. """ - log.debug(u'Resizing album art to {0} pixels wide and encoding at quality + log.debug(u'Resizing album art to {0} pixels wide and encoding at quality \ level {1}', maxwidth, quality) imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath), quality=quality) From 0af1bf5fbbe9be452cd18cb6ad34d456739623cc Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 13:00:21 -0500 Subject: [PATCH 380/613] Pass a default quality in here --- beets/util/artresizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index aee06a5b6..8f14c8baf 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -212,7 +212,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)): else: return path_in - def proxy_url(self, maxwidth, url): + def proxy_url(self, maxwidth, url, quality=0): """Modifies an image URL according the method, returning a new URL. For WEBPROXY, a URL on the proxy server is returned. Otherwise, the URL is returned unmodified. From fe8ba17ced235df664d66b644c4bf5f05a816288 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 13:36:35 -0500 Subject: [PATCH 381/613] Add quality setting to fetchart plugin --- beetsplug/fetchart.py | 12 ++++++++---- docs/plugins/fetchart.rst | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad96ece23..2fe8b0b2c 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -135,7 +135,8 @@ class Candidate(object): def resize(self, plugin): if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path) + self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path, + quality=plugin.quality) def _logged_get(log, *args, **kwargs): @@ -777,6 +778,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'auto': True, 'minwidth': 0, 'maxwidth': 0, + 'quality': 0, 'enforce_ratio': False, 'cautious': False, 'cover_names': ['cover', 'front', 'art', 'album', 'folder'], @@ -793,6 +795,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.minwidth = self.config['minwidth'].get(int) self.maxwidth = self.config['maxwidth'].get(int) + self.quality = self.config['quality'].get(int) # allow both pixel and percentage-based margin specifications self.enforce_ratio = self.config['enforce_ratio'].get( @@ -922,9 +925,10 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): def art_for_album(self, album, paths, local_only=False): """Given an Album object, returns a path to downloaded art for the album (or None if no art is found). If `maxwidth`, then images are - resized to this maximum pixel size. If `local_only`, then only local - image files from the filesystem are returned; no network requests - are made. + resized to this maximum pixel size. If `quality` then resized images + are saved at the specified quality level. If `local_only`, then only + local image files from the filesystem are returned; no network + requests are made. """ out = None diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 68212a582..23362aee0 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -42,6 +42,9 @@ file. The available options are: - **maxwidth**: A maximum image width to downscale fetched images if they are too big. The resize operation reduces image width to at most ``maxwidth`` pixels. The height is recomputed so that the aspect ratio is preserved. +- **quality**: The quality level to use when encoding the image file when + downscaling to ``maxwidth``. + Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. It is also possible to specify a certain deviation to the exact ratio to From d2e32a6b2056ec4237dc327c38194a50c4b27893 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 23 Feb 2020 23:29:15 +0100 Subject: [PATCH 382/613] Raising error on missing configuration file inclusion --- beets/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/__init__.py b/beets/__init__.py index 20075073c..20aed95e3 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -35,6 +35,8 @@ class IncludeLazyConfig(confuse.LazyConfig): filename = view.as_filename() if os.path.isfile(filename): self.set_file(filename) + else: + raise FileNotFoundError("Warning! Configuration file({0}) does not exist!".format(filename)) except confuse.NotFoundError: pass From dfa45f62a59b7bc431af2084fe35d84c2c2d08bd Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 00:07:24 +0100 Subject: [PATCH 383/613] fixed flake8 long line warning --- beets/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/__init__.py b/beets/__init__.py index 20aed95e3..c9863ee66 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -36,7 +36,8 @@ class IncludeLazyConfig(confuse.LazyConfig): if os.path.isfile(filename): self.set_file(filename) else: - raise FileNotFoundError("Warning! Configuration file({0}) does not exist!".format(filename)) + raise FileNotFoundError( + "Warning! Configuration file({0}) does not exist!".format(filename)) except confuse.NotFoundError: pass From c90f7aacfcb65eef3386c29d3c146f9f0a120ee1 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 00:22:34 +0100 Subject: [PATCH 384/613] fixed flake8 long line warning (maybe) --- beets/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index c9863ee66..ed3af5588 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -36,8 +36,8 @@ class IncludeLazyConfig(confuse.LazyConfig): if os.path.isfile(filename): self.set_file(filename) else: - raise FileNotFoundError( - "Warning! Configuration file({0}) does not exist!".format(filename)) + raise FileNotFoundError("Warning! Configuration file({0}) " + "does not exist!".format(filename)) except confuse.NotFoundError: pass From 3db55c7bf4f25c59102b4884165d2956ae80cfba Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 10:20:54 +0100 Subject: [PATCH 385/613] Simple warning on missing (included) configuration file. --- beets/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index ed3af5588..0b1048bdb 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -33,13 +33,12 @@ class IncludeLazyConfig(confuse.LazyConfig): try: for view in self['include']: filename = view.as_filename() - if os.path.isfile(filename): - self.set_file(filename) - else: - raise FileNotFoundError("Warning! Configuration file({0}) " - "does not exist!".format(filename)) + self.set_file(filename) except confuse.NotFoundError: pass + except confuse.ConfigReadError as err: + print("Warning! Missing configuration file! {}".format(err.reason)) + pass config = IncludeLazyConfig('beets', __name__) From 253d4c76d00f6fcf0e7cd6b30934aef0cf44212a Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 10:43:03 +0100 Subject: [PATCH 386/613] removed redundant import and redundant filename variable --- beets/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index 0b1048bdb..bed59d05f 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -15,8 +15,6 @@ from __future__ import division, absolute_import, print_function -import os - import confuse __version__ = u'1.5.0' @@ -32,8 +30,7 @@ class IncludeLazyConfig(confuse.LazyConfig): try: for view in self['include']: - filename = view.as_filename() - self.set_file(filename) + self.set_file(view.as_filename()) except confuse.NotFoundError: pass except confuse.ConfigReadError as err: From 131227eff44b0fa596abfcd7780b6ea81cc50829 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 22:38:30 +0100 Subject: [PATCH 387/613] writing warning message to stderr --- beets/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/__init__.py b/beets/__init__.py index bed59d05f..e9cc10e08 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -16,6 +16,7 @@ from __future__ import division, absolute_import, print_function import confuse +from sys import stderr __version__ = u'1.5.0' __author__ = u'Adrian Sampson ' @@ -34,7 +35,8 @@ class IncludeLazyConfig(confuse.LazyConfig): except confuse.NotFoundError: pass except confuse.ConfigReadError as err: - print("Warning! Missing configuration file! {}".format(err.reason)) + stderr.write("Configuration 'import' failed: {}" + .format(err.reason)) pass From d06665413c5b871c9375c28f5544eb80e6cc0d1c Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Tue, 25 Feb 2020 15:23:52 +0100 Subject: [PATCH 388/613] minor fixes and changelog entry --- beets/__init__.py | 3 +-- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index e9cc10e08..e3e5fdf83 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -35,9 +35,8 @@ class IncludeLazyConfig(confuse.LazyConfig): except confuse.NotFoundError: pass except confuse.ConfigReadError as err: - stderr.write("Configuration 'import' failed: {}" + stderr.write("configuration `import` failed: {}" .format(err.reason)) - pass config = IncludeLazyConfig('beets', __name__) diff --git a/docs/changelog.rst b/docs/changelog.rst index 97cc7ca12..9a4ad0988 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -160,6 +160,8 @@ Fixes: :bug:`3480` * :doc:`/plugins/parentwork`: Don't save tracks when nothing has changed. :bug:`3492` +* Added a warning when configuration files defined in the `include` directive + of the configuration file fail to be imported. For plugin developers: From 20d28948a3cb51c0a4993d913bc716dc1a476e4a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 25 Feb 2020 08:56:06 -0800 Subject: [PATCH 389/613] Changelog bug link for #3498 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a4ad0988..81ba590f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -162,6 +162,7 @@ Fixes: :bug:`3492` * Added a warning when configuration files defined in the `include` directive of the configuration file fail to be imported. + :bug:`3498` For plugin developers: From a9e11fcfebbb80f3a0fe90664856a3398c7b5b6a Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Wed, 26 Feb 2020 19:23:59 -0500 Subject: [PATCH 390/613] Add some info about valid quality levels --- docs/plugins/embedart.rst | 9 ++++++--- docs/plugins/fetchart.rst | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index bd87abed6..ece5e0350 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -59,9 +59,12 @@ file. The available options are: caveats about image resizing. Default: 0 (disabled). - **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. The default behaviour depends on the method used - to scale the images. ImageMagick tries to estimate the input image quality and - uses 92 if it cannot be determined. Pillow defaults to 75. + downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. + Higher numbers result in better image quality while lower numbers will result + in smaller files. 65-75 is a good starting point. The default behaviour + depends on the method used to scale the images. ImageMagick tries to estimate + the input image quality and uses 92 if it cannot be determined. Pillow + defaults to 75. Default: 0 (disabled) - **remove_art_file**: Automatically remove the album art file for the album after it has been embedded. This option is best used alongside the diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 23362aee0..8c5d0eb7b 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -43,7 +43,9 @@ file. The available options are: too big. The resize operation reduces image width to at most ``maxwidth`` pixels. The height is recomputed so that the aspect ratio is preserved. - **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. + downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. + Higher numbers result in better image quality while lower numbers will result + in smaller files. 65-75 is a good starting point. Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. From b3fec61f5409b05fa2c31d19f5160e1526332fd4 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Wed, 26 Feb 2020 19:31:42 -0500 Subject: [PATCH 391/613] Add details to changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 97cc7ca12..dc09a404f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog New features: +* :doc:`plugins/fetchart`: and :doc:`plugins/embedart`: Added a new ``quality`` + option that controls the quality of the image output when the image is + resized. * :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ Thanks to :user:`BrainDamage`. * :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to From 00c6d1439ee29b3b4592d6f65cfa57b1d1f96575 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 26 Feb 2020 19:15:44 -0800 Subject: [PATCH 392/613] Docs tweaks for #3493 --- docs/changelog.rst | 2 +- docs/plugins/embedart.rst | 13 ++++++------- docs/plugins/fetchart.rst | 10 ++++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b16593f1d..7ef19871b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog New features: -* :doc:`plugins/fetchart`: and :doc:`plugins/embedart`: Added a new ``quality`` +* :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is resized. * :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index ece5e0350..defd3fa4b 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -58,13 +58,12 @@ file. The available options are: the aspect ratio is preserved. See also :ref:`image-resizing` for further caveats about image resizing. Default: 0 (disabled). -- **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. - Higher numbers result in better image quality while lower numbers will result - in smaller files. 65-75 is a good starting point. The default behaviour - depends on the method used to scale the images. ImageMagick tries to estimate - the input image quality and uses 92 if it cannot be determined. Pillow - defaults to 75. +- **quality**: The JPEG quality level to use when compressing images (when + ``maxwidth`` is set). This should be either a number from 1 to 100 or 0 to + use the default quality. 65–75 is usually a good starting point. The default + behavior depends on the imaging tool used for scaling: ImageMagick tries to + estimate the input image quality and uses 92 if it cannot be determined, and + PIL defaults to 75. Default: 0 (disabled) - **remove_art_file**: Automatically remove the album art file for the album after it has been embedded. This option is best used alongside the diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 8c5d0eb7b..4441d4e30 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -42,10 +42,12 @@ file. The available options are: - **maxwidth**: A maximum image width to downscale fetched images if they are too big. The resize operation reduces image width to at most ``maxwidth`` pixels. The height is recomputed so that the aspect ratio is preserved. -- **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. - Higher numbers result in better image quality while lower numbers will result - in smaller files. 65-75 is a good starting point. +- **quality**: The JPEG quality level to use when compressing images (when + ``maxwidth`` is set). This should be either a number from 1 to 100 or 0 to + use the default quality. 65–75 is usually a good starting point. The default + behavior depends on the imaging tool used for scaling: ImageMagick tries to + estimate the input image quality and uses 92 if it cannot be determined, and + PIL defaults to 75. Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. From 594da1597a392a4917dab407198211bd23e8c2b2 Mon Sep 17 00:00:00 2001 From: smichel17 Date: Sun, 1 Mar 2020 11:45:14 -0500 Subject: [PATCH 393/613] Document running chroma plugin in verbose mode For #941 --- docs/plugins/chroma.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index 1b86073b8..58a51da2a 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -96,7 +96,9 @@ Usage Once you have all the dependencies sorted out, enable the ``chroma`` plugin in your configuration (see :ref:`using-plugins`) to benefit from fingerprinting -the next time you run ``beet import``. +the next time you run ``beet import``. The first time you do this, you may wish +to run in verbose mode (``beet -v import``) in order to verify that the +``chroma`` plugin is operational, since it does not show an indicator otherwise. You can also use the ``beet fingerprint`` command to generate fingerprints for items already in your library. (Provide a query to fingerprint a subset of your From fcc1951e7a104f4bc363070264a58278fa2b882d Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 00:16:16 +0100 Subject: [PATCH 394/613] corrected typo whilst reading --- beets/dbcore/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 3195b52c9..b13f2638a 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -189,7 +189,7 @@ class LazyConvertDict(object): class Model(object): """An abstract object representing an object in the database. Model - objects act like dictionaries (i.e., the allow subscript access like + objects act like dictionaries (i.e., they allow subscript access like ``obj['field']``). The same field set is available via attribute access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are available: From 545c65d903e38d37fd2c1734ec69eac609bea035 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 1 Mar 2020 19:35:23 -0500 Subject: [PATCH 395/613] Massage wording for #3504 --- docs/plugins/chroma.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index 58a51da2a..a6b60e6d8 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -96,9 +96,9 @@ Usage Once you have all the dependencies sorted out, enable the ``chroma`` plugin in your configuration (see :ref:`using-plugins`) to benefit from fingerprinting -the next time you run ``beet import``. The first time you do this, you may wish -to run in verbose mode (``beet -v import``) in order to verify that the -``chroma`` plugin is operational, since it does not show an indicator otherwise. +the next time you run ``beet import``. (The plugin doesn't produce any obvious +output by default. If you want to confirm that it's enabled, you can try +running in verbose mode once with ``beet -v import``.) You can also use the ``beet fingerprint`` command to generate fingerprints for items already in your library. (Provide a query to fingerprint a subset of your From d2d2b646c1bf26629663035b000b1a85ce949c56 Mon Sep 17 00:00:00 2001 From: jmwatte Date: Tue, 17 Nov 2015 12:37:15 +0100 Subject: [PATCH 396/613] Add plugin for Fish shell tab completion --- beetsplug/fish.py | 274 ++++++++++++++++++++++++++++++++++++++++++ docs/plugins/fish.rst | 43 +++++++ 2 files changed, 317 insertions(+) create mode 100644 beetsplug/fish.py create mode 100644 docs/plugins/fish.rst diff --git a/beetsplug/fish.py b/beetsplug/fish.py new file mode 100644 index 000000000..3a7682c2a --- /dev/null +++ b/beetsplug/fish.py @@ -0,0 +1,274 @@ +# This file is part of beets. +# Copyright 2015, winters jean-marie. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""If you use the fish-shell http://fishshell.com/ ... this will do +autocomplete for you. It does the main commands and options for beet +and the plugins. +It gives you all the album and itemfields (like genre, album) but not all the +values for these. It suggest genre: or album: but not genre: Pop..Jazz...Rock +You can get that by specifying ex. --extravalues genre. +""" + +from __future__ import (division, absolute_import, print_function, + unicode_literals) + +from beets.plugins import BeetsPlugin +from beets import library, ui +from beets.ui import commands +from operator import attrgetter +import os +BL_NEED2 = """complete -c beet -n '__fish_beet_needs_command' {} {}\n""" +BL_USE3 = """complete -c beet -n '__fish_beet_using_command {}' {} {}\n""" +BL_SUBS = """complete -c beet -n '__fish_at_level {} ""' {} {}\n""" +BL_EXTRA3 = """complete -c beet -n '__fish_beet_use_extra {}' {} {}\n""" + +HEAD = ''' +function __fish_beet_needs_command + set cmd (commandline -opc) + if test (count $cmd) -eq 1 + return 0 + end + return 1 +end + +function __fish_beet_using_command + set cmd (commandline -opc) + set needle (count $cmd) + if test $needle -gt 1 + if begin test $argv[1] = $cmd[2]; + and not contains -- $cmd[$needle] $FIELDS; end + return 0 + end + end + return 1 +end + +function __fish_beet_use_extra + set cmd (commandline -opc) + set needle (count $cmd) + if test $argv[2] = $cmd[$needle] + return 0 + end + return 1 +end +''' + + +class FishPlugin(BeetsPlugin): + + def commands(self): + cmd = ui.Subcommand('fish', help='make fish autocomplete beet') + cmd.func = self.run + cmd.parser.add_option('-f', '--noFields', action='store_true', + default=False, + help='no item/album fields for autocomplete') + cmd.parser.add_option( + '-e', + '--extravalues', + action='append', + type='choice', + choices=library.Item.all_keys() + + library.Album.all_keys(), + help='pick field, get field-values for autocomplete') + return [cmd] + + def run(self, lib, opts, args): + # we gather the commands from beet and from the plugins. + # we take the album and item fields. + # it wanted, we take the values from these fields. + # we make a giant string of tehm formatted in a way that + # allows fish to do autocompletion for beet. + homeDir = os.path.expanduser("~") + completePath = os.path.join(homeDir, '.config/fish/completions') + try: + os.makedirs(completePath) + except OSError: + if not os.path.isdir(completePath): + raise + pathAndFile = os.path.join(completePath, 'beet.fish') + nobasicfields = opts.noFields # do not complete for item/album fields + extravalues = opts.extravalues # ex complete all artist values + beetcmds = sorted( + (commands.default_commands + + commands.plugins.commands()), + key=attrgetter('name')) + fields = sorted(set( + library.Album.all_keys() + library.Item.all_keys())) + # collect cmds and their aliases and their help message + cmd_names_help = [] + for cmd in beetcmds: + names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names.append(cmd.name) + for name in names: + cmd_names_help.append((name, cmd.help)) + # here we go assembling the string + totstring = HEAD + "\n" + totstring += get_cmds_list([name[0] for name in cmd_names_help]) + totstring += '' if nobasicfields else get_standard_fields(fields) + totstring += get_extravalues(lib, extravalues) if extravalues else '' + totstring += "\n" + "# ====== {} =====".format( + "setup basic beet completion") + "\n" * 2 + totstring += get_basic_beet_options() + totstring += "\n" + "# ====== {} =====".format( + "setup field completion for subcommands") + "\n" + totstring += get_subcommands( + cmd_names_help, nobasicfields, extravalues) + # setup completion for all the command-options + totstring += get_all_commands(beetcmds) + + with open(pathAndFile, 'w') as fish_file: + fish_file.write(totstring) + + +def get_cmds_list(cmds_names): + # make list of all commands in beet&plugins + substr = '' + substr += ( + "set CMDS " + " ".join(cmds_names) + ("\n" * 2) + ) + return substr + + +def get_standard_fields(fields): + # make list of item/album fields & append with ':' + fields = (field + ":" for field in fields) + substr = '' + substr += ( + "set FIELDS " + " ".join(fields) + ("\n" * 2) + ) + return substr + + +def get_extravalues(lib, extravalues): + # make list of all values from a item/album field + # so type artist: and get completion for stones, beatles .. + word = '' + setOfValues = get_set_of_values_for_field(lib, extravalues) + for fld in extravalues: + extraname = fld.upper() + 'S' + word += ( + "set " + extraname + " " + " ".join(sorted(setOfValues[fld])) + + ("\n" * 2) + ) + return word + + +def get_set_of_values_for_field(lib, fields): + # get the unique values from a item/album field + dictOfFields = {} + for each in fields: + dictOfFields[each] = set() + for item in lib.items(): + for field in fields: + dictOfFields[field].add(wrap(item[field])) + return dictOfFields + + +def get_basic_beet_options(): + word = ( + BL_NEED2.format("-l format-item", + "-f -d 'print with custom format'") + + BL_NEED2.format("-l format-album", + "-f -d 'print with custom format'") + + BL_NEED2.format("-s l -l library", + "-f -r -d 'library database file to use'") + + BL_NEED2.format("-s d -l directory", + "-f -r -d 'destination music directory'") + + BL_NEED2.format("-s v -l verbose", + "-f -d 'print debugging information'") + + + BL_NEED2.format("-s c -l config", + "-f -r -d 'path to configuration file'") + + BL_NEED2.format("-s h -l help", + "-f -d 'print this help message and exit'")) + return word + + +def get_subcommands(cmd_name_and_help, nobasicfields, extravalues): + # formatting for fish to complete our fields/values + word = "" + for cmdname, cmdhelp in cmd_name_and_help: + word += "\n" + "# ------ {} -------".format( + "fieldsetups for " + cmdname) + "\n" + word += ( + BL_NEED2.format( + ("-a " + cmdname), + ("-f " + "-d " + wrap(clean_whitespace(cmdhelp))))) + + if nobasicfields is False: + word += ( + BL_USE3.format( + cmdname, + ("-a " + wrap("$FIELDS")), + ("-f " + "-d " + wrap("fieldname")))) + + if extravalues: + for f in extravalues: + setvar = wrap("$" + f.upper() + "S") + word += " ".join(BL_EXTRA3.format( + (cmdname + " " + f + ":"), + ('-f ' + '-A ' + '-a ' + setvar), + ('-d ' + wrap(f))).split()) + "\n" + return word + + +def get_all_commands(beetcmds): + # formatting for fish to complete command-options + word = "" + for cmd in beetcmds: + names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names.append(cmd.name) + for name in names: + word += "\n" + word += ("\n" * 2) + "# ====== {} =====".format( + "completions for " + name) + "\n" + + for option in cmd.parser._get_all_options()[1:]: + cmd_LO = (" -l " + option._long_opts[0].replace('--', '') + )if option._long_opts else '' + cmd_SO = (" -s " + option._short_opts[0].replace('-', '') + ) if option._short_opts else '' + cmd_needARG = ' -r ' if option.nargs in [1] else '' + cmd_helpstr = (" -d " + wrap(' '.join(option.help.split())) + ) if option.help else '' + cmd_arglist = (' -a ' + wrap(" ".join(option.choices)) + ) if option.choices else '' + + word += " ".join(BL_USE3.format( + name, + (cmd_needARG + cmd_SO + cmd_LO + " -f " + cmd_arglist), + cmd_helpstr).split()) + "\n" + + word = (word + " ".join(BL_USE3.format( + name, + ("-s " + "h " + "-l " + "help" + " -f "), + ('-d ' + wrap("print help") + "\n") + ).split())) + return word + + +def clean_whitespace(word): + # remove to much whitespace,tabs in string + return " ".join(word.split()) + + +def wrap(word): + # need " or ' around strings but watch out if they're in the string + sptoken = '\"' + if ('"') in word and ("'") in word: + word.replace('"', sptoken) + return '"' + word + '"' + + tok = '"' if "'" in word else "'" + return tok + word + tok diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst new file mode 100644 index 000000000..a6e41a46d --- /dev/null +++ b/docs/plugins/fish.rst @@ -0,0 +1,43 @@ +Fish plugins +============ + +The ``fish`` plugin adds a ``beet fish`` command that will create a fish +autocompletion file ``beet.fish`` in ``~/.config/fish/completions`` +This makes `fish`_ - a different shell - autocomplete commands for beet. + +.. _fish: http://fishshell.com/ + +Configuring +=========== + +This will only make sense if you have the `fish`_ shell installed. +Enable the ``fish`` plugin (see :ref:`using-plugins`). +If you install or disable plugins, run ``beet fish`` again. It takes the values +from the plugins you have enabled. + +Using +===== + +Type ``beet fish``. Hit ``enter`` and will see the file ``beet.fish`` appear +in ``.config/fish/completions`` in your home folder. + +For a not-fish user: After you type ``beet`` in your fish-prompt and ``TAB`` +you will get the autosuggestions for all your plugins/commands and +typing ``-`` will get you all the options available to you. +If you type ``beet ls`` and you ``TAB`` you will get a list of all the album/item +fields that beet offers. Start typing ``genr`` ``TAB`` and fish completes +``genre:`` ... ready to type on... + +Options +======= + +The default is that you get autocompletion for all the album/item fields. +You can disable that with ``beet fish -f`` In that case you only get all +the plugins/commands/options. Everything else you type in yourself. +If you want completion for a specific album/item field, you can get that like +this ``beet fish -e genre`` or ``beet fish -e genre -e albumartist`` . +Then when you type at your fish-prompt ``beet list genre:`` and you ``TAB`` +you will get a list of all your genres to choose from. +REMEMBER : we get all the values of these fields and put them in the completion +file. It is not meant to be a replacement of your database. In other words : +speed and size matters. From b53a91662361f0434232e51a54c0fcef83351eac Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 11:53:33 +0100 Subject: [PATCH 397/613] added normalize method to the Integer class --- beets/dbcore/types.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index 521a5a1ee..5aa2b9812 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -131,6 +131,14 @@ class Integer(Type): query = query.NumericQuery model_type = int + def normalize(self, value): + try: + return self.model_type(round(float(value))) + except ValueError: + return self.null + except TypeError: + return self.null + class PaddedInt(Integer): """An integer field that is formatted with a given number of digits, From 5fc4d7c35e6d508b9084e1db518f2e24cbe3af28 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 13:00:15 +0100 Subject: [PATCH 398/613] - added `field_two` as type STRING in ModelFixture1 - renamed test test_format_fixed_field to test_format_fixed_field_string - test_format_fixed_field_string not tests `field_two` with string values - added new test_format_fixed_field_integer to test field `field_one` as INTEGER - added new test_format_fixed_field_integer_normalized to test rounding float values --- test/test_dbcore.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 9bf78de67..0d40896da 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -53,6 +53,7 @@ class ModelFixture1(dbcore.Model): _fields = { 'id': dbcore.types.PRIMARY_ID, 'field_one': dbcore.types.INTEGER, + 'field_two': dbcore.types.STRING, } _types = { 'some_float_field': dbcore.types.FLOAT, @@ -355,7 +356,7 @@ class ModelTest(unittest.TestCase): def test_items(self): model = ModelFixture1(self.db) model.id = 5 - self.assertEqual({('id', 5), ('field_one', 0)}, + self.assertEqual({('id', 5), ('field_one', 0), ('field_two', '')}, set(model.items())) def test_delete_internal_field(self): @@ -370,10 +371,28 @@ class ModelTest(unittest.TestCase): class FormatTest(unittest.TestCase): - def test_format_fixed_field(self): + def test_format_fixed_field_integer(self): model = ModelFixture1() - model.field_one = u'caf\xe9' + model.field_one = 155 value = model.formatted().get('field_one') + self.assertEqual(value, u'155') + + def test_format_fixed_field_integer_normalized(self): + """The normalize method of the Integer class rounds floats + """ + model = ModelFixture1() + model.field_one = 142.432 + value = model.formatted().get('field_one') + self.assertEqual(value, u'142') + + model.field_one = 142.863 + value = model.formatted().get('field_one') + self.assertEqual(value, u'143') + + def test_format_fixed_field_string(self): + model = ModelFixture1() + model.field_two = u'caf\xe9' + value = model.formatted().get('field_two') self.assertEqual(value, u'caf\xe9') def test_format_flex_field(self): From 832c7326af8660e900988807ee2a1890cf24e67b Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 13:03:18 +0100 Subject: [PATCH 399/613] corrected test to account for `year` and `disctotal` field now being treated as `types.INTEGER` --- test/test_autotag.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 28b7fd209..5ab3c4b49 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -21,6 +21,8 @@ import re import copy import unittest +from beets.dbcore.types import Integer, PaddedInt + from test import _common from beets import autotag from beets.autotag import match @@ -91,7 +93,10 @@ class PluralityTest(_common.TestCase): for i in range(5)] likelies, _ = match.current_metadata(items) for f in fields: - self.assertEqual(likelies[f], '%s_1' % f) + if type(items[0]._fields[f]) in (Integer, PaddedInt): + self.assertEqual(likelies[f], 0) + else: + self.assertEqual(likelies[f], '%s_1' % f) def _make_item(title, track, artist=u'some artist'): From 82c3867fc086e6729218c2884f494cfefbd11f56 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 13:52:35 +0100 Subject: [PATCH 400/613] Rewrite Fish completion plugin docs & code comments --- beetsplug/fish.py | 58 ++++++++++++++++--------------- docs/plugins/fish.rst | 79 ++++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 63 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index 3a7682c2a..b81b9c387 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -1,5 +1,6 @@ # This file is part of beets. # Copyright 2015, winters jean-marie. +# Copyright 2020, Justin Mayer # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -12,12 +13,13 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""If you use the fish-shell http://fishshell.com/ ... this will do -autocomplete for you. It does the main commands and options for beet -and the plugins. -It gives you all the album and itemfields (like genre, album) but not all the -values for these. It suggest genre: or album: but not genre: Pop..Jazz...Rock -You can get that by specifying ex. --extravalues genre. +"""This plugin generates tab completions for Beets commands for the Fish shell +, including completions for Beets commands, plugin +commands, and option flags. Also generated are completions for all the album +and track fields, suggesting for example `genre:` or `album:` when querying the +Beets database. Completions for the *values* of those fields are not generated by +default but can be included via the `-e` or `--extravalues` flag. For example: +`beet fish -e genre -e albumartist` """ from __future__ import (division, absolute_import, print_function, @@ -68,11 +70,11 @@ end class FishPlugin(BeetsPlugin): def commands(self): - cmd = ui.Subcommand('fish', help='make fish autocomplete beet') + cmd = ui.Subcommand('fish', help='generate Fish shell tab completions') cmd.func = self.run cmd.parser.add_option('-f', '--noFields', action='store_true', default=False, - help='no item/album fields for autocomplete') + help='omit album/track field completions') cmd.parser.add_option( '-e', '--extravalues', @@ -80,15 +82,15 @@ class FishPlugin(BeetsPlugin): type='choice', choices=library.Item.all_keys() + library.Album.all_keys(), - help='pick field, get field-values for autocomplete') + help='include specified field *values* in completions') return [cmd] def run(self, lib, opts, args): - # we gather the commands from beet and from the plugins. - # we take the album and item fields. - # it wanted, we take the values from these fields. - # we make a giant string of tehm formatted in a way that - # allows fish to do autocompletion for beet. + # Gather the commands from Beets core and its plugins. + # Collect the album and track fields. + # If specified, also collect the values for these fields. + # Make a giant string of all the above, formatted in a way that + # allows Fish to do tab completion for the `beet` command. homeDir = os.path.expanduser("~") completePath = os.path.join(homeDir, '.config/fish/completions') try: @@ -97,22 +99,22 @@ class FishPlugin(BeetsPlugin): if not os.path.isdir(completePath): raise pathAndFile = os.path.join(completePath, 'beet.fish') - nobasicfields = opts.noFields # do not complete for item/album fields - extravalues = opts.extravalues # ex complete all artist values + nobasicfields = opts.noFields # Do not complete for album/track fields + extravalues = opts.extravalues # e.g., Also complete artists names beetcmds = sorted( (commands.default_commands + commands.plugins.commands()), key=attrgetter('name')) fields = sorted(set( library.Album.all_keys() + library.Item.all_keys())) - # collect cmds and their aliases and their help message + # Collect commands, their aliases, and their help text cmd_names_help = [] for cmd in beetcmds: names = ["\?" if alias == "?" else alias for alias in cmd.aliases] names.append(cmd.name) for name in names: cmd_names_help.append((name, cmd.help)) - # here we go assembling the string + # Concatenate the string totstring = HEAD + "\n" totstring += get_cmds_list([name[0] for name in cmd_names_help]) totstring += '' if nobasicfields else get_standard_fields(fields) @@ -124,7 +126,7 @@ class FishPlugin(BeetsPlugin): "setup field completion for subcommands") + "\n" totstring += get_subcommands( cmd_names_help, nobasicfields, extravalues) - # setup completion for all the command-options + # Set up completion for all the command options totstring += get_all_commands(beetcmds) with open(pathAndFile, 'w') as fish_file: @@ -132,7 +134,7 @@ class FishPlugin(BeetsPlugin): def get_cmds_list(cmds_names): - # make list of all commands in beet&plugins + # Make a list of all Beets core & plugin commands substr = '' substr += ( "set CMDS " + " ".join(cmds_names) + ("\n" * 2) @@ -141,7 +143,7 @@ def get_cmds_list(cmds_names): def get_standard_fields(fields): - # make list of item/album fields & append with ':' + # Make a list of album/track fields and append with ':' fields = (field + ":" for field in fields) substr = '' substr += ( @@ -151,8 +153,8 @@ def get_standard_fields(fields): def get_extravalues(lib, extravalues): - # make list of all values from a item/album field - # so type artist: and get completion for stones, beatles .. + # Make a list of all values from an album/track field. + # 'beet ls albumartist: ' yields completions for ABBA, Beatles, etc. word = '' setOfValues = get_set_of_values_for_field(lib, extravalues) for fld in extravalues: @@ -165,7 +167,7 @@ def get_extravalues(lib, extravalues): def get_set_of_values_for_field(lib, fields): - # get the unique values from a item/album field + # Get unique values from a specified album/track field dictOfFields = {} for each in fields: dictOfFields[each] = set() @@ -196,7 +198,7 @@ def get_basic_beet_options(): def get_subcommands(cmd_name_and_help, nobasicfields, extravalues): - # formatting for fish to complete our fields/values + # Formatting for Fish to complete our fields/values word = "" for cmdname, cmdhelp in cmd_name_and_help: word += "\n" + "# ------ {} -------".format( @@ -224,7 +226,7 @@ def get_subcommands(cmd_name_and_help, nobasicfields, extravalues): def get_all_commands(beetcmds): - # formatting for fish to complete command-options + # Formatting for Fish to complete command options word = "" for cmd in beetcmds: names = ["\?" if alias == "?" else alias for alias in cmd.aliases] @@ -259,12 +261,12 @@ def get_all_commands(beetcmds): def clean_whitespace(word): - # remove to much whitespace,tabs in string + # Remove excess whitespace and tabs in a string return " ".join(word.split()) def wrap(word): - # need " or ' around strings but watch out if they're in the string + # Need " or ' around strings but watch out if they're in the string sptoken = '\"' if ('"') in word and ("'") in word: word.replace('"', sptoken) diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst index a6e41a46d..b2cb096ee 100644 --- a/docs/plugins/fish.rst +++ b/docs/plugins/fish.rst @@ -1,43 +1,52 @@ -Fish plugins -============ - -The ``fish`` plugin adds a ``beet fish`` command that will create a fish -autocompletion file ``beet.fish`` in ``~/.config/fish/completions`` -This makes `fish`_ - a different shell - autocomplete commands for beet. - -.. _fish: http://fishshell.com/ - -Configuring +Fish Plugin =========== -This will only make sense if you have the `fish`_ shell installed. -Enable the ``fish`` plugin (see :ref:`using-plugins`). -If you install or disable plugins, run ``beet fish`` again. It takes the values -from the plugins you have enabled. +The ``fish`` plugin adds a ``beet fish`` command that creates a `Fish shell`_ +tab-completion file named ``beet.fish`` in ``~/.config/fish/completions``. +This enables tab-completion of ``beet`` commands for the `Fish shell`_. -Using -===== +.. _Fish shell: https://fishshell.com/ -Type ``beet fish``. Hit ``enter`` and will see the file ``beet.fish`` appear -in ``.config/fish/completions`` in your home folder. +Configuration +------------- -For a not-fish user: After you type ``beet`` in your fish-prompt and ``TAB`` -you will get the autosuggestions for all your plugins/commands and -typing ``-`` will get you all the options available to you. -If you type ``beet ls`` and you ``TAB`` you will get a list of all the album/item -fields that beet offers. Start typing ``genr`` ``TAB`` and fish completes -``genre:`` ... ready to type on... +Enable the ``fish`` plugin (see :ref:`using-plugins`) on a system running the +`Fish shell`_. + +Usage +----- + +Type ``beet fish`` to generate the ``beet.fish`` completions file at: +``~/.config/fish/completions/``. If you later install or disable plugins, run +``beet fish`` again to update the completions based on the enabled plugins. + +For users not accustomed to tab completion… After you type ``beet`` followed by +a space in your shell prompt and then the ``TAB`` key, you should see a list of +the beets commands (and their abbreviated versions) that can be invoked in your +current environment. Similarly, typing ``beet -`` will show you all the +option flags available to you, which also applies to subcommands such as +``beet import -``. If you type ``beet ls`` followed by a space and then the +and the ``TAB`` key, you will see a list of all the album/track fields that can +be used in beets queries. For example, typing ``beet ls ge`` will complete +to ``genre:`` and leave you ready to type the rest of your query. Options -======= +------- -The default is that you get autocompletion for all the album/item fields. -You can disable that with ``beet fish -f`` In that case you only get all -the plugins/commands/options. Everything else you type in yourself. -If you want completion for a specific album/item field, you can get that like -this ``beet fish -e genre`` or ``beet fish -e genre -e albumartist`` . -Then when you type at your fish-prompt ``beet list genre:`` and you ``TAB`` -you will get a list of all your genres to choose from. -REMEMBER : we get all the values of these fields and put them in the completion -file. It is not meant to be a replacement of your database. In other words : -speed and size matters. +In addition to beets commands, plugin commands, and option flags, the generated +completions also include by default all the album/track fields. If you only want +the former and do not want the album/track fields included in the generated +completions, use ``beet fish -f`` to only generate completions for beets/plugin +commands and option flags. + +If you want generated completions to also contain album/track field *values* for +the items in your library, you can use the ``-e`` or ``--extravalues`` option. +For example: ``beet fish -e genre`` or ``beet fish -e genre -e albumartist`` +In the latter case, subsequently typing ``beet list genre: `` will display +a list of all the genres in your library and ``beet list albumartist: `` +will show a list of the album artists in your library. Keep in mind that all of +these values will be put into the generated completions file, so use this option +with care when specified fields contain a large number of values. Libraries with, +for example, very large numbers of genres/artists may result in higher memory +utilization, completion latency, et cetera. This option is not meant to replace +database queries altogether. From 05db0d18eb7f01f319bb30deb6d348d0ffe5150d Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 15:38:56 +0100 Subject: [PATCH 401/613] Don't escape question marks in Fish completions Fish shell previously interpreted question marks as glob characters, but that behavior has been deprecated and will soon be removed. Plus, the completion for `help` and its alias `?` does not currently seem to behave as expected anyway and is thus, at present, of limited utility. --- beetsplug/fish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index b81b9c387..fd9753733 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -110,7 +110,7 @@ class FishPlugin(BeetsPlugin): # Collect commands, their aliases, and their help text cmd_names_help = [] for cmd in beetcmds: - names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names = [alias for alias in cmd.aliases] names.append(cmd.name) for name in names: cmd_names_help.append((name, cmd.help)) @@ -229,7 +229,7 @@ def get_all_commands(beetcmds): # Formatting for Fish to complete command options word = "" for cmd in beetcmds: - names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names = [alias for alias in cmd.aliases] names.append(cmd.name) for name in names: word += "\n" From f465c90e78d6bb102aec307057e37ad85e55422a Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 15:46:04 +0100 Subject: [PATCH 402/613] Enforce PEP-8 compliance on Fish completion plugin --- beetsplug/fish.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index fd9753733..b842ac70f 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # This file is part of beets. # Copyright 2015, winters jean-marie. # Copyright 2020, Justin Mayer @@ -17,13 +18,12 @@ , including completions for Beets commands, plugin commands, and option flags. Also generated are completions for all the album and track fields, suggesting for example `genre:` or `album:` when querying the -Beets database. Completions for the *values* of those fields are not generated by -default but can be included via the `-e` or `--extravalues` flag. For example: +Beets database. Completions for the *values* of those fields are not generated +by default but can be added via the `-e` / `--extravalues` flag. For example: `beet fish -e genre -e albumartist` """ -from __future__ import (division, absolute_import, print_function, - unicode_literals) +from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets import library, ui @@ -91,14 +91,14 @@ class FishPlugin(BeetsPlugin): # If specified, also collect the values for these fields. # Make a giant string of all the above, formatted in a way that # allows Fish to do tab completion for the `beet` command. - homeDir = os.path.expanduser("~") - completePath = os.path.join(homeDir, '.config/fish/completions') + home_dir = os.path.expanduser("~") + completion_dir = os.path.join(home_dir, '.config/fish/completions') try: - os.makedirs(completePath) + os.makedirs(completion_dir) except OSError: - if not os.path.isdir(completePath): + if not os.path.isdir(completion_dir): raise - pathAndFile = os.path.join(completePath, 'beet.fish') + completion_file_path = os.path.join(completion_dir, 'beet.fish') nobasicfields = opts.noFields # Do not complete for album/track fields extravalues = opts.extravalues # e.g., Also complete artists names beetcmds = sorted( @@ -129,7 +129,7 @@ class FishPlugin(BeetsPlugin): # Set up completion for all the command options totstring += get_all_commands(beetcmds) - with open(pathAndFile, 'w') as fish_file: + with open(completion_file_path, 'w') as fish_file: fish_file.write(totstring) @@ -156,11 +156,11 @@ def get_extravalues(lib, extravalues): # Make a list of all values from an album/track field. # 'beet ls albumartist: ' yields completions for ABBA, Beatles, etc. word = '' - setOfValues = get_set_of_values_for_field(lib, extravalues) + values_set = get_set_of_values_for_field(lib, extravalues) for fld in extravalues: extraname = fld.upper() + 'S' word += ( - "set " + extraname + " " + " ".join(sorted(setOfValues[fld])) + "set " + extraname + " " + " ".join(sorted(values_set[fld])) + ("\n" * 2) ) return word @@ -168,13 +168,13 @@ def get_extravalues(lib, extravalues): def get_set_of_values_for_field(lib, fields): # Get unique values from a specified album/track field - dictOfFields = {} + fields_dict = {} for each in fields: - dictOfFields[each] = set() + fields_dict[each] = set() for item in lib.items(): for field in fields: - dictOfFields[field].add(wrap(item[field])) - return dictOfFields + fields_dict[field].add(wrap(item[field])) + return fields_dict def get_basic_beet_options(): @@ -237,11 +237,11 @@ def get_all_commands(beetcmds): "completions for " + name) + "\n" for option in cmd.parser._get_all_options()[1:]: - cmd_LO = (" -l " + option._long_opts[0].replace('--', '') - )if option._long_opts else '' - cmd_SO = (" -s " + option._short_opts[0].replace('-', '') - ) if option._short_opts else '' - cmd_needARG = ' -r ' if option.nargs in [1] else '' + cmd_l = (" -l " + option._long_opts[0].replace('--', '') + )if option._long_opts else '' + cmd_s = (" -s " + option._short_opts[0].replace('-', '') + ) if option._short_opts else '' + cmd_need_arg = ' -r ' if option.nargs in [1] else '' cmd_helpstr = (" -d " + wrap(' '.join(option.help.split())) ) if option.help else '' cmd_arglist = (' -a ' + wrap(" ".join(option.choices)) @@ -249,7 +249,7 @@ def get_all_commands(beetcmds): word += " ".join(BL_USE3.format( name, - (cmd_needARG + cmd_SO + cmd_LO + " -f " + cmd_arglist), + (cmd_need_arg + cmd_s + cmd_l + " -f " + cmd_arglist), cmd_helpstr).split()) + "\n" word = (word + " ".join(BL_USE3.format( From 14a654bbdbbeacf2c5792d112a0ef4eb03c06a10 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 09:56:40 +0100 Subject: [PATCH 403/613] Add Fish completion to changelog & plugin index --- docs/changelog.rst | 2 ++ docs/plugins/index.rst | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7ef19871b..e33cf8c12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets * :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is resized. @@ -217,6 +218,7 @@ For packagers: the test may no longer be necessary. * This version drops support for Python 3.4. +.. _Fish shell: https://fishshell.com/ .. _MediaFile: https://github.com/beetbox/mediafile .. _Confuse: https://github.com/beetbox/confuse .. _works: https://musicbrainz.org/doc/Work diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 383466a68..6c643ce61 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -78,6 +78,7 @@ following to your configuration:: export fetchart filefilter + fish freedesktop fromfilename ftintitle @@ -184,6 +185,7 @@ Interoperability * :doc:`badfiles`: Check audio file integrity. * :doc:`embyupdate`: Automatically notifies `Emby`_ whenever the beets library changes. +* :doc:`fish`: Adds `Fish shell`_ tab autocompletion to ``beet`` commands. * :doc:`importfeeds`: Keep track of imported files via ``.m3u`` playlist file(s) or symlinks. * :doc:`ipfs`: Import libraries from friends and get albums from them via ipfs. * :doc:`kodiupdate`: Automatically notifies `Kodi`_ whenever the beets library @@ -203,6 +205,7 @@ Interoperability .. _Emby: https://emby.media +.. _Fish shell: https://fishshell.com/ .. _Plex: https://plex.tv .. _Kodi: https://kodi.tv .. _Sonos: https://sonos.com @@ -326,4 +329,4 @@ Here are a few of the plugins written by the beets community: .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser -.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ \ No newline at end of file +.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ From fbd1266bc5ed6c71dda030ef9313763c9039a376 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sat, 14 Mar 2020 21:30:22 +0100 Subject: [PATCH 404/613] simplified condition in test and added changelog entry --- docs/changelog.rst | 3 +++ test/test_autotag.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7ef19871b..fca3e686f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -166,6 +166,9 @@ Fixes: * Added a warning when configuration files defined in the `include` directive of the configuration file fail to be imported. :bug:`3498` +* Added the normalize method to the dbcore.types.INTEGER class which now + properly returns integer values. + :bug:`762` and :bug:`3507` For plugin developers: diff --git a/test/test_autotag.py b/test/test_autotag.py index 5ab3c4b49..f8f329024 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -93,7 +93,7 @@ class PluralityTest(_common.TestCase): for i in range(5)] likelies, _ = match.current_metadata(items) for f in fields: - if type(items[0]._fields[f]) in (Integer, PaddedInt): + if isinstance(likelies[f], int): self.assertEqual(likelies[f], 0) else: self.assertEqual(likelies[f], '%s_1' % f) From d5a52cbd26beee9f571fe6ea433b65bc75ee246d Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sat, 14 Mar 2020 21:51:46 +0100 Subject: [PATCH 405/613] removed unused imports --- test/test_autotag.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index f8f329024..a11bc8fac 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -21,8 +21,6 @@ import re import copy import unittest -from beets.dbcore.types import Integer, PaddedInt - from test import _common from beets import autotag from beets.autotag import match From 99a3343c0c5860f4708d5272e5e1d680dc787ad6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 14 Mar 2020 20:27:57 -0400 Subject: [PATCH 406/613] A little more detail in changelog for #3508 --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f0997271..d3379b36c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -168,8 +168,9 @@ Fixes: of the configuration file fail to be imported. :bug:`3498` * Added the normalize method to the dbcore.types.INTEGER class which now - properly returns integer values. - :bug:`762` and :bug:`3507` + properly returns integer values, which should avoid problems where fields + like ``bpm`` would sometimes store non-integer values. + :bug:`762` :bug:`3507` :bug:`3508` For plugin developers: From 8988d908c5f39d6192fed61777bb37da464a20c8 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 12:52:12 +0100 Subject: [PATCH 407/613] match method converted to instance method --- beets/dbcore/query.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 8fb64e206..239eaa965 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -156,10 +156,9 @@ class NoneQuery(FieldQuery): def col_clause(self): return self.field + " IS NULL", () - @classmethod - def match(cls, item): + def match(self, item): try: - return item[cls.field] is None + return item.get(self.field) is None except KeyError: return True From eb4d2ef5c9655da307a238b7644ec00f25d3e6f9 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 12:52:56 +0100 Subject: [PATCH 408/613] added missing abstract method --- beets/dbcore/query.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 239eaa965..1bac90c06 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -162,6 +162,10 @@ class NoneQuery(FieldQuery): except KeyError: return True + @classmethod + def value_match(cls, pattern, value): + return pattern == value + def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) From 93744ff00a10dc3ecba33545d41b8b0e57dbf3f5 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 14:40:40 +0100 Subject: [PATCH 409/613] removed unnecessary KeyError test --- beets/dbcore/query.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 1bac90c06..5835ddc32 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -157,10 +157,7 @@ class NoneQuery(FieldQuery): return self.field + " IS NULL", () def match(self, item): - try: - return item.get(self.field) is None - except KeyError: - return True + return item.get(self.field, default=None) is None @classmethod def value_match(cls, pattern, value): From d6538e5f0c0e7eae2b4e9fee8243a2724d3da066 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 14:57:02 +0100 Subject: [PATCH 410/613] removed value_match method - not reachable? --- beets/dbcore/query.py | 4 ---- test/test_query.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 5835ddc32..db4b861c4 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -159,10 +159,6 @@ class NoneQuery(FieldQuery): def match(self, item): return item.get(self.field, default=None) is None - @classmethod - def value_match(cls, pattern, value): - return pattern == value - def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) diff --git a/test/test_query.py b/test/test_query.py index c0ab2a171..5e1556a45 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -772,6 +772,25 @@ class NoneQueryTest(unittest.TestCase, TestHelper): matched = self.lib.items(NoneQuery(u'rg_track_gain')) self.assertInResult(item, matched) + def test_match_slow(self): + item = self.add_item() + matched = self.lib.items(NoneQuery(u'rg_track_peak', fast=False)) + self.assertInResult(item, matched) + + def test_match_slow_after_set_none(self): + item = self.add_item(rg_track_gain=0) + matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) + self.assertNotInResult(item, matched) + + item['rg_track_gain'] = None + item.store() + matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) + self.assertInResult(item, matched) + + def test_match_repr(self): + q = NoneQuery(u'rg_track_gain', fast=False) + self.assertEquals("NoneQuery('rg_track_gain', False)", str(q)) + class NotQueryMatchTest(_common.TestCase): """Test `query.NotQuery` matching against a single item, using the same From 238f2244c91baaa0f5a9cb16c3ec6ca4ebb8a853 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:23:52 +0100 Subject: [PATCH 411/613] value_match is now correctly implemented --- beets/dbcore/query.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index db4b861c4..03b647a0d 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -152,12 +152,16 @@ class NoneQuery(FieldQuery): def __init__(self, field, fast=True): super(NoneQuery, self).__init__(field, None, fast) + self.pattern = None def col_clause(self): return self.field + " IS NULL", () def match(self, item): - return item.get(self.field, default=None) is None + return self.value_match(self.pattern, item.get(self.field, default=None)) + + def value_match(self, pattern, value): + return value is pattern def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) From 3f2f125b090c20889bbeca28f72be4dd4ea85d3c Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:24:17 +0100 Subject: [PATCH 412/613] better repr testing --- test/test_query.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 5e1556a45..855dd8967 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -787,9 +787,11 @@ class NoneQueryTest(unittest.TestCase, TestHelper): matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) self.assertInResult(item, matched) - def test_match_repr(self): - q = NoneQuery(u'rg_track_gain', fast=False) - self.assertEquals("NoneQuery('rg_track_gain', False)", str(q)) + def test_query_repr(self): + fld = u'rg_track_gain' + self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld))) + self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld, fast=True))) + self.assertEquals("NoneQuery('{}', False)".format(str(fld)), str(NoneQuery(fld, fast=False))) class NotQueryMatchTest(_common.TestCase): From 532c6d7c825ae40a9091e32f5eb54863f591a72b Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:41:32 +0100 Subject: [PATCH 413/613] better repr testing #2 --- test/test_query.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 855dd8967..aa41497b7 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -788,10 +788,12 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) def test_query_repr(self): - fld = u'rg_track_gain' - self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld))) - self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery('{}', False)".format(str(fld)), str(NoneQuery(fld, fast=False))) + self.assertEquals("NoneQuery('rg_track_gain', True)", + str(NoneQuery(u'rg_track_gain'))) + self.assertEquals("NoneQuery('rg_track_gain', True)", + str(NoneQuery(u'rg_track_gain', fast=True))) + self.assertEquals("NoneQuery('rg_track_gain', False)", + str(NoneQuery(u'rg_track_gain', fast=False))) class NotQueryMatchTest(_common.TestCase): From 8e68b5ff2e89dbd270b7493e8153cb3c90e2fd08 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:54:01 +0100 Subject: [PATCH 414/613] cleaning up --- beets/dbcore/query.py | 3 ++- test/test_query.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 03b647a0d..e4f5b8a29 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -158,7 +158,8 @@ class NoneQuery(FieldQuery): return self.field + " IS NULL", () def match(self, item): - return self.value_match(self.pattern, item.get(self.field, default=None)) + return self.value_match(self.pattern, + item.get(self.field, default=None)) def value_match(self, pattern, value): return value is pattern diff --git a/test/test_query.py b/test/test_query.py index aa41497b7..10206eb98 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -788,11 +788,11 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) def test_query_repr(self): - self.assertEquals("NoneQuery('rg_track_gain', True)", + self.assertEquals("NoneQuery(u'rg_track_gain', True)", str(NoneQuery(u'rg_track_gain'))) - self.assertEquals("NoneQuery('rg_track_gain', True)", + self.assertEquals("NoneQuery(u'rg_track_gain', True)", str(NoneQuery(u'rg_track_gain', fast=True))) - self.assertEquals("NoneQuery('rg_track_gain', False)", + self.assertEquals("NoneQuery(u'rg_track_gain', False)", str(NoneQuery(u'rg_track_gain', fast=False))) From 935768d98326c662e9d62add0fdb9aa05d9bb05f Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 19:19:09 +0100 Subject: [PATCH 415/613] fixing repr tests --- test/test_query.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 10206eb98..ad770e1b9 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -788,12 +788,21 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) def test_query_repr(self): - self.assertEquals("NoneQuery(u'rg_track_gain', True)", - str(NoneQuery(u'rg_track_gain'))) - self.assertEquals("NoneQuery(u'rg_track_gain', True)", - str(NoneQuery(u'rg_track_gain', fast=True))) - self.assertEquals("NoneQuery(u'rg_track_gain', False)", - str(NoneQuery(u'rg_track_gain', fast=False))) + fld = u'rg_track_gain' + if sys.version_info <= (2, 7): + self.assertEquals("NoneQuery('u{}', True)".format(fld), + str(NoneQuery(fld))) + self.assertEquals("NoneQuery('u{}', True)".format(fld), + str(NoneQuery(fld, fast=True))) + self.assertEquals("NoneQuery('u{}', False)".format(fld), + str(NoneQuery(fld, fast=False))) + else: + self.assertEquals("NoneQuery('{}', True)".format(fld), + str(NoneQuery(fld))) + self.assertEquals("NoneQuery('{}', True)".format(fld), + str(NoneQuery(fld, fast=True))) + self.assertEquals("NoneQuery('{}', False)".format(fld), + str(NoneQuery(fld, fast=False))) class NotQueryMatchTest(_common.TestCase): From ceb901fcca8958c7930a79dab1830533833e477c Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 19:30:24 +0100 Subject: [PATCH 416/613] struggling with old python --- test/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_query.py b/test/test_query.py index ad770e1b9..06f2deec2 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -789,7 +789,7 @@ class NoneQueryTest(unittest.TestCase, TestHelper): def test_query_repr(self): fld = u'rg_track_gain' - if sys.version_info <= (2, 7): + if sys.version_info <= (3, 0): self.assertEquals("NoneQuery('u{}', True)".format(fld), str(NoneQuery(fld))) self.assertEquals("NoneQuery('u{}', True)".format(fld), From ac1a3851faa04fb7bca41bc5cbbdf403c42c0cc9 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 19:36:38 +0100 Subject: [PATCH 417/613] typo fix --- test/test_query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 06f2deec2..fc8e76735 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -790,11 +790,11 @@ class NoneQueryTest(unittest.TestCase, TestHelper): def test_query_repr(self): fld = u'rg_track_gain' if sys.version_info <= (3, 0): - self.assertEquals("NoneQuery('u{}', True)".format(fld), + self.assertEquals("NoneQuery(u'{}', True)".format(fld), str(NoneQuery(fld))) - self.assertEquals("NoneQuery('u{}', True)".format(fld), + self.assertEquals("NoneQuery(u'{}', True)".format(fld), str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery('u{}', False)".format(fld), + self.assertEquals("NoneQuery(u'{}', False)".format(fld), str(NoneQuery(fld, fast=False))) else: self.assertEquals("NoneQuery('{}', True)".format(fld), From 4c6993989caf9e01d269ae03715adb6a175b23fd Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 20:57:47 +0100 Subject: [PATCH 418/613] changelog entry --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d3379b36c..457275763 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -171,6 +171,8 @@ Fixes: properly returns integer values, which should avoid problems where fields like ``bpm`` would sometimes store non-integer values. :bug:`762` :bug:`3507` :bug:`3508` +* Removed `@classmethod`` decorator from dbcore.query.NoneQuery.match method + failing with AttributeError when called. It is now an instance method.` For plugin developers: From 4ba7b8da313a535da8ad9972cc984557c04a5909 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 16 Mar 2020 10:12:09 +0100 Subject: [PATCH 419/613] sampsyo's docs/changelog.rst correction Co-Authored-By: Adrian Sampson --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 457275763..a51c85cad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -171,8 +171,9 @@ Fixes: properly returns integer values, which should avoid problems where fields like ``bpm`` would sometimes store non-integer values. :bug:`762` :bug:`3507` :bug:`3508` -* Removed `@classmethod`` decorator from dbcore.query.NoneQuery.match method - failing with AttributeError when called. It is now an instance method.` +* Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method + failing with AttributeError when called. It is now an instance method. + :bug:`3516` :bug:`3517` For plugin developers: From 611659d03c6e75b25aea994cf9daa0732df61679 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 16 Mar 2020 10:16:18 +0100 Subject: [PATCH 420/613] removed value_match and repr tests --- beets/dbcore/query.py | 7 +------ test/test_query.py | 17 ----------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index e4f5b8a29..db4b861c4 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -152,17 +152,12 @@ class NoneQuery(FieldQuery): def __init__(self, field, fast=True): super(NoneQuery, self).__init__(field, None, fast) - self.pattern = None def col_clause(self): return self.field + " IS NULL", () def match(self, item): - return self.value_match(self.pattern, - item.get(self.field, default=None)) - - def value_match(self, pattern, value): - return value is pattern + return item.get(self.field, default=None) is None def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) diff --git a/test/test_query.py b/test/test_query.py index fc8e76735..f88a12c92 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -787,23 +787,6 @@ class NoneQueryTest(unittest.TestCase, TestHelper): matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) self.assertInResult(item, matched) - def test_query_repr(self): - fld = u'rg_track_gain' - if sys.version_info <= (3, 0): - self.assertEquals("NoneQuery(u'{}', True)".format(fld), - str(NoneQuery(fld))) - self.assertEquals("NoneQuery(u'{}', True)".format(fld), - str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery(u'{}', False)".format(fld), - str(NoneQuery(fld, fast=False))) - else: - self.assertEquals("NoneQuery('{}', True)".format(fld), - str(NoneQuery(fld))) - self.assertEquals("NoneQuery('{}', True)".format(fld), - str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery('{}', False)".format(fld), - str(NoneQuery(fld, fast=False))) - class NotQueryMatchTest(_common.TestCase): """Test `query.NotQuery` matching against a single item, using the same From b34d1f71a93b3663f70f46a4d5ed08702b14c5ca Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 16 Mar 2020 09:08:57 -0400 Subject: [PATCH 421/613] Slightly terser `get` call for #3517 --- beets/dbcore/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index db4b861c4..4f19f4f8d 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -157,7 +157,7 @@ class NoneQuery(FieldQuery): return self.field + " IS NULL", () def match(self, item): - return item.get(self.field, default=None) is None + return item.get(self.field) is None def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) From 19d6dfc8f301c4123989227084842d1c25807594 Mon Sep 17 00:00:00 2001 From: x1ppy <> Date: Thu, 23 Jan 2020 15:07:26 -0500 Subject: [PATCH 422/613] Support extra tags for MusicBrainz queries --- beets/autotag/hooks.py | 16 +++++++++++----- beets/autotag/match.py | 9 ++++++++- beets/autotag/mb.py | 22 ++++++++++++++++++++-- beets/config_default.yaml | 1 + beets/plugins.py | 7 ++++--- docs/changelog.rst | 2 ++ docs/reference/config.rst | 20 ++++++++++++++++++++ test/test_importer.py | 2 +- 8 files changed, 67 insertions(+), 12 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 030f371ba..d7c701db6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -614,17 +614,21 @@ def tracks_for_id(track_id): @plugins.notify_info_yielded(u'albuminfo_received') -def album_candidates(items, artist, album, va_likely): +def album_candidates(items, artist, album, va_likely, extra_tags): """Search for album matches. ``items`` is a list of Item objects that make up the album. ``artist`` and ``album`` are the respective names (strings), which may be derived from the item list or may be entered by the user. ``va_likely`` is a boolean indicating whether - the album is likely to be a "various artists" release. + the album is likely to be a "various artists" release. ``extra_tags`` + is an optional dictionary of additional tags used to further + constrain the search. """ + # Base candidates if we have album and artist to match. if artist and album: try: - for candidate in mb.match_album(artist, album, len(items)): + for candidate in mb.match_album(artist, album, len(items), + extra_tags): yield candidate except mb.MusicBrainzAPIError as exc: exc.log(log) @@ -632,13 +636,15 @@ def album_candidates(items, artist, album, va_likely): # Also add VA matches from MusicBrainz where appropriate. if va_likely and album: try: - for candidate in mb.match_album(None, album, len(items)): + for candidate in mb.match_album(None, album, len(items), + extra_tags): yield candidate except mb.MusicBrainzAPIError as exc: exc.log(log) # Candidates from plugins. - for candidate in plugins.candidates(items, artist, album, va_likely): + for candidate in plugins.candidates(items, artist, album, va_likely, + extra_tags): yield candidate diff --git a/beets/autotag/match.py b/beets/autotag/match.py index 71b62adb7..f57cac739 100644 --- a/beets/autotag/match.py +++ b/beets/autotag/match.py @@ -447,6 +447,12 @@ def tag_album(items, search_artist=None, search_album=None, search_artist, search_album = cur_artist, cur_album log.debug(u'Search terms: {0} - {1}', search_artist, search_album) + extra_tags = None + if config['musicbrainz']['extra_tags']: + tag_list = config['musicbrainz']['extra_tags'].get() + extra_tags = {k: v for (k, v) in likelies.items() if k in tag_list} + log.debug(u'Additional search terms: {0}', extra_tags) + # Is this album likely to be a "various artist" release? va_likely = ((not consensus['artist']) or (search_artist.lower() in VA_ARTISTS) or @@ -457,7 +463,8 @@ def tag_album(items, search_artist=None, search_album=None, for matched_candidate in hooks.album_candidates(items, search_artist, search_album, - va_likely): + va_likely, + extra_tags): _add_candidate(items, candidates, matched_candidate) log.debug(u'Evaluating {0} candidates.', len(candidates)) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 1a6e0b1f1..f86d3be71 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -38,6 +38,14 @@ else: SKIPPED_TRACKS = ['[data track]'] +FIELDS_TO_MB_KEYS = { + 'catalognum': 'catno', + 'country': 'country', + 'label': 'label', + 'media': 'format', + 'year': 'date', +} + musicbrainzngs.set_useragent('beets', beets.__version__, 'https://beets.io/') @@ -411,13 +419,13 @@ def album_info(release): return info -def match_album(artist, album, tracks=None): +def match_album(artist, album, tracks=None, extra_tags=None): """Searches for a single album ("release" in MusicBrainz parlance) and returns an iterator over AlbumInfo objects. May raise a MusicBrainzAPIError. The query consists of an artist name, an album name, and, - optionally, a number of tracks on the album. + optionally, a number of tracks on the album and any other extra tags. """ # Build search criteria. criteria = {'release': album.lower().strip()} @@ -429,6 +437,16 @@ def match_album(artist, album, tracks=None): if tracks is not None: criteria['tracks'] = six.text_type(tracks) + # Additional search cues from existing metadata. + if extra_tags: + for tag in extra_tags: + key = FIELDS_TO_MB_KEYS[tag] + value = six.text_type(extra_tags.get(tag, '')).lower().strip() + if key == 'catno': + value = value.replace(u' ', '') + if value: + criteria[key] = value + # Abort if we have no search terms. if not any(criteria.values()): return diff --git a/beets/config_default.yaml b/beets/config_default.yaml index cf9ae6bf9..0fd6eb592 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -103,6 +103,7 @@ musicbrainz: ratelimit: 1 ratelimit_interval: 1.0 searchlimit: 5 + extra_tags: [] match: strong_rec_thresh: 0.04 diff --git a/beets/plugins.py b/beets/plugins.py index 73d85cdd3..8606ebc69 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -172,7 +172,7 @@ class BeetsPlugin(object): """ return beets.autotag.hooks.Distance() - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): """Should return a sequence of AlbumInfo objects that match the album whose items are provided. """ @@ -379,11 +379,12 @@ def album_distance(items, album_info, mapping): return dist -def candidates(items, artist, album, va_likely): +def candidates(items, artist, album, va_likely, extra_tags=None): """Gets MusicBrainz candidates for an album from each plugin. """ for plugin in find_plugins(): - for candidate in plugin.candidates(items, artist, album, va_likely): + for candidate in plugin.candidates(items, artist, album, va_likely, + extra_tags): yield candidate diff --git a/docs/changelog.rst b/docs/changelog.rst index a51c85cad..3769164be 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog New features: +* A new :ref:`extra_tags` configuration option allows more tagged metadata + to be included in MusicBrainz queries. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets * :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 7dcd53801..46f14f2c5 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -701,6 +701,26 @@ MusicBrainz server. Default: ``5``. +.. _extra_tags: + +extra_tags +~~~~~~~~~~ + +By default, beets will use only the artist, album, and track count to query +MusicBrainz. Additional tags to be queried can be supplied with the +``extra_tags`` setting. For example:: + + musicbrainz: + extra_tags: [year, catalognum, country, media, label] + +This setting should improve the autotagger results if the metadata with the +given tags match the metadata returned by MusicBrainz. + +Note that the only tags supported by this setting are the ones listed in the +above example. + +Default: ``[]`` + .. _match-config: Autotagger Matching Options diff --git a/test/test_importer.py b/test/test_importer.py index 8f637a077..3418d4628 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -79,7 +79,7 @@ class AutotagStub(object): autotag.mb.album_for_id = self.mb_album_for_id autotag.mb.track_for_id = self.mb_track_for_id - def match_album(self, albumartist, album, tracks): + def match_album(self, albumartist, album, tracks, extra_tags): if self.matching == self.IDENT: yield self._make_album_match(albumartist, album, tracks) From f306591a99f0a153d3116f36aaffa0ed5e5d4788 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sat, 28 Mar 2020 17:36:02 +1000 Subject: [PATCH 423/613] add the extra_tags option to all required plugins --- beetsplug/beatport.py | 2 +- beetsplug/chroma.py | 2 +- beetsplug/cue.py | 2 +- beetsplug/discogs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 6a45ab93a..df0abb2fc 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -355,7 +355,7 @@ class BeatportPlugin(BeetsPlugin): config=self.config ) - def candidates(self, items, artist, release, va_likely): + def candidates(self, items, artist, release, va_likely, extra_tags=None): """Returns a list of AlbumInfo objects for beatport search results matching release and artist (if not various). """ diff --git a/beetsplug/chroma.py b/beetsplug/chroma.py index c4230b069..54ae90098 100644 --- a/beetsplug/chroma.py +++ b/beetsplug/chroma.py @@ -191,7 +191,7 @@ class AcoustidPlugin(plugins.BeetsPlugin): dist.add_expr('track_id', info.track_id not in recording_ids) return dist - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): albums = [] for relid in prefix(_all_releases(items), MAX_RELEASES): album = hooks.album_for_mbid(relid) diff --git a/beetsplug/cue.py b/beetsplug/cue.py index fd564b55c..92ca8784a 100644 --- a/beetsplug/cue.py +++ b/beetsplug/cue.py @@ -24,7 +24,7 @@ class CuePlugin(BeetsPlugin): # self.register_listener('import_task_start', self.look_for_cues) - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): import pdb pdb.set_trace() diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 0ba27d7dd..86ace9aa8 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -175,7 +175,7 @@ class DiscogsPlugin(BeetsPlugin): config=self.config ) - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): """Returns a list of AlbumInfo objects for discogs search results matching an album and artist (if not various). """ From 333d5d1dd3a1e578427328a3ce5a09ee342fcd4c Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 30 Mar 2020 20:08:40 +0100 Subject: [PATCH 424/613] fetchart: Add Last.fm artwork source --- beetsplug/fetchart.py | 70 ++++++++++++++++++++++++++++++++++++++- docs/changelog.rst | 3 ++ docs/plugins/fetchart.rst | 17 ++++++++-- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 2fe8b0b2c..28067c310 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -21,6 +21,7 @@ from contextlib import closing import os import re from tempfile import NamedTemporaryFile +from collections import OrderedDict import requests @@ -742,11 +743,72 @@ class FileSystem(LocalArtSource): match=Candidate.MATCH_FALLBACK) +class LastFM(RemoteArtSource): + NAME = u"Last.fm" + + # Sizes in priority order. + SIZES = OrderedDict([ + ('mega', (300, 300)), + ('extralarge', (300, 300)), + ('large', (174, 174)), + ('medium', (64, 64)), + ('small', (34, 34)), + ]) + + if util.SNI_SUPPORTED: + API_URL = 'https://ws.audioscrobbler.com/2.0' + else: + API_URL = 'http://ws.audioscrobbler.com/2.0' + + def __init__(self, *args, **kwargs): + super(LastFM, self).__init__(*args, **kwargs) + self.key = self._config['lastfm_key'].get(), + + def get(self, album, plugin, paths): + if not album.mb_albumid: + return + + try: + response = self.request(self.API_URL, params={ + 'method': 'album.getinfo', + 'api_key': self.key, + 'mbid': album.mb_albumid, + 'format': 'json', + }) + except requests.RequestException: + self._log.debug(u'lastfm: error receiving response') + return + + try: + data = response.json() + + if 'error' in data: + if data['error'] == 6: + self._log.debug('lastfm: no results for {}', + album.mb_albumid) + else: + self._log.error( + 'lastfm: failed to get album info: {} ({})', + data['message'], data['error']) + else: + images = {image['size']: image['#text'] + for image in data['album']['image']} + + # Provide candidates in order of size. + for size in self.SIZES.keys(): + if size in images: + yield self._candidate(url=images[size], + size=self.SIZES[size]) + except ValueError: + self._log.debug(u'lastfm: error loading response: {}' + .format(response.text)) + return + # Try each source in turn. SOURCES_ALL = [u'filesystem', u'coverart', u'itunes', u'amazon', u'albumart', - u'wikipedia', u'google', u'fanarttv'] + u'wikipedia', u'google', u'fanarttv', u'lastfm'] ART_SOURCES = { u'filesystem': FileSystem, @@ -757,6 +819,7 @@ ART_SOURCES = { u'wikipedia': Wikipedia, u'google': GoogleImages, u'fanarttv': FanartTV, + u'lastfm': LastFM, } SOURCE_NAMES = {v: k for k, v in ART_SOURCES.items()} @@ -787,11 +850,13 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'google_key': None, 'google_engine': u'001442825323518660753:hrh5ch1gjzm', 'fanarttv_key': None, + 'lastfm_key': None, 'store_source': False, 'high_resolution': False, }) self.config['google_key'].redact = True self.config['fanarttv_key'].redact = True + self.config['lastfm_key'].redact = True self.minwidth = self.config['minwidth'].get(int) self.maxwidth = self.config['maxwidth'].get(int) @@ -831,6 +896,9 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): if not self.config['google_key'].get() and \ u'google' in available_sources: available_sources.remove(u'google') + if not self.config['lastfm_key'].get() and \ + u'lastfm' in available_sources: + available_sources.remove(u'lastfm') available_sources = [(s, c) for s in available_sources for c in ART_SOURCES[s].VALID_MATCHING_CRITERIA] diff --git a/docs/changelog.rst b/docs/changelog.rst index 3769164be..33ad386ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -113,6 +113,8 @@ New features: titles. Thanks to :user:`cole-miller`. :bug:`3459` +* :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. + :bug:`3530` Fixes: @@ -233,6 +235,7 @@ For packagers: .. _works: https://musicbrainz.org/doc/Work .. _Deezer: https://www.deezer.com .. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli +.. _last.fm: https://last.fm 1.4.9 (May 30, 2019) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 4441d4e30..e8f7b6d92 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -58,9 +58,9 @@ file. The available options are: - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. Default: ``filesystem coverart itunes amazon albumart``, i.e., everything but - ``wikipedia``, ``google`` and ``fanarttv``. Enable those sources for more - matches at the cost of some speed. They are searched in the given order, - thus in the default config, no remote (Web) art source are queried if + ``wikipedia``, ``google``, ``fanarttv`` and ``lastfm``. Enable those sources + for more matches at the cost of some speed. They are searched in the given + order, thus in the default config, no remote (Web) art source are queried if local art is found in the filesystem. To use a local image as fallback, move it to the end of the list. For even more fine-grained control over the search order, see the section on :ref:`album-art-sources` below. @@ -71,6 +71,8 @@ file. The available options are: Default: The `beets custom search engine`_, which searches the entire web. - **fanarttv_key**: The personal API key for requesting art from fanart.tv. See below. +- **lastfm_key**: The personal API key for requesting art from Last.fm. See + below. - **store_source**: If enabled, fetchart stores the artwork's source in a flexible tag named ``art_source``. See below for the rationale behind this. Default: ``no``. @@ -221,6 +223,15 @@ personal key will give you earlier access to new art. .. _on their blog: https://fanart.tv/2015/01/personal-api-keys/ +Last.fm +''''''' + +To use the Last.fm backend, you need to `register for a Last.fm API key`_. Set +the ``lastfm_key`` configuration option to your API key, then add ``lastfm`` to +the list of sources in your configutation. + +.. _register for a Last.fm API key: https://www.last.fm/api/account/create + Storing the Artwork's Source ---------------------------- From 1bec3c4c9f6f29289c1b0f3709289cc2cdccc6ad Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 30 Mar 2020 22:15:13 +0200 Subject: [PATCH 425/613] added all my plugins + corrected 1 typo --- docs/plugins/index.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 6c643ce61..76ed19fa3 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -300,12 +300,20 @@ Here are a few of the plugins written by the beets community: * `beet-summarize`_ can compute lots of counts and statistics about your music library. -* `beets-mosaic`_ generates a montage of a mosiac from cover art. +* `beets-mosaic`_ generates a montage of a mosaic from cover art. + +* `beets-goingrunning`_ generates playlists to go with your running sessions. + +* `beets-xtractor`_ extracts low- and high-level musical information from your songs. + +* `beets-yearfixer`_ attempts to fix all missing ``original_year`` and ``year`` fields. + +* `beets-autofix`_ automates repetitive tasks to keep your library in order. + +* `beets-describe`_ gives you the full picture of a single attribute of your library items. * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). -* `beets-goingrunning`_ copies songs to external device to go with your running session. - .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -328,5 +336,9 @@ Here are a few of the plugins written by the beets community: .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic +.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning +.. _beets-xtractor: https://github.com/adamjakab/BeetsPluginXtractor +.. _beets-yearfixer: https://github.com/adamjakab/BeetsPluginYearFixer +.. _beets-autofix: https://github.com/adamjakab/BeetsPluginAutofix +.. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser -.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ From 8908416f6f66d2596a08daefeba0144cb27a7b98 Mon Sep 17 00:00:00 2001 From: x1ppy <59340737+x1ppy@users.noreply.github.com> Date: Mon, 6 Apr 2020 17:00:17 -0400 Subject: [PATCH 426/613] Add beets-originquery plugin --- docs/plugins/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 76ed19fa3..1247d0ed0 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -314,6 +314,9 @@ Here are a few of the plugins written by the beets community: * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). +* `beets-originquery`_ augments MusicBrainz queries with locally-sourced data + to improve autotagger results. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -342,3 +345,4 @@ Here are a few of the plugins written by the beets community: .. _beets-autofix: https://github.com/adamjakab/BeetsPluginAutofix .. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser +.. _beets-originquery: https://github.com/x1ppy/beets-originquery From 3c089117940de5d66504400db8b37603d98b60bf Mon Sep 17 00:00:00 2001 From: x1ppy <59340737+x1ppy@users.noreply.github.com> Date: Mon, 6 Apr 2020 17:01:11 -0400 Subject: [PATCH 427/613] Add missing extra_tags parameter to MetadataSourcePlugin (#3540) --- beets/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 8606ebc69..695725cb8 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -715,7 +715,7 @@ class MetadataSourcePlugin(object): return id_ return None - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): """Returns a list of AlbumInfo objects for Search API results matching an ``album`` and ``artist`` (if not various). From 8e28f0b694bc04fa0735c8d3f6feb8232647c8d7 Mon Sep 17 00:00:00 2001 From: lijacky Date: Sun, 12 Apr 2020 00:05:17 -0400 Subject: [PATCH 428/613] added null check for genius lyrics scrape --- beetsplug/lyrics.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0e797d5a3..bb225007d 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,7 +373,13 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. - lyrics = html.find("div", class_="lyrics").get_text() + try: + lyrics = html.find("div", class_="lyrics").get_text() + except AttributeError as exc: + # html is a NoneType cannot retrieve lyrics + self._log.debug(u'Genius lyrics for {0} not found: {1}', + page_url, exc) + return None return lyrics From 29d7b80847fd4f1d5c4f47d50d758160d6866c39 Mon Sep 17 00:00:00 2001 From: lijacky Date: Wed, 15 Apr 2020 22:23:27 -0400 Subject: [PATCH 429/613] Added unit test for null check --- test/test_lyrics.py | 61 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index f7ea538e2..e72b2d4f8 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -24,7 +24,7 @@ import sys import unittest from mock import patch -from test import _common +import _common from beets import logging from beets.library import Item @@ -39,6 +39,7 @@ from mock import MagicMock log = logging.getLogger('beets.test_lyrics') raw_backend = lyrics.Backend({}, log) google = lyrics.Google(MagicMock(), log) +genius = lyrics.Genius(MagicMock(), log) class LyricsPluginTest(unittest.TestCase): @@ -213,6 +214,29 @@ class MockFetchUrl(object): content = f.read() return content +class GeniusMockGet(object): + def __init__(self, pathval='fetched_path'): + self.pathval = pathval + self.fetched = None + + def __call__(self, url, headers=False): + from requests.models import Response + # for the first requests.get() return a path + if headers: + response = Response() + response.status_code = 200 + response._content = b'{"meta":{"status":200},"response":{"song":{"path":"/lyrics/sample"}}}' + return response + # for the second requests.get() return the genius page + else: + from mock import PropertyMock + self.fetched = url + fn = url_to_filename(url) + with open(fn, 'r') as f: + content = f.read() + response = Response() + type(response).text = PropertyMock(return_value=content) + return response def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" @@ -395,6 +419,41 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): google.is_page_candidate(url, url_title, s['title'], u'Sunn O)))') +class LyricsGeniusBaseTest(unittest.TestCase): + + def setUp(self): + """Set up configuration.""" + try: + __import__('bs4') + except ImportError: + self.skipTest('Beautiful Soup 4 not available') + if sys.version_info[:3] < (2, 7, 3): + self.skipTest("Python's built-in HTML parser is not good enough") + + +class LyricsGeniusScrapTest(LyricsGeniusBaseTest): + """Checks that Genius backend works as intended. + """ + import requests + def setUp(self): + """Set up configuration""" + LyricsGeniusBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @patch.object(requests, 'get', GeniusMockGet()) + def test_no_lyrics_div(self): + """Ensure that `lyrics_from_song_api_path` doesn't crash when the html + for a Genius page contain
...
+ """ + # https://github.com/beetbox/beets/issues/3535 + # expected return value None + try: + self.assertEqual(genius.lyrics_from_song_api_path('/no_lyric_page'), None) + except AttributeError: + # if AttributeError we aren't doing a null check + self.assertTrue(False) + + class SlugTests(unittest.TestCase): def test_slug(self): From 525202e52919e971781a867f20d49b1548161d43 Mon Sep 17 00:00:00 2001 From: lijacky Date: Wed, 15 Apr 2020 22:30:31 -0400 Subject: [PATCH 430/613] adding genius sample html --- test/rsrc/lyrics/geniuscom/sample.txt | 1119 +++++++++++++++++++++++++ 1 file changed, 1119 insertions(+) create mode 100644 test/rsrc/lyrics/geniuscom/sample.txt diff --git a/test/rsrc/lyrics/geniuscom/sample.txt b/test/rsrc/lyrics/geniuscom/sample.txt new file mode 100644 index 000000000..da8a4d0b6 --- /dev/null +++ b/test/rsrc/lyrics/geniuscom/sample.txt @@ -0,0 +1,1119 @@ + + + + + + + +SAMPLE – SONG Lyrics | Genius Lyrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + {{:: 'cloud_flare_always_on_short_message' | i18n }} +
Check @genius for updates. We'll have things fixed soon. + +
+
+ + +
+ GENIUS +
+ + + +
+ + + + + + + + + + + +
+ + +
+ + + +
+ + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+
+
+ Https%3a%2f%2fimages +
+
+ +
+
+ +

SONG

+

+ SAMPLE +

+ +

+ + +

+

+ +
+ Produced by + + + Carter Lang & The Antydote + +
+ + +

+

+ +
+ Album + ALBUM +
+ + +

+ +
+
+
+
+
+
+
+ + +
+
+
+ +

SONG Lyrics

+ +
+
+ + +

[Verse 1]
+Fearin' not growin' up
+Keepin' me up at night
+Am I doin' enough?
+Feel like I'm wastin' time
+
+[Pre-Chorus]
+Promise to get a little
+Better as I get older
+And you're so patient
+And sick of waitin'
+Promise to do better
+Shoulda coulda
+Prolly wanna let me go
+But you can't, oh
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+
+[Chorus]
+Please don't take it, don't take it personal
+Like I know you usually do
+Please don't take it, don't take it personal
+Like I know you usually do
+Please, please
+Don't take it personal
+Don't take it personal
+Darling, like I know you will, ooh
+
+[Verse 2]
+Forget to call your mama on the weekend
+You should put yourself in time out
+(Shame, shame on you)
+But lately you've been feelin' so good
+I forget my future, never pull out
+(Shame, shame on me)
+Baby the money'll make it easier for me
+To run and hide out somewhere
+(So far away)
+Hoppin' through poppy fields
+Dodgin' evil witches
+These houses keep droppin' everywhere
+
+[Pre-Chorus]
+Promise to get a little
+Better as I get older
+And you're so patient
+And sick of waitin'
+Promise to do better
+Shoulda coulda
+Prolly wanna let me go
+But you can't, oh
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+Right now it's really pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+
+[Chorus]
+Please don't take it, don't take it personal
+Like I know you usually do
+Please don't take it, take it personal
+
+[Outro]
+Like winters fall on us, heavy
+Take it off me, all it off
+Winter, I can't stand this
+Snow is falling all on me

+ + + + +
+
+ +
+ +
+
More on Genius
+ +
+ +
+
+
+ +
+
+ +
+
+
+ + + +
+ +

+ About “SONG” +

+ + +
+
+

The fifth track on SAMPLE’s debut album, ALBUM is the most reminiscent of her previous ep’s with a pop-disco sound.

+ +

She reflects about her insecurities on not maturing as quickly as her partner is. SONG season is usually the last big event before graduation for a high schooler, and SAMPLE is too busy living in the moment that she forgets she has her future to look forward to.

+ +

+ +

This is also around the age when friends are growing apart and have decided where and what they’re doing with their lives, and a lot are trying to hold on to the people they’ve known, even though they’re growing up differently. The person SAMPLE is talking to in the song seems to take everything, even change, to heart.

+
+
+
+ +
+
+
    + +
  • +
    +

    What have the artists said about this song

    +
    + +
    +

    SAMPLE tweeted:

    + +

    Iss called SONG cus I ain’t have no friends in high school so I went to Miami w my mama for SONG instead . Lol

    +
    + + +
    • + + + +
+
+ + +
+
+

"SONG" Track Info

+ + +
+ Written By + + + The Antydote, Carter Lang & SAMPLE + +
+ + + + + +
+ Released by + + + RCA Records & Top Dawg Entertainment + +
+ + + + +
+ Mixing + + + Mixed by Ali + +
+ + + + + +
+ Recorded At + TDE Red Room, Carson, California +
+ + + +
+ Release Date + June 9, 2017 +
+ + + + + + + + + + + + + + + + + + +
+ Interpolated By + + +
+ Wicked Games (StarFox Remix) by The Weeknd +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+
+
+ + Https%3a%2f%2fimages + +
+ + ALBUM + + + SAMPLE +
+
+ +
+ +
+
+ + 1.   + + + + # + + + + +
+
+ +
+
+ + 2.   + + + + # + + + + +
+
+ +
+
+ + 3.   + + + + # + + + + +
+
+ +
+
+ + 4.   + + + + # + + + + +
+
+ +
+
+ + 5.   + + + + SONG + + + + +
+
+ +
+
+ + 6.   + + + + # + + + + +
+
+ +
+
+ + 7.   + + + + # + + + + +
+
+ +
+
+ + 8.   + + + + # + + + + +
+
+ +
+
+ + 9.   + + + + # + + + + +
+
+ +
+
+ + 10.   + + + + # + + + + +
+
+ +
+
+ + 11.   + + + + # + + + + +
+
+ +
+
+ + 12.   + + + + # + + + + +
+
+ +
+
+ + 13.   + + + + # + + + + +
+
+ +
+
+ + 14.   + + + + # + + + + +
+
+ + + +
+ +
+
+ +
+
+
+ + + + + + + + + +
+
+ + + + +
+ © 2020 Genius Media Group Inc. +
+
+ +
+ Verified Artists + + All Artists: +
    + +
  • + A +
    • + +
  • + B +
    • + +
  • + C +
    • + +
  • + D +
    • + +
  • + E +
    • + +
  • + F +
    • + +
  • + G +
    • + +
  • + H +
    • + +
  • + I +
    • + +
  • + J +
    • + +
  • + K +
    • + +
  • + L +
    • + +
  • + M +
    • + +
  • + N +
    • + +
  • + O +
    • + +
  • + P +
    • + +
  • + Q +
    • + +
  • + R +
    • + +
  • + S +
    • + +
  • + T +
    • + +
  • + U +
    • + +
  • + V +
    • + +
  • + W +
    • + +
  • + X +
    • + +
  • + Y +
    • + +
  • + Z +
    • + +
  • + # +
    • + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + From b7ecf32f2876848611dcced77c0a2624198d35a9 Mon Sep 17 00:00:00 2001 From: lijacky Date: Thu, 16 Apr 2020 17:20:08 -0400 Subject: [PATCH 431/613] style changes --- test/test_lyrics.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index e72b2d4f8..6c23c627a 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -24,7 +24,7 @@ import sys import unittest from mock import patch -import _common +from test import _common from beets import logging from beets.library import Item @@ -214,7 +214,9 @@ class MockFetchUrl(object): content = f.read() return content + class GeniusMockGet(object): + def __init__(self, pathval='fetched_path'): self.pathval = pathval self.fetched = None @@ -225,7 +227,8 @@ class GeniusMockGet(object): if headers: response = Response() response.status_code = 200 - response._content = b'{"meta":{"status":200},"response":{"song":{"path":"/lyrics/sample"}}}' + response._content = b'{"meta":{"status":200},\ + "response":{"song":{"path":"/lyrics/sample"}}}' return response # for the second requests.get() return the genius page else: @@ -238,6 +241,7 @@ class GeniusMockGet(object): type(response).text = PropertyMock(return_value=content) return response + def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" if not text: @@ -432,9 +436,11 @@ class LyricsGeniusBaseTest(unittest.TestCase): class LyricsGeniusScrapTest(LyricsGeniusBaseTest): + """Checks that Genius backend works as intended. """ import requests + def setUp(self): """Set up configuration""" LyricsGeniusBaseTest.setUp(self) @@ -443,12 +449,13 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): """Ensure that `lyrics_from_song_api_path` doesn't crash when the html - for a Genius page contain
...
+ for a Genius page contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None try: - self.assertEqual(genius.lyrics_from_song_api_path('/no_lyric_page'), None) + self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + None) except AttributeError: # if AttributeError we aren't doing a null check self.assertTrue(False) From 9ec0d725e52a06863ac733e2f560dbc2b08fc014 Mon Sep 17 00:00:00 2001 From: lijacky Date: Fri, 17 Apr 2020 17:14:21 -0400 Subject: [PATCH 432/613] Changes given feedback on https://github.com/beetbox/beets/pull/3554 and trimmed sample html --- beetsplug/lyrics.py | 13 +- test/rsrc/lyrics/geniuscom/sample.txt | 1289 +++++-------------------- test/test_lyrics.py | 8 +- 3 files changed, 229 insertions(+), 1081 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index bb225007d..00b8820f4 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,13 +373,14 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. - try: - lyrics = html.find("div", class_="lyrics").get_text() - except AttributeError as exc: - # html is a NoneType cannot retrieve lyrics - self._log.debug(u'Genius lyrics for {0} not found: {1}', - page_url, exc) + lyrics_div = html.find("div", class_="lyrics") + + # nullcheck + if lyrics_div is None: + self._log.debug(u'Genius lyrics for {0} not found', + page_url) return None + lyrics = lyrics_div.get_text() return lyrics diff --git a/test/rsrc/lyrics/geniuscom/sample.txt b/test/rsrc/lyrics/geniuscom/sample.txt index da8a4d0b6..1648d070a 100644 --- a/test/rsrc/lyrics/geniuscom/sample.txt +++ b/test/rsrc/lyrics/geniuscom/sample.txt @@ -1,1119 +1,270 @@ - + + //]]> + -SAMPLE – SONG Lyrics | Genius Lyrics + SAMPLE – SONG Lyrics | g-example Lyrics - - + + - + - + + + - + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ga('create', "UA-10346621-1", 'auto', {'useAmpClientId': true}); + ga('set', 'dimension1', "false"); + ga('set', 'dimension2', "songs#show"); + ga('set', 'dimension3', "r-b"); + ga('set', 'dimension4', "true"); + ga('set', 'dimension5', 'false'); + ga('set', 'dimension6', "none"); + ga('send', 'pageview'); + + - - + - - +
+
+
- - - - -
- - {{:: 'cloud_flare_always_on_short_message' | i18n }} -
Check @genius for updates. We'll have things fixed soon. - -
-
- - -
- GENIUS -
- - - -
- - - - - - - - - - - -
- - -
- - - -
- - - - - - - - - - - - - - - - -
- -
- -
-
-
-
-
- Https%3a%2f%2fimages -
-
- -
-
- -

SONG

-

- SAMPLE -

- -

- - -

-

- -
- Produced by - - - Carter Lang & The Antydote - -
- - -

-

- -
- Album - ALBUM -
- - -

- -
-
-
-
-
-
-
- - -
-
-
- -

SONG Lyrics

- -
-
- - -

[Verse 1]
-Fearin' not growin' up
-Keepin' me up at night
-Am I doin' enough?
-Feel like I'm wastin' time
-
-[Pre-Chorus]
-Promise to get a little
-Better as I get older
-And you're so patient
-And sick of waitin'
-Promise to do better
-Shoulda coulda
-Prolly wanna let me go
-But you can't, oh
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-
-[Chorus]
-Please don't take it, don't take it personal
-Like I know you usually do
-Please don't take it, don't take it personal
-Like I know you usually do
-Please, please
-Don't take it personal
-Don't take it personal
-Darling, like I know you will, ooh
-
-[Verse 2]
-Forget to call your mama on the weekend
-You should put yourself in time out
-(Shame, shame on you)
-But lately you've been feelin' so good
-I forget my future, never pull out
-(Shame, shame on me)
-Baby the money'll make it easier for me
-To run and hide out somewhere
-(So far away)
-Hoppin' through poppy fields
-Dodgin' evil witches
-These houses keep droppin' everywhere
-
-[Pre-Chorus]
-Promise to get a little
-Better as I get older
-And you're so patient
-And sick of waitin'
-Promise to do better
-Shoulda coulda
-Prolly wanna let me go
-But you can't, oh
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-Right now it's really pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-
-[Chorus]
-Please don't take it, don't take it personal
-Like I know you usually do
-Please don't take it, take it personal
-
-[Outro]
-Like winters fall on us, heavy
-Take it off me, all it off
-Winter, I can't stand this
-Snow is falling all on me

- - - - -
-
- -
-
-
More on Genius
- +
+ g-example
+ + + +
-
-
-
- -
-
-
+ + - +
+
+
+
+
+ # +
+
-
+
+
+

SONG

+

+ + SAMPLE + +

+

+
+ Produced by + + Person 1 & Person 2 + +
+

+

+
+ Album + ALBUM +
+

+
+
+ +
+
+
+ +
+
+
+

SONG Lyrics

+
+
+ !!!! MISSING LYRICS HERE !!! +
+
+
+
+
More on g-example
+
+
+
+
+
-

- About “SONG” -

- - -
-
-

The fifth track on SAMPLE’s debut album, ALBUM is the most reminiscent of her previous ep’s with a pop-disco sound.

+
+ Released by -

She reflects about her insecurities on not maturing as quickly as her partner is. SONG season is usually the last big event before graduation for a high schooler, and SAMPLE is too busy living in the moment that she forgets she has her future to look forward to.

- -

- -

This is also around the age when friends are growing apart and have decided where and what they’re doing with their lives, and a lot are trying to hold on to the people they’ve known, even though they’re growing up differently. The person SAMPLE is talking to in the song seems to take everything, even change, to heart.

-
-
-
- -
-
-
    - -
  • -
    -

    What have the artists said about this song

    -
    - -
    -

    SAMPLE tweeted:

    - -

    Iss called SONG cus I ain’t have no friends in high school so I went to Miami w my mama for SONG instead . Lol

    -
    - - -
    • - - - -
-
- - -
-
-

"SONG" Track Info

- - -
- Written By - - - The Antydote, Carter Lang & SAMPLE - -
- - - - - -
- Released by - - - RCA Records & Top Dawg Entertainment - -
- - - - -
- Mixing - - - Mixed by Ali - -
- - - - - -
- Recorded At - TDE Red Room, Carson, California -
- - - -
- Release Date - June 9, 2017 -
- - - - - - - - - - - - - - - - - - -
- Interpolated By - - -
- Wicked Games (StarFox Remix) by The Weeknd + + Records & Top +
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - +
+ Mixing + + Mixed by Person + +
+ +
+ Recorded At + City, Place +
+ +
+ Release Date + Feb 30, 1290 +
+ +
+ Interpolated By + + +
+ # +
+ +
-
-
- - Https%3a%2f%2fimages - -
- - ALBUM - - - SAMPLE -
-
- -
- -
-
- - 1.   - - - - # - - - - -
-
- -
-
- - 2.   - - - - # - - - - -
-
- -
-
- - 3.   - - - - # - - - - -
-
- -
-
- - 4.   - - - - # - - - - -
-
- -
-
- - 5.   - - - - SONG - - - - -
-
- -
-
- - 6.   - - - - # - - - - -
-
- -
-
- - 7.   - - - - # - - - - -
-
- -
-
- - 8.   - - - - # - - - - -
-
- -
-
- - 9.   - - - - # - - - - -
-
- -
-
- - 10.   - - - - # - - - - -
-
- -
-
- - 11.   - - - - # - - - - -
-
- -
-
- - 12.   - - - - # - - - - -
-
- -
-
- - 13.   - - - - # - - - - -
-
- -
-
- - 14.   - - - - # - - - - -
-
- - - -
- + + # + +
+ + ALBUM + + SAMPLE +
-
- -
-
-
- - - - - - - + +
-
- -
- Privacy Policy - - - Licensing - Jobs - Developers - Terms of Use - Copyright Policy - Contact us - - Do Not Sell My Personal Information - - - Sign in + + + - All Artists: -
    - -
  • - A -
    • - -
  • - B -
    • - -
  • - C -
    • - -
  • - D -
    • - -
  • - E -
    • - -
  • - F -
    • - -
  • - G -
    • - -
  • - H -
    • - -
  • - I -
    • - -
  • - J -
    • - -
  • - K -
    • - -
  • - L -
    • - -
  • - M -
    • - -
  • - N -
    • - -
  • - O -
    • - -
  • - P -
    • - -
  • - Q -
    • - -
  • - R -
    • - -
  • - S -
    • - -
  • - T -
    • - -
  • - U -
    • - -
  • - V -
    • - -
  • - W -
    • - -
  • - X -
    • - -
  • - Y -
    • - -
  • - Z -
    • - -
  • - # -
    • - -
- -
- -
- - - - - - - - - - - - - - - - - + + + + + - diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 6c23c627a..83a229f62 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -453,12 +453,8 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): """ # https://github.com/beetbox/beets/issues/3535 # expected return value None - try: - self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), - None) - except AttributeError: - # if AttributeError we aren't doing a null check - self.assertTrue(False) + self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + None) class SlugTests(unittest.TestCase): From 175c392ef33928f7decffed038cce90cdf00a7b6 Mon Sep 17 00:00:00 2001 From: lijacky Date: Fri, 17 Apr 2020 17:25:36 -0400 Subject: [PATCH 433/613] add fix to changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 33ad386ad..fd0c225f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,6 +178,8 @@ Fixes: * Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method failing with AttributeError when called. It is now an instance method. :bug:`3516` :bug:`3517` +* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper + :bug:`3535` For plugin developers: From 269e309513b54ab362a54f0a82851ac29630823f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 17 Apr 2020 18:28:21 -0400 Subject: [PATCH 434/613] Slight simplification for #3554 --- beetsplug/lyrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 00b8820f4..cca0fa94e 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -374,15 +374,12 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. lyrics_div = html.find("div", class_="lyrics") - - # nullcheck if lyrics_div is None: self._log.debug(u'Genius lyrics for {0} not found', page_url) return None - lyrics = lyrics_div.get_text() - return lyrics + return lyrics_div.get_text() def fetch(self, artist, title): search_url = self.base_url + "/search" From c93ea0b36b89bb7e62d7fe6cbd068c80ca6f9556 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 17 Apr 2020 18:29:27 -0400 Subject: [PATCH 435/613] Link to fix in #3554 --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fd0c225f9..6bb08b143 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,8 +178,9 @@ Fixes: * Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method failing with AttributeError when called. It is now an instance method. :bug:`3516` :bug:`3517` -* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper - :bug:`3535` +* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper. + Thanks to :user:`thejli21`. + :bug:`3535` :bug:`3554` For plugin developers: From a5ef8c7b09db2dc95df269de158d8f34f33ad015 Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 19 Apr 2020 00:50:14 +0530 Subject: [PATCH 436/613] Add artist_sort as the last entry in the artists to search This will handle cases where the artist name includes some characters which can cause a search failure for some websites - mainly unicode. Fixes https://github.com/beetbox/beets/issues/3340. --- beetsplug/lyrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index cca0fa94e..19acbcadc 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -200,12 +200,13 @@ def search_pairs(item): alternatives.append(match.group(1)) return alternatives - title, artist = item.title, item.artist + title, artist, artist_sort = item.title, item.artist, item.artist_sort patterns = [ # Remove any featuring artists from the artists name r"(.*?) {0}".format(plugins.feat_tokens())] artists = generate_alternatives(artist, patterns) + artists.append(artist_sort) patterns = [ # Remove a parenthesized suffix from a title string. Common From 45c8e51d6cd92574d0dafffe73a98ad21a627aad Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 19 Apr 2020 19:08:42 +0530 Subject: [PATCH 437/613] Fix redundant requests for artist == artist_sort --- beetsplug/lyrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 19acbcadc..0aebe9606 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -206,7 +206,10 @@ def search_pairs(item): # Remove any featuring artists from the artists name r"(.*?) {0}".format(plugins.feat_tokens())] artists = generate_alternatives(artist, patterns) - artists.append(artist_sort) + # Use the artist_sort as fallback only if it differs from artist to avoid + # repeated remote requests with the same search terms + if artist != artist_sort: + artists.append(artist_sort) patterns = [ # Remove a parenthesized suffix from a title string. Common From 625e9253c03bc4f2c5cab6c54e014cfbe8bca49e Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 19 Apr 2020 19:05:55 +0530 Subject: [PATCH 438/613] Add tests for artist_sort as lyrics search fallback Adjust doc comment to highlight that artist_sort is used as a fallback --- beetsplug/lyrics.py | 3 +++ test/test_lyrics.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0aebe9606..20e1c8858 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -187,6 +187,9 @@ def search_pairs(item): In addition to the artist and title obtained from the `item` the method tries to strip extra information like paranthesized suffixes and featured artists from the strings and add them as candidates. + The artist sort name is added as a fallback candidate to help in + cases where artist name includes special characters or is in a + non-latin script. The method also tries to split multiple titles separated with `/`. """ def generate_alternatives(string, patterns): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 83a229f62..81d62fea9 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -95,6 +95,27 @@ class LyricsPluginTest(unittest.TestCase): self.assertEqual(('Alice and Bob', ['song']), list(lyrics.search_pairs(item))[0]) + def test_search_artist_sort(self): + item = Item(artist='CHVRCHΞS', title='song', artist_sort='CHVRCHES') + self.assertIn(('CHVRCHΞS', ['song']), + lyrics.search_pairs(item)) + self.assertIn(('CHVRCHES', ['song']), + lyrics.search_pairs(item)) + + # Make sure that the original artist name is still the first entry + self.assertEqual(('CHVRCHΞS', ['song']), + list(lyrics.search_pairs(item))[0]) + + item = Item(artist='横山克', title='song', artist_sort='Masaru Yokoyama') + self.assertIn(('横山克', ['song']), + lyrics.search_pairs(item)) + self.assertIn(('Masaru Yokoyama', ['song']), + lyrics.search_pairs(item)) + + # Make sure that the original artist name is still the first entry + self.assertEqual(('横山克', ['song']), + list(lyrics.search_pairs(item))[0]) + def test_search_pairs_multi_titles(self): item = Item(title='1 / 2', artist='A') self.assertIn(('A', ['1 / 2']), lyrics.search_pairs(item)) From 425bbc366ff505e44777dd308c06af47382caaa9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 19 Apr 2020 11:12:24 -0400 Subject: [PATCH 439/613] Changelog/thanks for #3558 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6bb08b143..69f855507 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -181,6 +181,10 @@ Fixes: * :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper. Thanks to :user:`thejli21`. :bug:`3535` :bug:`3554` +* :doc:`/plugins/lyrics`: Use the artist sort name to search for lyrics, which + can help find matches when the artist name has special characters. + Thanks to :user:`hashhar`. + :bug:`3340` :bug:`3558` For plugin developers: From 1c03f58e0d606184d68b219daa4aa846834a157a Mon Sep 17 00:00:00 2001 From: Kylie McClain Date: Sat, 18 Apr 2020 22:23:53 -0400 Subject: [PATCH 440/613] fetchart: improve description of -q flag --- beetsplug/fetchart.py | 2 +- docs/plugins/fetchart.rst | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 28067c310..763440238 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -979,7 +979,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): cmd.parser.add_option( u'-q', u'--quiet', dest='quiet', action='store_true', default=False, - help=u'shows only quiet art' + help=u'quiet mode: do not output albums that already have artwork' ) def func(lib, opts, args): diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index e8f7b6d92..92ed5f32a 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -126,8 +126,9 @@ art:: $ beet fetchart [-q] [query] -By default the command will display all results, the ``-q`` or ``--quiet`` -switch will only display results for album arts that are still missing. +By default the command will display all albums matching the ``query``. When the +``-q`` or ``--quiet`` switch is given, only albums for which artwork has been +fetched, or for which artwork could not be found will be printed. .. _image-resizing: From 5143e630d8e23ba69893988bbd7d635629809ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Koutensk=C3=BD?= Date: Wed, 22 Apr 2020 19:44:47 +0200 Subject: [PATCH 441/613] Fix #3533 - crash when calculating RG/R128 mix --- beetsplug/replaygain.py | 8 ++++---- docs/changelog.rst | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 646e3acce..916541df6 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1326,10 +1326,10 @@ class ReplayGainPlugin(BeetsPlugin): if (any([self.should_use_r128(item) for item in album.items()]) and not all(([self.should_use_r128(item) for item in album.items()]))): - raise ReplayGainError( - u"Mix of ReplayGain and EBU R128 detected" - u" for some tracks in album {0}".format(album) - ) + self._log.info( + u"Cannot calculate gain for album {0} (incompatible formats)", + album) + return tag_vals = self.tag_specific_values(album.items()) store_track_gain, store_album_gain, target_level, peak = tag_vals diff --git a/docs/changelog.rst b/docs/changelog.rst index 69f855507..e2f1521eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -185,6 +185,12 @@ Fixes: can help find matches when the artist name has special characters. Thanks to :user:`hashhar`. :bug:`3340` :bug:`3558` +* :doc:`/plugins/replaygain`: Trying to calculate volume gain for an album + consisting of some formats using ``ReplayGain`` and some using ``R128`` + will no longer crash; instead it is skipped and and a message is logged. + The log message has also been rewritten for to improve clarity. + Thanks to :user:`autrimpo`. + :bug:`3533` For plugin developers: From 47deb2f084ab293ead30d99a7b874e2e23317531 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 23 Apr 2020 08:03:51 -0400 Subject: [PATCH 442/613] Switch log level for #3563, fix #3533 --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 916541df6..6f15b454e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1326,7 +1326,7 @@ class ReplayGainPlugin(BeetsPlugin): if (any([self.should_use_r128(item) for item in album.items()]) and not all(([self.should_use_r128(item) for item in album.items()]))): - self._log.info( + self._log.error( u"Cannot calculate gain for album {0} (incompatible formats)", album) return From 981d4dc829b91c8bbc3d1821a93a6c68cedc9e86 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 19:50:59 +0200 Subject: [PATCH 443/613] First try adding new albuminfo and trackinfo class --- beets/autotag/hooks.py | 171 ++++++++++++----------------------------- 1 file changed, 51 insertions(+), 120 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index d7c701db6..2555ed69f 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -39,8 +39,51 @@ except AttributeError: # Classes used to represent candidate options. +class Map(dict): + """ + Example: + m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) + """ + def __init__(self, *args, **kwargs): + super(Map, self).__init__(*args, **kwargs) + for arg in args: + if isinstance(arg, dict): + for k, v in arg.iteritems(): + self[k] = v + + if kwargs: + for k, v in kwargs.iteritems(): + self[k] = v + + def __getattr__(self, attr): + return self.get(attr) + + def __setattr__(self, key, value): + self.__setitem__(key, value) + + def __setitem__(self, key, value): + super(Map, self).__setitem__(key, value) + self.__dict__.update({key: value}) + + def __delattr__(self, item): + self.__delitem__(item) + + def __delitem__(self, key): + super(Map, self).__delitem__(key) + del self.__dict__[key] + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self,state): + for key in state: + self.__setattr__(key,state[key]) + def __hash__(self): + return hash(tuple(sorted(self.items()))) -class AlbumInfo(object): + + +class AlbumInfo(Map): """Describes a canonical release that may be used to match a release in the library. Consists of these data members: @@ -49,77 +92,18 @@ class AlbumInfo(object): - ``artist``: name of the release's primary artist - ``artist_id`` - ``tracks``: list of TrackInfo objects making up the release - - ``asin``: Amazon ASIN - - ``albumtype``: string describing the kind of release - - ``va``: boolean: whether the release has "various artists" - - ``year``: release year - - ``month``: release month - - ``day``: release day - - ``label``: music label responsible for the release - - ``mediums``: the number of discs in this release - - ``artist_sort``: name of the release's artist for sorting - - ``releasegroup_id``: MBID for the album's release group - - ``catalognum``: the label's catalog number for the release - - ``script``: character set used for metadata - - ``language``: human language of the metadata - - ``country``: the release country - - ``albumstatus``: MusicBrainz release status (Official, etc.) - - ``media``: delivery mechanism (Vinyl, etc.) - - ``albumdisambig``: MusicBrainz release disambiguation comment - - ``releasegroupdisambig``: MusicBrainz release group - disambiguation comment. - - ``artist_credit``: Release-specific artist name - - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) - - ``data_url``: The data source release URL. ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, album, album_id, artist, artist_id, tracks, asin=None, - albumtype=None, va=False, year=None, month=None, day=None, - label=None, mediums=None, artist_sort=None, - releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, style=None, genre=None, - albumstatus=None, media=None, albumdisambig=None, - releasegroupdisambig=None, artist_credit=None, - original_year=None, original_month=None, - original_day=None, data_source=None, data_url=None, - discogs_albumid=None, discogs_labelid=None, - discogs_artistid=None): + def __init__(self, album, album_id, artist, artist_id, tracks, **kwargs): self.album = album self.album_id = album_id self.artist = artist self.artist_id = artist_id self.tracks = tracks - self.asin = asin - self.albumtype = albumtype - self.va = va - self.year = year - self.month = month - self.day = day - self.label = label - self.mediums = mediums - self.artist_sort = artist_sort - self.releasegroup_id = releasegroup_id - self.catalognum = catalognum - self.script = script - self.language = language - self.country = country - self.style = style - self.genre = genre - self.albumstatus = albumstatus - self.media = media - self.albumdisambig = albumdisambig - self.releasegroupdisambig = releasegroupdisambig - self.artist_credit = artist_credit - self.original_year = original_year - self.original_month = original_month - self.original_day = original_day - self.data_source = data_source - self.data_url = data_url - self.discogs_albumid = discogs_albumid - self.discogs_labelid = discogs_labelid - self.discogs_artistid = discogs_artistid + for arg in kwargs: + self.__setattr__(arg,kwargs[arg]) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -143,75 +127,22 @@ class AlbumInfo(object): track.decode(codec) -class TrackInfo(object): +class TrackInfo(Map): """Describes a canonical track present on a release. Appears as part of an AlbumInfo's ``tracks`` list. Consists of these data members: - ``title``: name of the track - ``track_id``: MusicBrainz ID; UUID fragment only - - ``release_track_id``: MusicBrainz ID respective to a track on a - particular release; UUID fragment only - - ``artist``: individual track artist name - - ``artist_id`` - - ``length``: float: duration of the track in seconds - - ``index``: position on the entire release - - ``media``: delivery mechanism (Vinyl, etc.) - - ``medium``: the disc number this track appears on in the album - - ``medium_index``: the track's position on the disc - - ``medium_total``: the number of tracks on the item's disc - - ``artist_sort``: name of the track artist for sorting - - ``disctitle``: name of the individual medium (subtitle) - - ``artist_credit``: Recording-specific artist name - - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) - - ``data_url``: The data source release URL. - - ``lyricist``: individual track lyricist name - - ``composer``: individual track composer name - - ``composer_sort``: individual track composer sort name - - ``arranger`: individual track arranger name - - ``track_alt``: alternative track number (tape, vinyl, etc.) - - ``work`: individual track work title - - ``mb_workid`: individual track work id - - ``work_disambig`: individual track work diambiguation Only ``title`` and ``track_id`` are required. The rest of the fields may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, title, track_id, release_track_id=None, artist=None, - artist_id=None, length=None, index=None, medium=None, - medium_index=None, medium_total=None, artist_sort=None, - disctitle=None, artist_credit=None, data_source=None, - data_url=None, media=None, lyricist=None, composer=None, - composer_sort=None, arranger=None, track_alt=None, - work=None, mb_workid=None, work_disambig=None, bpm=None, - initial_key=None, genre=None): + def __init__(self, title, track_id,**kwargs): self.title = title self.track_id = track_id - self.release_track_id = release_track_id - self.artist = artist - self.artist_id = artist_id - self.length = length - self.index = index - self.media = media - self.medium = medium - self.medium_index = medium_index - self.medium_total = medium_total - self.artist_sort = artist_sort - self.disctitle = disctitle - self.artist_credit = artist_credit - self.data_source = data_source - self.data_url = data_url - self.lyricist = lyricist - self.composer = composer - self.composer_sort = composer_sort - self.arranger = arranger - self.track_alt = track_alt - self.work = work - self.mb_workid = mb_workid - self.work_disambig = work_disambig - self.bpm = bpm - self.initial_key = initial_key - self.genre = genre + for arg in kwargs: + self.__setattr__(arg,kwargs[arg]) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): From da43ff9c188276548c30237e6d93c6c45106bebe Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 20:43:30 +0200 Subject: [PATCH 444/613] arrange __getattr__ to behave normally --- beets/autotag/hooks.py | 94 ++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 2555ed69f..cc8e6a68a 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -39,49 +39,53 @@ except AttributeError: # Classes used to represent candidate options. -class Map(dict): - """ - Example: - m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) - """ - def __init__(self, *args, **kwargs): - super(Map, self).__init__(*args, **kwargs) - for arg in args: - if isinstance(arg, dict): - for k, v in arg.iteritems(): - self[k] = v - - if kwargs: - for k, v in kwargs.iteritems(): - self[k] = v - - def __getattr__(self, attr): - return self.get(attr) - - def __setattr__(self, key, value): - self.__setitem__(key, value) - - def __setitem__(self, key, value): - super(Map, self).__setitem__(key, value) - self.__dict__.update({key: value}) - - def __delattr__(self, item): - self.__delitem__(item) - - def __delitem__(self, key): - super(Map, self).__delitem__(key) - del self.__dict__[key] - - def __getstate__(self): - return self.__dict__ - - def __setstate__(self,state): - for key in state: - self.__setattr__(key,state[key]) - def __hash__(self): - return hash(tuple(sorted(self.items()))) +class Map(dict): + """ + Example: + m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, + sports=['Soccer']) + """ + def __init__(self, *args, **kwargs): + super(Map, self).__init__(*args, **kwargs) + for arg in args: + if isinstance(arg, dict): + for k, v in arg.iteritems(): + self[k] = v + + if kwargs: + for k, v in kwargs.iteritems(): + self[k] = v + + def __getattr__(self, attr): + if attr in self: + return self.get(attr) + else: + raise AttributeError + + def __setattr__(self, key, value): + self.__setitem__(key, value) + + def __setitem__(self, key, value): + super(Map, self).__setitem__(key, value) + self.__dict__.update({key: value}) + + def __delattr__(self, item): + self.__delitem__(item) + + def __delitem__(self, key): + super(Map, self).__delitem__(key) + del self.__dict__[key] + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, state): + for key in state: + self.__setattr__(key, state[key]) + + def __hash__(self): + return hash(tuple(sorted(self.items()))) - class AlbumInfo(Map): """Describes a canonical release that may be used to match a release @@ -103,7 +107,7 @@ class AlbumInfo(Map): self.artist_id = artist_id self.tracks = tracks for arg in kwargs: - self.__setattr__(arg,kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -138,11 +142,11 @@ class TrackInfo(Map): may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, title, track_id,**kwargs): + def __init__(self, title, track_id, **kwargs): self.title = title self.track_id = track_id for arg in kwargs: - self.__setattr__(arg,kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): From fea6ffc0386eed693fdeef27556635eef4d44bbd Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 22:24:24 +0200 Subject: [PATCH 445/613] arrange decode, set all attributes to be flexible --- beets/autotag/hooks.py | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index cc8e6a68a..dd9726b84 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -100,14 +100,9 @@ class AlbumInfo(Map): ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, album, album_id, artist, artist_id, tracks, **kwargs): - self.album = album - self.album_id = album_id - self.artist = artist - self.artist_id = artist_id - self.tracks = tracks + def __init__(self, **kwargs): for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -116,19 +111,11 @@ class AlbumInfo(Map): """Ensure that all string attributes on this object, and the constituent `TrackInfo` objects, are decoded to Unicode. """ - for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', - 'catalognum', 'script', 'language', 'country', 'style', - 'genre', 'albumstatus', 'albumdisambig', - 'releasegroupdisambig', 'artist_credit', - 'media', 'discogs_albumid', 'discogs_labelid', - 'discogs_artistid']: + for fld in self: value = getattr(self, fld) - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) - - if self.tracks: - for track in self.tracks: - track.decode(codec) + if type(value) == str: + if isinstance(value, bytes): + setattr(self, fld, value.decode(codec, 'ignore')) class TrackInfo(Map): @@ -142,22 +129,20 @@ class TrackInfo(Map): may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, title, track_id, **kwargs): - self.title = title - self.track_id = track_id + def __init__(self, **kwargs): for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): """Ensure that all string attributes on this object are decoded to Unicode. """ - for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', - 'artist_credit', 'media']: + for fld in self: value = getattr(self, fld) - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) + if type(value) == str: + if isinstance(value, bytes): + setattr(self, fld, value.decode(codec, 'ignore')) # Candidate distance scoring. From 53ce6f8b3d7d62c4ca54966042530f6c6e7d7c45 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 22:32:03 +0200 Subject: [PATCH 446/613] all attributes are flexible, so no positional arguments when initiating the class --- beets/autotag/mb.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index f86d3be71..ea8ef24da 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -193,8 +193,8 @@ def track_info(recording, index=None, medium=None, medium_index=None, the number of tracks on the medium. Each number is a 1-based index. """ info = beets.autotag.hooks.TrackInfo( - recording['title'], - recording['id'], + title=recording['title'], + track_id=recording['id'], index=index, medium=medium, medium_index=medium_index, @@ -341,11 +341,11 @@ def album_info(release): track_infos.append(ti) info = beets.autotag.hooks.AlbumInfo( - release['title'], - release['id'], - artist_name, - release['artist-credit'][0]['artist']['id'], - track_infos, + album=release['title'], + album_id=release['id'], + artist=artist_name, + artist_id=release['artist-credit'][0]['artist']['id'], + tracks=track_infos, mediums=len(release['medium-list']), artist_sort=artist_sort_name, artist_credit=artist_credit_name, From 1b2c8398b11ade93621583e06c0aea24b83c39b5 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 23:13:08 +0200 Subject: [PATCH 447/613] cleaning up beets/autotag/__init__.py --- beets/autotag/__init__.py | 154 ++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 65 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index f9e38413e..2dcf93c52 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -41,31 +41,10 @@ log = logging.getLogger('beets') def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ - item.artist = track_info.artist - item.artist_sort = track_info.artist_sort - item.artist_credit = track_info.artist_credit - item.title = track_info.title - item.mb_trackid = track_info.track_id - item.mb_releasetrackid = track_info.release_track_id - if track_info.artist_id: - item.mb_artistid = track_info.artist_id - if track_info.data_source: - item.data_source = track_info.data_source - - if track_info.lyricist is not None: - item.lyricist = track_info.lyricist - if track_info.composer is not None: - item.composer = track_info.composer - if track_info.composer_sort is not None: - item.composer_sort = track_info.composer_sort - if track_info.arranger is not None: - item.arranger = track_info.arranger - if track_info.work is not None: - item.work = track_info.work - if track_info.mb_workid is not None: - item.mb_workid = track_info.mb_workid - if track_info.work_disambig is not None: - item.work_disambig = track_info.work_disambig + print('zer' in track_info) + for attr in track_info: + print(attr in track_info) + item.__setattr__(attr, getattr(track_info, attr)) # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? @@ -78,25 +57,50 @@ def apply_metadata(album_info, mapping): for item, track_info in mapping.items(): # Artist or artist credit. if config['artist_credit']: - item.artist = (track_info.artist_credit or - track_info.artist or - album_info.artist_credit or - album_info.artist) - item.albumartist = (album_info.artist_credit or - album_info.artist) + + if 'artist_credit' in track_info: + item.artist = track_info.artist_credit + elif 'artist' in track_info: + item.artist = track_info.artist + elif 'artist_credit' in album_info: + item.artist = album_info.artist_credit + elif 'artist' in album_info: + item.artist = album_info.artist + + if 'artist_credit' in album_info: + item.albumartist = album_info.artist_credit + elif 'artist' in album_info: + item.albumartist = album_info.artist + else: - item.artist = (track_info.artist or album_info.artist) - item.albumartist = album_info.artist + if 'artist' in track_info: + item.artist = track_info.artist + elif 'artist' in album_info: + item.artist = album_info.artist + + if 'artist' in album_info: + item.albumartist = album_info.artist # Album. - item.album = album_info.album + if 'album' in album_info: + item.album = album_info.album # Artist sort and credit names. - item.artist_sort = track_info.artist_sort or album_info.artist_sort - item.artist_credit = (track_info.artist_credit or - album_info.artist_credit) - item.albumartist_sort = album_info.artist_sort - item.albumartist_credit = album_info.artist_credit + if 'artist_sort' in track_info: + item.artist_sort = track_info.artist_sort + elif 'artist_sort' in album_info: + item.artist_sort = album_info.artist_sort + + if 'artist_credit' in track_info: + item.artist_credit = track_info.artist_credit + elif 'artist_credit' in album_info: + item.artist_credit = album_info.artist_credit + + if 'albumartist_sort' in album_info: + item.albumartist_sort = album_info.artist_sort + + if 'albumartist_credit' in album_info: + item.albumartist_credit = album_info.artist_credit # Release date. for prefix in '', 'original_': @@ -106,7 +110,10 @@ def apply_metadata(album_info, mapping): for suffix in 'year', 'month', 'day': key = prefix + suffix - value = getattr(album_info, key) or 0 + if key in album_info: + value = getattr(album_info, key) + else: + value = 0 # If we don't even have a year, apply nothing. if suffix == 'year' and not value: @@ -122,40 +129,55 @@ def apply_metadata(album_info, mapping): item[suffix] = value # Title. - item.title = track_info.title + if 'title' in track_info: + item.title = track_info.title if config['per_disc_numbering']: # We want to let the track number be zero, but if the medium index # is not provided we need to fall back to the overall index. - if track_info.medium_index is not None: + if 'medium_index' in track_info: item.track = track_info.medium_index - else: + elif 'index' in track_info: item.track = track_info.index - item.tracktotal = track_info.medium_total or len(album_info.tracks) + if 'medium_total' in track_info: + item.tracktotal = track_info.medium_total + elif 'tracks' in album_info: + item.tracktotal = len(album_info.tracks) else: - item.track = track_info.index - item.tracktotal = len(album_info.tracks) + if 'index' in track_info: + item.track = track_info.index + if 'tracks' in album_info: + item.tracktotal = len(album_info.tracks) # Disc and disc count. - item.disc = track_info.medium - item.disctotal = album_info.mediums + if 'medium' in track_info: + item.disc = track_info.medium + if 'mediums' in album_info: + item.disctotal = album_info.mediums # MusicBrainz IDs. - item.mb_trackid = track_info.track_id - item.mb_releasetrackid = track_info.release_track_id - item.mb_albumid = album_info.album_id - if track_info.artist_id: + if 'track_id' in track_info: + item.mb_trackid = track_info.track_id + if 'release_track_id' in track_info: + item.mb_releasetrackid = track_info.release_track_id + if 'album_id' in album_info: + item.mb_albumid = album_info.album_id + if 'artist_id' in track_info: item.mb_artistid = track_info.artist_id - else: + elif 'artist_id' in album_info: item.mb_artistid = album_info.artist_id - item.mb_albumartistid = album_info.artist_id - item.mb_releasegroupid = album_info.releasegroup_id + if 'artist_id' in album_info: + item.mb_albumartistid = album_info.artist_id + if 'releasegroup_id' in album_info: + item.mb_releasegroupid = album_info.releasegroup_id # Compilation flag. - item.comp = album_info.va + if 'va' in album_info: + item.comp = album_info.va # Track alt. - item.track_alt = track_info.track_alt + if 'track_alt' in track_info: + item.track_alt = track_info.track_alt # Miscellaneous/nullable metadata. misc_fields = { @@ -197,14 +219,16 @@ def apply_metadata(album_info, mapping): # field is explicitly allowed to be overwritten for field in misc_fields['album']: clobber = field in config['overwrite_null']['album'].as_str_seq() - value = getattr(album_info, field) - if value is None and not clobber: - continue - item[field] = value + if field in album_info: + value = getattr(album_info, field) + if value is None and not clobber: + continue + item[field] = value for field in misc_fields['track']: clobber = field in config['overwrite_null']['track'].as_str_seq() - value = getattr(track_info, field) - if value is None and not clobber: - continue - item[field] = value + if field in track_info: + value = getattr(track_info, field) + if value is None and not clobber: + continue + item[field] = value From 62566ee61ddde9c6de8d6e527820fa32acaf579f Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 23:13:38 +0200 Subject: [PATCH 448/613] remove prints for testing --- beets/autotag/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 2dcf93c52..530dcaf6e 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -41,9 +41,7 @@ log = logging.getLogger('beets') def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ - print('zer' in track_info) for attr in track_info: - print(attr in track_info) item.__setattr__(attr, getattr(track_info, attr)) # At the moment, the other metadata is left intact (including album From f507f0463906adfaae0aec6f0c0acbe3b1cb2814 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:21:13 +0200 Subject: [PATCH 449/613] reintroduce default arguments, adapt all occurences of TrackInfo and AlbumInfo to the absence of positional arguments --- beets/autotag/hooks.py | 84 +++++++++++++++++++++++++++++++++++++++++- beetsplug/cue.py | 3 +- beetsplug/discogs.py | 7 ++-- test/test_autotag.py | 47 +++++++++++------------ test/test_ui.py | 6 ++- 5 files changed, 116 insertions(+), 31 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index dd9726b84..fcb4ba161 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -100,7 +100,51 @@ class AlbumInfo(Map): ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, **kwargs): + def __init__(self, album=None, album_id=None, artist=None, artist_id=None, + tracks, asin=None, albumtype=None, va=False, year=None, + month=None, day=None, label=None, mediums=None, + artist_sort=None, releasegroup_id=None, catalognum=None, + script=None, language=None, country=None, style=None, + genre=None, albumstatus=None, media=None, albumdisambig=None, + releasegroupdisambig=None, artist_credit=None, + original_year=None, original_month=None, + original_day=None, data_source=None, data_url=None, + discogs_albumid=None, discogs_labelid=None, + discogs_artistid=None, **kwargs): + self.album = album + self.album_id = album_id + self.artist = artist + self.artist_id = artist_id + self.tracks = tracks + self.asin = asin + self.albumtype = albumtype + self.va = va + self.year = year + self.month = month + self.day = day + self.label = label + self.mediums = mediums + self.artist_sort = artist_sort + self.releasegroup_id = releasegroup_id + self.catalognum = catalognum + self.script = script + self.language = language + self.country = country + self.style = style + self.genre = genre + self.albumstatus = albumstatus + self.media = media + self.albumdisambig = albumdisambig + self.releasegroupdisambig = releasegroupdisambig + self.artist_credit = artist_credit + self.original_year = original_year + self.original_month = original_month + self.original_day = original_day + self.data_source = data_source + self.data_url = data_url + self.discogs_albumid = discogs_albumid + self.discogs_labelid = discogs_labelid + self.discogs_artistid = discogs_artistid for arg in kwargs: self.__setattr__(arg, kwargs[arg]) @@ -129,7 +173,43 @@ class TrackInfo(Map): may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, **kwargs): + def __init__(self, title=None, track_id=None, release_track_id=None, + artist=None, artist_id=None, length=None, index=None, + medium=None, medium_index=None, medium_total=None, + artist_sort=None, disctitle=None, artist_credit=None, + data_source=None, data_url=None, media=None, lyricist=None, + composer=None, composer_sort=None, arranger=None, + performer=None, track_alt=None, work=None, mb_workid=None, + work_disambig=None, bpm=None, initial_key=None, genre=None, + **kwargs): + self.title = title + self.track_id = track_id + self.release_track_id = release_track_id + self.artist = artist + self.artist_id = artist_id + self.length = length + self.index = index + self.media = media + self.medium = medium + self.medium_index = medium_index + self.medium_total = medium_total + self.artist_sort = artist_sort + self.disctitle = disctitle + self.artist_credit = artist_credit + self.data_source = data_source + self.data_url = data_url + self.lyricist = lyricist + self.composer = composer + self.composer_sort = composer_sort + self.arranger = arranger + self.performer = performer + self.track_alt = track_alt + self.work = work + self.mb_workid = mb_workid + self.work_disambig = work_disambig + self.bpm = bpm + self.initial_key = initial_key + self.genre = genre for arg in kwargs: self.__setattr__(arg, kwargs[arg]) diff --git a/beetsplug/cue.py b/beetsplug/cue.py index 92ca8784a..1ff817b2b 100644 --- a/beetsplug/cue.py +++ b/beetsplug/cue.py @@ -53,5 +53,6 @@ class CuePlugin(BeetsPlugin): title = "dunno lol" track_id = "wtf" index = int(path.basename(t)[len("split-track"):-len(".wav")]) - yield TrackInfo(title, track_id, index=index, artist=artist) + yield TrackInfo(title=title, track_id=track_id, index=index, + artist=artist) # generate TrackInfo instances diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 86ace9aa8..a0a6ea654 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -356,7 +356,8 @@ class DiscogsPlugin(BeetsPlugin): # a master release, otherwise fetch the master release. original_year = self.get_master_year(master_id) if master_id else year - return AlbumInfo(album, album_id, artist, artist_id, tracks, asin=None, + return AlbumInfo(album=album, album_id=album_id, artist=artist, + artist_id=artist_id, tracks=tracks, asin=None, albumtype=albumtype, va=va, year=year, month=None, day=None, label=label, mediums=len(set(mediums)), artist_sort=None, releasegroup_id=master_id, @@ -567,8 +568,8 @@ class DiscogsPlugin(BeetsPlugin): track.get('artists', []) ) length = self.get_track_length(track['duration']) - return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, - length=length, index=index, + return TrackInfo(title=title, track_id=track_id, artist=artist, + artist_id=artist_id, length=length, index=index, medium=medium, medium_index=medium_index, artist_sort=None, disctitle=None, artist_credit=None) diff --git a/test/test_autotag.py b/test/test_autotag.py index a11bc8fac..3b9ff3e67 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -106,9 +106,9 @@ def _make_item(title, track, artist=u'some artist'): def _make_trackinfo(): return [ - TrackInfo(u'one', None, artist=u'some artist', length=1, index=1), - TrackInfo(u'two', None, artist=u'some artist', length=1, index=2), - TrackInfo(u'three', None, artist=u'some artist', length=1, index=3), + TrackInfo(title=u'one', track_id=one, artist=u'some artist', length=1, index=1), + TrackInfo(title=u'two', track_id=None, artist=u'some artist', length=1, index=2), + TrackInfo(title=u'three', track_id=None, artist=u'some artist', length=1, index=3), ] @@ -503,9 +503,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 2)) items.append(self.item(u'two', 3)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'two', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'two', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -522,9 +522,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 1)) items.append(self.item(u'two', 1)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'two', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'two', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -540,9 +540,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'one', 1)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'two', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'two', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -558,8 +558,8 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'two', 2)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, [items[1]]) @@ -595,7 +595,8 @@ class AssignmentTest(unittest.TestCase): items.append(item(12, 186.45916150485752)) def info(index, title, length): - return TrackInfo(title, None, length=length, index=index) + return TrackInfo(title=title, track_id=None, length=length, + index=index) trackinfo = [] trackinfo.append(info(1, u'Alone', 238.893)) trackinfo.append(info(2, u'The Woman in You', 341.44)) @@ -638,8 +639,8 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item({})) trackinfo = [] trackinfo.append(TrackInfo( - u'oneNew', - u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', medium=1, medium_index=1, medium_total=1, @@ -648,8 +649,8 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): artist_sort='trackArtistSort', )) trackinfo.append(TrackInfo( - u'twoNew', - u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', medium=2, medium_index=1, index=2, @@ -828,15 +829,15 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): self.items.append(Item({})) trackinfo = [] trackinfo.append(TrackInfo( - u'oneNew', - u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', artist=u'artistOneNew', artist_id=u'a05686fc-9db2-4c23-b99e-77f5db3e5282', index=1, )) trackinfo.append(TrackInfo( - u'twoNew', - u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', artist=u'artistTwoNew', artist_id=u'80b3cf5e-18fe-4c59-98c7-e5bb87210710', index=2, diff --git a/test/test_ui.py b/test/test_ui.py index 110e80782..c4d502c73 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1051,8 +1051,10 @@ class ShowChangeTest(_common.TestCase): self.items[0].track = 1 self.items[0].path = b'/path/to/file.mp3' self.info = autotag.AlbumInfo( - u'the album', u'album id', u'the artist', u'artist id', [ - autotag.TrackInfo(u'the title', u'track id', index=1) + album=u'the album', album_id=u'album id', artist=u'the artist', + artist_id=u'artist id', tracks=[ + autotag.TrackInfo(title=u'the title', track_id=u'track id', + index=1) ] ) From 32d81d801bd7d9eea9a6887cdf443e0675ce6215 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:25:42 +0200 Subject: [PATCH 450/613] forgot a positional argument --- beets/autotag/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index fcb4ba161..0017f9dd6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -101,7 +101,7 @@ class AlbumInfo(Map): The others are optional and may be None. """ def __init__(self, album=None, album_id=None, artist=None, artist_id=None, - tracks, asin=None, albumtype=None, va=False, year=None, + tracks=None, asin=None, albumtype=None, va=False, year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, language=None, country=None, style=None, From 805f4e801f36cf73be8735daf601647b45b55770 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:31:39 +0200 Subject: [PATCH 451/613] typo in tests --- test/test_autotag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 3b9ff3e67..b894a65f3 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -106,7 +106,7 @@ def _make_item(title, track, artist=u'some artist'): def _make_trackinfo(): return [ - TrackInfo(title=u'one', track_id=one, artist=u'some artist', length=1, index=1), + TrackInfo(title=u'one', track_id=None, artist=u'some artist', length=1, index=1), TrackInfo(title=u'two', track_id=None, artist=u'some artist', length=1, index=2), TrackInfo(title=u'three', track_id=None, artist=u'some artist', length=1, index=3), ] From bd543136d66037e13400793d518ee74593d7f5f5 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:37:19 +0200 Subject: [PATCH 452/613] scale back some changes in __init__.py and hooks.py --- beets/autotag/__init__.py | 155 +++++++++++++++++--------------------- beets/autotag/hooks.py | 10 ++- 2 files changed, 76 insertions(+), 89 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 530dcaf6e..05b341a2d 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -41,8 +41,33 @@ log = logging.getLogger('beets') def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ - for attr in track_info: - item.__setattr__(attr, getattr(track_info, attr)) + item.artist = track_info.artist + item.artist_sort = track_info.artist_sort + item.artist_credit = track_info.artist_credit + item.title = track_info.title + item.mb_trackid = track_info.track_id + item.mb_releasetrackid = track_info.release_track_id + if track_info.artist_id: + item.mb_artistid = track_info.artist_id + if track_info.data_source: + item.data_source = track_info.data_source + + if track_info.lyricist is not None: + item.lyricist = track_info.lyricist + if track_info.composer is not None: + item.composer = track_info.composer + if track_info.composer_sort is not None: + item.composer_sort = track_info.composer_sort + if track_info.arranger is not None: + item.arranger = track_info.arranger + if track_info.performer is not None: + item.performer = track_info.performer + if track_info.work is not None: + item.work = track_info.work + if track_info.mb_workid is not None: + item.mb_workid = track_info.mb_workid + if track_info.work_disambig is not None: + item.work_disambig = track_info.work_disambig # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? @@ -55,50 +80,25 @@ def apply_metadata(album_info, mapping): for item, track_info in mapping.items(): # Artist or artist credit. if config['artist_credit']: - - if 'artist_credit' in track_info: - item.artist = track_info.artist_credit - elif 'artist' in track_info: - item.artist = track_info.artist - elif 'artist_credit' in album_info: - item.artist = album_info.artist_credit - elif 'artist' in album_info: - item.artist = album_info.artist - - if 'artist_credit' in album_info: - item.albumartist = album_info.artist_credit - elif 'artist' in album_info: - item.albumartist = album_info.artist - + item.artist = (track_info.artist_credit or + track_info.artist or + album_info.artist_credit or + album_info.artist) + item.albumartist = (album_info.artist_credit or + album_info.artist) else: - if 'artist' in track_info: - item.artist = track_info.artist - elif 'artist' in album_info: - item.artist = album_info.artist - - if 'artist' in album_info: - item.albumartist = album_info.artist + item.artist = (track_info.artist or album_info.artist) + item.albumartist = album_info.artist # Album. - if 'album' in album_info: - item.album = album_info.album + item.album = album_info.album # Artist sort and credit names. - if 'artist_sort' in track_info: - item.artist_sort = track_info.artist_sort - elif 'artist_sort' in album_info: - item.artist_sort = album_info.artist_sort - - if 'artist_credit' in track_info: - item.artist_credit = track_info.artist_credit - elif 'artist_credit' in album_info: - item.artist_credit = album_info.artist_credit - - if 'albumartist_sort' in album_info: - item.albumartist_sort = album_info.artist_sort - - if 'albumartist_credit' in album_info: - item.albumartist_credit = album_info.artist_credit + item.artist_sort = track_info.artist_sort or album_info.artist_sort + item.artist_credit = (track_info.artist_credit or + album_info.artist_credit) + item.albumartist_sort = album_info.artist_sort + item.albumartist_credit = album_info.artist_credit # Release date. for prefix in '', 'original_': @@ -108,10 +108,7 @@ def apply_metadata(album_info, mapping): for suffix in 'year', 'month', 'day': key = prefix + suffix - if key in album_info: - value = getattr(album_info, key) - else: - value = 0 + value = getattr(album_info, key) or 0 # If we don't even have a year, apply nothing. if suffix == 'year' and not value: @@ -127,55 +124,40 @@ def apply_metadata(album_info, mapping): item[suffix] = value # Title. - if 'title' in track_info: - item.title = track_info.title + item.title = track_info.title if config['per_disc_numbering']: # We want to let the track number be zero, but if the medium index # is not provided we need to fall back to the overall index. - if 'medium_index' in track_info: + if track_info.medium_index is not None: item.track = track_info.medium_index - elif 'index' in track_info: + else: item.track = track_info.index - if 'medium_total' in track_info: - item.tracktotal = track_info.medium_total - elif 'tracks' in album_info: - item.tracktotal = len(album_info.tracks) + item.tracktotal = track_info.medium_total or len(album_info.tracks) else: - if 'index' in track_info: - item.track = track_info.index - if 'tracks' in album_info: - item.tracktotal = len(album_info.tracks) + item.track = track_info.index + item.tracktotal = len(album_info.tracks) # Disc and disc count. - if 'medium' in track_info: - item.disc = track_info.medium - if 'mediums' in album_info: - item.disctotal = album_info.mediums + item.disc = track_info.medium + item.disctotal = album_info.mediums # MusicBrainz IDs. - if 'track_id' in track_info: - item.mb_trackid = track_info.track_id - if 'release_track_id' in track_info: - item.mb_releasetrackid = track_info.release_track_id - if 'album_id' in album_info: - item.mb_albumid = album_info.album_id - if 'artist_id' in track_info: + item.mb_trackid = track_info.track_id + item.mb_releasetrackid = track_info.release_track_id + item.mb_albumid = album_info.album_id + if track_info.artist_id: item.mb_artistid = track_info.artist_id - elif 'artist_id' in album_info: + else: item.mb_artistid = album_info.artist_id - if 'artist_id' in album_info: - item.mb_albumartistid = album_info.artist_id - if 'releasegroup_id' in album_info: - item.mb_releasegroupid = album_info.releasegroup_id + item.mb_albumartistid = album_info.artist_id + item.mb_releasegroupid = album_info.releasegroup_id # Compilation flag. - if 'va' in album_info: - item.comp = album_info.va + item.comp = album_info.va # Track alt. - if 'track_alt' in track_info: - item.track_alt = track_info.track_alt + item.track_alt = track_info.track_alt # Miscellaneous/nullable metadata. misc_fields = { @@ -204,6 +186,7 @@ def apply_metadata(album_info, mapping): 'composer', 'composer_sort', 'arranger', + 'performer', 'work', 'mb_workid', 'work_disambig', @@ -217,16 +200,14 @@ def apply_metadata(album_info, mapping): # field is explicitly allowed to be overwritten for field in misc_fields['album']: clobber = field in config['overwrite_null']['album'].as_str_seq() - if field in album_info: - value = getattr(album_info, field) - if value is None and not clobber: - continue - item[field] = value + value = getattr(album_info, field) + if value is None and not clobber: + continue + item[field] = value for field in misc_fields['track']: clobber = field in config['overwrite_null']['track'].as_str_seq() - if field in track_info: - value = getattr(track_info, field) - if value is None and not clobber: - continue - item[field] = value + value = getattr(track_info, field) + if value is None and not clobber: + continue + item[field] = value diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 0017f9dd6..1e856ca0a 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -155,7 +155,12 @@ class AlbumInfo(Map): """Ensure that all string attributes on this object, and the constituent `TrackInfo` objects, are decoded to Unicode. """ - for fld in self: + for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', + 'catalognum', 'script', 'language', 'country', 'style', + 'genre', 'albumstatus', 'albumdisambig', + 'releasegroupdisambig', 'artist_credit', + 'media', 'discogs_albumid', 'discogs_labelid', + 'discogs_artistid']: value = getattr(self, fld) if type(value) == str: if isinstance(value, bytes): @@ -218,7 +223,8 @@ class TrackInfo(Map): """Ensure that all string attributes on this object are decoded to Unicode. """ - for fld in self: + for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', + 'artist_credit', 'media']: value = getattr(self, fld) if type(value) == str: if isinstance(value, bytes): From 363cbf8147940f471558e3545cc58ee2f491fb3e Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:39:20 +0200 Subject: [PATCH 453/613] mixed two PR --- beets/autotag/__init__.py | 3 --- beets/autotag/hooks.py | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 05b341a2d..f9e38413e 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -60,8 +60,6 @@ def apply_item_metadata(item, track_info): item.composer_sort = track_info.composer_sort if track_info.arranger is not None: item.arranger = track_info.arranger - if track_info.performer is not None: - item.performer = track_info.performer if track_info.work is not None: item.work = track_info.work if track_info.mb_workid is not None: @@ -186,7 +184,6 @@ def apply_metadata(album_info, mapping): 'composer', 'composer_sort', 'arranger', - 'performer', 'work', 'mb_workid', 'work_disambig', diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 1e856ca0a..48d1cae52 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -184,7 +184,7 @@ class TrackInfo(Map): artist_sort=None, disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, - performer=None, track_alt=None, work=None, mb_workid=None, + track_alt=None, work=None, mb_workid=None, work_disambig=None, bpm=None, initial_key=None, genre=None, **kwargs): self.title = title @@ -207,7 +207,6 @@ class TrackInfo(Map): self.composer = composer self.composer_sort = composer_sort self.arranger = arranger - self.performer = performer self.track_alt = track_alt self.work = work self.mb_workid = mb_workid From 14e1b33839afbf1c8fb1c4f94caf77bb99d82b96 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:44:51 +0200 Subject: [PATCH 454/613] lines too long --- test/test_autotag.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index b894a65f3..fcf2ee5b1 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -106,9 +106,12 @@ def _make_item(title, track, artist=u'some artist'): def _make_trackinfo(): return [ - TrackInfo(title=u'one', track_id=None, artist=u'some artist', length=1, index=1), - TrackInfo(title=u'two', track_id=None, artist=u'some artist', length=1, index=2), - TrackInfo(title=u'three', track_id=None, artist=u'some artist', length=1, index=3), + TrackInfo(title=u'one', track_id=None, artist=u'some artist', + length=1, index=1), + TrackInfo(title=u'two', track_id=None, artist=u'some artist', + length=1, index=2), + TrackInfo(title=u'three', track_id=None, artist=u'some artist', + length=1, index=3), ] From 8df2e5b8f5affc9a5472886e819a31789c8fa280 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 12:01:35 +0200 Subject: [PATCH 455/613] arrange decoder --- beets/autotag/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 48d1cae52..95533d11d 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -162,7 +162,7 @@ class AlbumInfo(Map): 'media', 'discogs_albumid', 'discogs_labelid', 'discogs_artistid']: value = getattr(self, fld) - if type(value) == str: + if isinstance(value, bytes): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) @@ -225,7 +225,7 @@ class TrackInfo(Map): for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', 'artist_credit', 'media']: value = getattr(self, fld) - if type(value) == str: + if isinstance(value, bytes): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) From 98389b6161bcdad8cbbd8638eeedd01d877b25a3 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 12:02:27 +0200 Subject: [PATCH 456/613] dupe test --- beets/autotag/hooks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 95533d11d..83abd8736 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -163,8 +163,7 @@ class AlbumInfo(Map): 'discogs_artistid']: value = getattr(self, fld) if isinstance(value, bytes): - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) + setattr(self, fld, value.decode(codec, 'ignore')) class TrackInfo(Map): @@ -226,8 +225,7 @@ class TrackInfo(Map): 'artist_credit', 'media']: value = getattr(self, fld) if isinstance(value, bytes): - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) + setattr(self, fld, value.decode(codec, 'ignore')) # Candidate distance scoring. From 6a03afc65df168cf60b83bc34c1b9b94c648c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Tempel?= Date: Sat, 25 Apr 2020 18:26:58 +0200 Subject: [PATCH 457/613] web plugin: support path queries by separating them with a backslash Without this change the web plugin does not support path queries as slashes are currently used for joining keywords in the QueryConverter. Moreover, flask cannot distinguish between an URL encoded and a plain '/' character during routing [0]. To work around this issue without introducing a breaking change (i.e. removing the QueryConverter) use the backslash character for path queries and convert it later on. Fixes #3566 [0]: https://github.com/pallets/flask/issues/900 --- beetsplug/web/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 21ff5d94e..49149772d 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -177,10 +177,11 @@ class QueryConverter(PathConverter): """ def to_python(self, value): - return value.split('/') + queries = value.split('/') + return [query.replace('\\', os.sep) for query in queries] def to_url(self, value): - return ','.join(value) + return ','.join([v.replace(os.sep, '\\') for v in value]) class EverythingConverter(PathConverter): From 57a759c8175eedabfddbd3a02e33b3af73fd13e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Tempel?= Date: Sun, 26 Apr 2020 17:39:16 +0200 Subject: [PATCH 458/613] docs: document joining of paths performed by web plugin for queries --- docs/plugins/web.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index d416b1b7d..99214ef01 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -210,7 +210,10 @@ If the server runs UNIX, you'll need to include an extra leading slash: ``GET /item/query/querystring`` +++++++++++++++++++++++++++++++ -Returns a list of tracks matching the query. The *querystring* must be a valid query as described in :doc:`/reference/query`. :: +Returns a list of tracks matching the query. The *querystring* must be a +valid query as described in :doc:`/reference/query`. Path elements are +joined as query keywords. For example, ``/item/query/foo/bar`` will be +converted to the query ``foo,bar``. :: { "results": [ From 226d8089e1ff16687bff1731f048e8e4ba130db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Tempel?= Date: Sun, 26 Apr 2020 17:45:37 +0200 Subject: [PATCH 459/613] docs: document special handling of backlash in web plugin queries --- docs/plugins/web.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 99214ef01..481727475 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -213,7 +213,11 @@ If the server runs UNIX, you'll need to include an extra leading slash: Returns a list of tracks matching the query. The *querystring* must be a valid query as described in :doc:`/reference/query`. Path elements are joined as query keywords. For example, ``/item/query/foo/bar`` will be -converted to the query ``foo,bar``. :: +converted to the query ``foo,bar``. As this conflicts with using a slash +character as a path seperator in path queries, a backlash character +should be used in these queries instead. This character is converted to +the path seperator actually used by the operating system before the +query is performed. :: { "results": [ From 969a2074186d277054c364d249320f3518edfaf0 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 27 Apr 2020 07:45:31 -0400 Subject: [PATCH 460/613] Simplify docs for #3567 --- docs/plugins/web.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 481727475..65d4743fb 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -211,13 +211,7 @@ If the server runs UNIX, you'll need to include an extra leading slash: +++++++++++++++++++++++++++++++ Returns a list of tracks matching the query. The *querystring* must be a -valid query as described in :doc:`/reference/query`. Path elements are -joined as query keywords. For example, ``/item/query/foo/bar`` will be -converted to the query ``foo,bar``. As this conflicts with using a slash -character as a path seperator in path queries, a backlash character -should be used in these queries instead. This character is converted to -the path seperator actually used by the operating system before the -query is performed. :: +valid query as described in :doc:`/reference/query`. :: { "results": [ @@ -226,6 +220,11 @@ query is performed. :: ] } +Path elements are joined as parts of a query. For example, +``/item/query/foo/bar`` will be converted to the query ``foo,bar``. +To specify literal path separators in a query, use a backslash instead of a +slash. + ``GET /item/6/file`` ++++++++++++++++++++ From f43a0c3270d18af19affea04fae9fb39ab02d784 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 27 Apr 2020 07:46:28 -0400 Subject: [PATCH 461/613] Changelog/thanks for #3567 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2f1521eb..5d8c26bb9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -115,6 +115,10 @@ New features: :bug:`3459` * :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. :bug:`3530` +* :doc:`/plugins/web`: The query API now interprets backslashes as path + separators to support path queries. + Thanks to :user:`nmeum`. + :bug:`3567` Fixes: From 63df7cf3edf9fc45fad4186d4332f30afe66a952 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 15:03:55 +0200 Subject: [PATCH 462/613] forgot to decode all tracks of an album --- beets/autotag/hooks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 83abd8736..a5d1ae428 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -165,6 +165,10 @@ class AlbumInfo(Map): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) + if 'tracks' in self: + for track in self.tracks: + track.decode(codec) + class TrackInfo(Map): """Describes a canonical track present on a release. Appears as part From 048b5c21517b88089723ecbca6282e337c976244 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 15:55:12 +0200 Subject: [PATCH 463/613] remove need for deepcopy, simplify __init__ --- beets/autotag/hooks.py | 18 +--- test/test_autotag.py | 199 +++++++++++++++++++++++++++++++++++++++-- test/test_ui.py | 7 +- 3 files changed, 202 insertions(+), 22 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index a5d1ae428..0c6e1eb80 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -45,13 +45,8 @@ class Map(dict): m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) """ - def __init__(self, *args, **kwargs): - super(Map, self).__init__(*args, **kwargs) - for arg in args: - if isinstance(arg, dict): - for k, v in arg.iteritems(): - self[k] = v - + def __init__(self, **kwargs): + super(Map, self).__init__(**kwargs) if kwargs: for k, v in kwargs.iteritems(): self[k] = v @@ -76,15 +71,8 @@ class Map(dict): super(Map, self).__delitem__(key) del self.__dict__[key] - def __getstate__(self): - return self.__dict__ - - def __setstate__(self, state): - for key in state: - self.__setattr__(key, state[key]) - def __hash__(self): - return hash(tuple(sorted(self.items()))) + return self.id class AlbumInfo(Map): diff --git a/test/test_autotag.py b/test/test_autotag.py index fcf2ee5b1..97030785b 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -753,13 +753,75 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].albumtype, 'album') def test_album_artist_overrides_empty_track_artist(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') def test_album_artist_overridden_by_nonempty_track_artist(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -781,7 +843,38 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].artist_sort, 'albumArtistSort') def test_full_date_applied(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -796,7 +889,38 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item(year=1, month=2, day=3)) self.items.append(Item(year=4, month=5, day=6)) - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.year = 2013 self._apply(info=my_info) @@ -816,7 +940,38 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[0].day, 3) def test_data_source_applied(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -827,6 +982,7 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def setUp(self): super(ApplyCompilationTest, self).setUp() + # make a deepcopy of self.info self.items = [] self.items.append(Item({})) self.items.append(Item({})) @@ -878,7 +1034,38 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): self.assertFalse(self.items[1].comp) def test_va_flag_sets_comp(self): - va_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + va_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) diff --git a/test/test_ui.py b/test/test_ui.py index c4d502c73..608c1d7fb 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1138,7 +1138,12 @@ class SummarizeItemsTest(_common.TestCase): summary = commands.summarize_items([self.item], False) self.assertEqual(summary, u"1 items, F, 4kbps, 10:54, 987.0 B") - i2 = deepcopy(self.item) + # make a copy of self.item + i2 = library.Item() + i2.bitrate = 4321 + i2.length = 10 * 60 + 54 + i2.format = "F" + summary = commands.summarize_items([self.item, i2], False) self.assertEqual(summary, u"2 items, F, 4kbps, 21:48, 1.9 KiB") From a51ef113d3fd1642f5c7298895697dae76fc9b1c Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 15:59:21 +0200 Subject: [PATCH 464/613] arranged hash --- beets/autotag/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 0c6e1eb80..350ca065c 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -72,7 +72,7 @@ class Map(dict): del self.__dict__[key] def __hash__(self): - return self.id + return id(self) class AlbumInfo(Map): From 2bc0027adf9824d7a312ae65ba0c045965eb4e60 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:08:47 +0200 Subject: [PATCH 465/613] remove some prints, unused libraries, __setitem__ method --- beets/autotag/hooks.py | 4 ---- test/test_autotag.py | 1 - test/test_ui.py | 1 - 3 files changed, 6 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 350ca065c..74145e78d 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -60,10 +60,6 @@ class Map(dict): def __setattr__(self, key, value): self.__setitem__(key, value) - def __setitem__(self, key, value): - super(Map, self).__setitem__(key, value) - self.__dict__.update({key: value}) - def __delattr__(self, item): self.__delitem__(item) diff --git a/test/test_autotag.py b/test/test_autotag.py index 97030785b..385b793ee 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -18,7 +18,6 @@ from __future__ import division, absolute_import, print_function import re -import copy import unittest from test import _common diff --git a/test/test_ui.py b/test/test_ui.py index 608c1d7fb..d0be10060 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -22,7 +22,6 @@ import shutil import re import subprocess import platform -from copy import deepcopy import six import unittest From eec7994dbe4cc3e12495de359989c1f6884f312c Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:16:37 +0200 Subject: [PATCH 466/613] corrected on deepcopy replacement --- test/test_autotag.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 385b793ee..af74b5c8f 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -1035,35 +1035,28 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info trackinfo = [] + trackinfo = [] trackinfo.append(TrackInfo( title=u'oneNew', track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, + artist=u'artistOneNew', + artist_id=u'a05686fc-9db2-4c23-b99e-77f5db3e5282', index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', )) trackinfo.append(TrackInfo( title=u'twoNew', track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, + artist=u'artistTwoNew', + artist_id=u'80b3cf5e-18fe-4c59-98c7-e5bb87210710', index=2, - medium_total=1, )) va_info = AlbumInfo( tracks=trackinfo, - artist=u'artistNew', + artist=u'variousNew', album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, + album_id='3b69ea40-39b8-487f-8818-04b6eff8c21a', + artist_id='89ad4ac3-39f7-470e-963a-56509c546377', + albumtype=u'compilation', ) va_info.va = True self._apply(info=va_info) From d07a3e36970aa785a8085afd56eaa4062be93787 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:17:11 +0200 Subject: [PATCH 467/613] remove __delitem__ and __delattr__ methods --- beets/autotag/hooks.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 74145e78d..36ea8db1e 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -60,13 +60,6 @@ class Map(dict): def __setattr__(self, key, value): self.__setitem__(key, value) - def __delattr__(self, item): - self.__delitem__(item) - - def __delitem__(self, key): - super(Map, self).__delitem__(key) - del self.__dict__[key] - def __hash__(self): return id(self) From 13db4063fa0d5515f8a479e90c1a2235c6e4cd06 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:31:38 +0200 Subject: [PATCH 468/613] actually, I don't need __init__ --- beets/autotag/hooks.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 36ea8db1e..1500538d3 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -45,11 +45,6 @@ class Map(dict): m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) """ - def __init__(self, **kwargs): - super(Map, self).__init__(**kwargs) - if kwargs: - for k, v in kwargs.iteritems(): - self[k] = v def __getattr__(self, attr): if attr in self: From b39ef0b8f949ea41d0d9e40674b20d9201c370a3 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 12:16:19 +0200 Subject: [PATCH 469/613] better deepcopy, docstring, minor improvements --- beets/autotag/hooks.py | 26 +++--- test/test_autotag.py | 198 +++++------------------------------------ 2 files changed, 40 insertions(+), 184 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 1500538d3..81c2fadb1 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -39,11 +39,19 @@ except AttributeError: # Classes used to represent candidate options. -class Map(dict): +class AttrDict(dict): """ - Example: - m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, - sports=['Soccer']) + Dictionary with flexible attributes + to get an tag value: + value = info.tag + or value = info[tag] + or value = info.get(tag) + or value = getattr(info, tag) + all raise AttributeError when info doesn't have tag + to set a tag value: + info.tag = value + or info[tag] = value + or setattr(info, tag, value) """ def __getattr__(self, attr): @@ -59,7 +67,7 @@ class Map(dict): return id(self) -class AlbumInfo(Map): +class AlbumInfo(AttrDict): """Describes a canonical release that may be used to match a release in the library. Consists of these data members: @@ -117,8 +125,7 @@ class AlbumInfo(Map): self.discogs_albumid = discogs_albumid self.discogs_labelid = discogs_labelid self.discogs_artistid = discogs_artistid - for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.update(kwargs) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -142,7 +149,7 @@ class AlbumInfo(Map): track.decode(codec) -class TrackInfo(Map): +class TrackInfo(AttrDict): """Describes a canonical track present on a release. Appears as part of an AlbumInfo's ``tracks`` list. Consists of these data members: @@ -189,8 +196,7 @@ class TrackInfo(Map): self.bpm = bpm self.initial_key = initial_key self.genre = genre - for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.update(kwargs) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): diff --git a/test/test_autotag.py b/test/test_autotag.py index af74b5c8f..3532a80ec 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -754,36 +754,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overrides_empty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') @@ -791,36 +765,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overridden_by_nonempty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -844,36 +792,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_full_date_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -890,36 +812,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.year = 2013 self._apply(info=my_info) @@ -941,36 +837,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_data_source_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -981,7 +851,6 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def setUp(self): super(ApplyCompilationTest, self).setUp() - # make a deepcopy of self.info self.items = [] self.items.append(Item({})) self.items.append(Item({})) @@ -1035,29 +904,10 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info trackinfo = [] - trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - artist=u'artistOneNew', - artist_id=u'a05686fc-9db2-4c23-b99e-77f5db3e5282', - index=1, - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - artist=u'artistTwoNew', - artist_id=u'80b3cf5e-18fe-4c59-98c7-e5bb87210710', - index=2, - )) - va_info = AlbumInfo( - tracks=trackinfo, - artist=u'variousNew', - album=u'albumNew', - album_id='3b69ea40-39b8-487f-8818-04b6eff8c21a', - artist_id='89ad4ac3-39f7-470e-963a-56509c546377', - albumtype=u'compilation', - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + va_info = AlbumInfo(self.info) + va_info.tracks = trackinfo va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) From ba2b22cac581d4cb304457b57fe5e1555bc18e45 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 12:59:52 +0200 Subject: [PATCH 470/613] try to correct deepcopy --- test/test_autotag.py | 66 ++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 3532a80ec..06e6b7e72 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -754,9 +754,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overrides_empty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') @@ -765,9 +770,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overridden_by_nonempty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' @@ -792,9 +802,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_full_date_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.year = 2013 my_info.month = 12 @@ -812,9 +827,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.year = 2013 self._apply(info=my_info) @@ -837,9 +857,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_data_source_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -904,9 +929,14 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - va_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + va_info = AlbumInfo() + va_info.update(self.info) va_info.tracks = trackinfo va_info.va = True self._apply(info=va_info) From 84bbd14c767d3a720e57b5572f9c8f443c18896a Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 13:09:04 +0200 Subject: [PATCH 471/613] trailing whitespace --- beets/autotag/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 81c2fadb1..2c78749e6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -42,13 +42,13 @@ except AttributeError: class AttrDict(dict): """ Dictionary with flexible attributes - to get an tag value: + to get an tag value: value = info.tag or value = info[tag] or value = info.get(tag) or value = getattr(info, tag) all raise AttributeError when info doesn't have tag - to set a tag value: + to set a tag value: info.tag = value or info[tag] = value or setattr(info, tag, value) From d7ed84646ef2b5a91cda234c0284433ff6c67bd5 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 14:18:27 +0200 Subject: [PATCH 472/613] create method for deepcopy --- beets/autotag/hooks.py | 14 +++++++++ test/test_autotag.py | 66 ++++-------------------------------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 2c78749e6..444ae7b59 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -148,6 +148,16 @@ class AlbumInfo(AttrDict): for track in self.tracks: track.decode(codec) + def dup_albuminfo(self): + dupe = AlbumInfo() + dupe.update(self) + if 'tracks' in self: + tracks = [] + for track in self.tracks: + tracks.append(track.dup_trackinfo()) + dupe.tracks = tracks + return dupe + class TrackInfo(AttrDict): """Describes a canonical track present on a release. Appears as part @@ -209,6 +219,10 @@ class TrackInfo(AttrDict): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) + def dup_trackinfo(self): + dupe = TrackInfo() + dupe.update(self) + # Candidate distance scoring. diff --git a/test/test_autotag.py b/test/test_autotag.py index 06e6b7e72..cef6364d0 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -753,32 +753,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overrides_empty_track_artist(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') def test_album_artist_overridden_by_nonempty_track_artist(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -801,16 +783,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_full_date_applied(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -826,16 +799,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item(year=4, month=5, day=6)) # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.year = 2013 self._apply(info=my_info) @@ -856,16 +820,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_data_source_applied(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -928,16 +883,7 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - va_info = AlbumInfo() - va_info.update(self.info) - va_info.tracks = trackinfo + va_info = self.info.dup_albuminfo() va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) From 370df6253d4c4da48183a018332c8c2778877979 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 14:19:59 +0200 Subject: [PATCH 473/613] forgot a return --- beets/autotag/hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 444ae7b59..c2775bf32 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -222,6 +222,7 @@ class TrackInfo(AttrDict): def dup_trackinfo(self): dupe = TrackInfo() dupe.update(self) + return dupe # Candidate distance scoring. From 499fcb8315e40e1554cfef5ccb4e8437fbfeb5b8 Mon Sep 17 00:00:00 2001 From: Joris Date: Sun, 3 May 2020 12:56:54 +0200 Subject: [PATCH 474/613] Update docs/plugins/subsonicplaylist.rst Co-authored-by: Adrian Sampson --- docs/plugins/subsonicplaylist.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index 22fe16816..c712a4823 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -23,11 +23,11 @@ flag. This resets all `subsonic_playlist` tag before importing playlists. Options to be defined in your config with their default value:: subsonicplaylist: - 'base_url': "https://your.subsonic.server" - 'delete': False, - 'playlist_ids': [], - 'playlist_names': [], - 'username': '', - 'password': '' + base_url: "https://your.subsonic.server" + delete: no + playlist_ids: [] + playlist_names: [] + username: '' + password: '' -Parameters `base_url`, `username` and `password` must be defined! \ No newline at end of file +Parameters `base_url`, `username` and `password` must be defined! From 82f0c59f46d6a0bd164313bc83540d2c4dcc442b Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 13:10:24 +0200 Subject: [PATCH 475/613] review: updated subsonicplaylist.rst --- docs/plugins/subsonicplaylist.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index c712a4823..b9a54c3de 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -2,11 +2,11 @@ Subsonic Playlist Plugin ======================== The ``subsonicplaylist`` plugin allows to import playlists from a subsonic server. -This is done by retrieving the track infos from the subsonic server, searching -them in the beets library and adding the playlist names to the +This is done by retrieving the track info from the subsonic server, searching +for them in the beets library, and adding the playlist names to the `subsonic_playlist` tag of the found items. The content of the tag has the format: - subsonic_playlist: ";first playlist;second playlist" + subsonic_playlist: ";first playlist;second playlist;" To get all items in a playlist use the query `;playlist name;`. @@ -15,12 +15,22 @@ Command Line Usage To use the ``subsonicplaylist`` plugin, enable it in your configuration (see :ref:`using-plugins`). Then use it by invoking the ``subsonicplaylist`` command. +Next, configure the plugin to connect to your Subsonic server, like this:: + + subsonicplaylist: + base_url: http://subsonic.example.com + username: someUser + password: somePassword + +After this you can import your playlists by invoking the `subsonicplaylist` command. + By default only the tags of the items found for playlists will be updated. This means that, if one imported a playlist, then delete one song from it and imported the playlist again, the deleted song will still have the playlist set in its `subsonic_playlist` tag. To solve this problem one can use the `-d/--delete` flag. This resets all `subsonic_playlist` tag before importing playlists. -Options to be defined in your config with their default value:: + +Here's an example configuration with all the available options and their default values:: subsonicplaylist: base_url: "https://your.subsonic.server" From cb7dfe3f6f9fa37045cf6741e6909003784c62b0 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 14:05:22 +0200 Subject: [PATCH 476/613] review: updated subsonicplaylist.py --- beetsplug/subsonicplaylist.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index fbbcd8ac3..561b6032d 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -19,7 +19,13 @@ from hashlib import md5 import xml.etree.ElementTree as ET import random import string +from urllib.parse import urlencode + import requests +from beets.dbcore.query import SubstringQuery + +from beets.dbcore import AndQuery, Query, MatchQuery + from beets import plugins from beets.ui import Subcommand @@ -46,19 +52,21 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def update_tags(self, playlist_dict, lib): for query, playlist_tag in playlist_dict.items(): - query = 'artist:"{}" album:"{}" title:"{}"'.format(*query) + query = AndQuery([SubstringQuery("artist", query[0]), + SubstringQuery("album", query[1]), + SubstringQuery("title", query[2])]) items = lib.items(query) - if len(items) <= 0: + if not items: self._log.warn(u"{} | track not found ({})", playlist_tag, query) continue for item in items: - item.update({'subsonic_playlist': playlist_tag}) + item.subsonic_playlist = playlist_tag with lib.transaction(): item.try_sync(write=True, move=False) def get_playlist(self, playlist_id): - xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text + xml = self.send('getPlaylist', {'id': playlist_id}).text playlist = ET.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' @@ -72,7 +80,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def commands(self): def build_playlist(lib, opts, args): - self._parse_opts(opts) + self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() if len(self.config['playlist_names'].as_str_seq()) > 0: playlists = ET.fromstring(self.send('getPlaylists').text)[ @@ -102,8 +110,6 @@ class SubsonicPlaylistPlugin(BeetsPlugin): item.try_sync(write=True, move=False) self.update_tags(playlist_dict, lib) - ## notify plugins - plugins.send('database_change', lib=lib, model=self) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' @@ -117,24 +123,18 @@ class SubsonicPlaylistPlugin(BeetsPlugin): subsonicplaylist_cmds.func = build_playlist return [subsonicplaylist_cmds] - def _parse_opts(self, opts): - - if opts.delete: - self.config['delete'].set(True) - - self.opts = opts - return True - def generate_token(self): salt = ''.join(random.choices(string.ascii_lowercase + string.digits)) return md5( (self.config['password'].get() + salt).encode()).hexdigest(), salt - def send(self, endpoint, params=''): + def send(self, endpoint, params=None): + if params is None: + params = dict() url = '{}/rest/{}?u={}&t={}&s={}&v=1.12.0&c=beets'.format( self.config['base_url'].get(), endpoint, self.config['username'], *self.generate_token()) - resp = requests.get(url + params) + resp = requests.get(url + urlencode(params)) return resp From 13b4a1413d1c69e79dcb497fb32675c7d96fb302 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 14:58:34 +0200 Subject: [PATCH 477/613] update all songs in one transaction --- beetsplug/subsonicplaylist.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 561b6032d..9c9c74974 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -51,18 +51,18 @@ class SubsonicPlaylistPlugin(BeetsPlugin): self.config['password'].redact = True def update_tags(self, playlist_dict, lib): - for query, playlist_tag in playlist_dict.items(): - query = AndQuery([SubstringQuery("artist", query[0]), - SubstringQuery("album", query[1]), - SubstringQuery("title", query[2])]) - items = lib.items(query) - if not items: - self._log.warn(u"{} | track not found ({})", playlist_tag, - query) - continue - for item in items: - item.subsonic_playlist = playlist_tag - with lib.transaction(): + with lib.transaction(): + for query, playlist_tag in playlist_dict.items(): + query = AndQuery([SubstringQuery("artist", query[0]), + SubstringQuery("album", query[1]), + SubstringQuery("title", query[2])]) + items = lib.items(query) + if not items: + self._log.warn(u"{} | track not found ({})", playlist_tag, + query) + continue + for item in items: + item.subsonic_playlist = playlist_tag item.try_sync(write=True, move=False) def get_playlist(self, playlist_id): From cb7ad191a35d6deda8872f633a7a44c8b26673d4 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 15:02:51 +0200 Subject: [PATCH 478/613] removed duplicate line from merge --- docs/changelog.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cbd9f3ce2..88d289e35 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,6 @@ Changelog New features: * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. -* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets From 5c3debe2369cc161c2c9a875e3f2c20d8a2f5903 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 16:07:27 +0200 Subject: [PATCH 479/613] removed unused imports --- beetsplug/subsonicplaylist.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 9c9c74974..14cd855b1 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -24,9 +24,7 @@ from urllib.parse import urlencode import requests from beets.dbcore.query import SubstringQuery -from beets.dbcore import AndQuery, Query, MatchQuery - -from beets import plugins +from beets.dbcore import AndQuery from beets.ui import Subcommand From a553693677e8cc11c645efa94d11b1669fd0e35c Mon Sep 17 00:00:00 2001 From: pants108 Date: Sun, 3 May 2020 00:06:55 +0000 Subject: [PATCH 480/613] fetchart: clean up invalid tmp files --- beetsplug/fetchart.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 763440238..d4ab714de 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -210,6 +210,9 @@ class ArtSource(RequestMixin): def fetch_image(self, candidate, plugin): raise NotImplementedError() + def cleanup_tmp(self, candidate): + raise NotImplementedError() + class LocalArtSource(ArtSource): IS_LOCAL = True @@ -218,6 +221,10 @@ class LocalArtSource(ArtSource): def fetch_image(self, candidate, plugin): pass + def cleanup_tmp(self, candidate): + # local art source does not create tmp files + pass + class RemoteArtSource(ArtSource): IS_LOCAL = False @@ -291,6 +298,13 @@ class RemoteArtSource(ArtSource): self._log.debug(u'error fetching art: {}', exc) return + def cleanup_tmp(self, candidate): + if candidate.path: + try: + util.remove(path=candidate.path) + except util.FilesystemError as exc: + self._log.debug(u'error cleaning up tmp art: {}', exc) + class CoverArtArchive(RemoteArtSource): NAME = u"Cover Art Archive" @@ -1017,6 +1031,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): u'using {0.LOC_STR} image {1}'.format( source, util.displayable_path(out.path))) break + # else: remove tmp images created by this invalid candidate + source.cleanup_tmp(candidate) if out: break From 4102aae100d07344a67432ee00dfe8ef30f3743b Mon Sep 17 00:00:00 2001 From: Joris Date: Tue, 5 May 2020 10:45:43 +0200 Subject: [PATCH 481/613] Update docs/plugins/subsonicplaylist.rst Co-authored-by: Adrian Sampson --- docs/plugins/subsonicplaylist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index b9a54c3de..98c83ebe1 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -40,4 +40,4 @@ Here's an example configuration with all the available options and their default username: '' password: '' -Parameters `base_url`, `username` and `password` must be defined! +The `base_url`, `username`, and `password` options are required. From 1a79ae5a9f511394ed5767e270bd9737110717f6 Mon Sep 17 00:00:00 2001 From: pants108 Date: Tue, 5 May 2020 18:39:05 +0000 Subject: [PATCH 482/613] code review 1 --- beetsplug/fetchart.py | 14 +++++--------- docs/changelog.rst | 4 +++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index d4ab714de..70477a624 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -210,8 +210,8 @@ class ArtSource(RequestMixin): def fetch_image(self, candidate, plugin): raise NotImplementedError() - def cleanup_tmp(self, candidate): - raise NotImplementedError() + def cleanup(self, candidate): + pass class LocalArtSource(ArtSource): @@ -221,10 +221,6 @@ class LocalArtSource(ArtSource): def fetch_image(self, candidate, plugin): pass - def cleanup_tmp(self, candidate): - # local art source does not create tmp files - pass - class RemoteArtSource(ArtSource): IS_LOCAL = False @@ -298,7 +294,7 @@ class RemoteArtSource(ArtSource): self._log.debug(u'error fetching art: {}', exc) return - def cleanup_tmp(self, candidate): + def cleanup(self, candidate): if candidate.path: try: util.remove(path=candidate.path) @@ -1031,8 +1027,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): u'using {0.LOC_STR} image {1}'.format( source, util.displayable_path(out.path))) break - # else: remove tmp images created by this invalid candidate - source.cleanup_tmp(candidate) + # Remove temporary files for invalid candidates. + source.cleanup(candidate) if out: break diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d8c26bb9..c09b1b8ab 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -122,9 +122,11 @@ New features: Fixes: -* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take +* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take environment variables such as proxy servers into account when making requests :bug:`3450` +* :doc:`/plugins/fetchart`: Temporary files for fetched album art that fail + validation are now removed * :doc:`/plugins/inline`: In function-style field definitions that refer to flexible attributes, values could stick around from one function invocation to the next. This meant that, when displaying a list of objects, later From 7c71bb87a2f549c063af979b50e2160cd32058ed Mon Sep 17 00:00:00 2001 From: soergeld Date: Fri, 8 May 2020 16:32:12 +0200 Subject: [PATCH 483/613] cleaning up, renaming dup_XXInfo() to copy() --- beets/autotag/hooks.py | 23 +++++-------- beetsplug/discogs.py | 19 ++++------ test/test_autotag.py | 78 +++++++++++++++--------------------------- test/test_ui.py | 5 +-- 4 files changed, 44 insertions(+), 81 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index c2775bf32..6e0cada53 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -80,9 +80,9 @@ class AlbumInfo(AttrDict): ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, album=None, album_id=None, artist=None, artist_id=None, - tracks=None, asin=None, albumtype=None, va=False, year=None, - month=None, day=None, label=None, mediums=None, + def __init__(self, tracks, album=None, album_id=None, artist=None, + artist_id=None, asin=None, albumtype=None, va=False, + year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, language=None, country=None, style=None, genre=None, albumstatus=None, media=None, albumdisambig=None, @@ -144,18 +144,13 @@ class AlbumInfo(AttrDict): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) - if 'tracks' in self: - for track in self.tracks: - track.decode(codec) + for track in self.tracks: + track.decode(codec) - def dup_albuminfo(self): - dupe = AlbumInfo() + def copy(self): + dupe = AlbumInfo([]) dupe.update(self) - if 'tracks' in self: - tracks = [] - for track in self.tracks: - tracks.append(track.dup_trackinfo()) - dupe.tracks = tracks + dupe.tracks = [track.copy() for track in self.tracks] return dupe @@ -219,7 +214,7 @@ class TrackInfo(AttrDict): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) - def dup_trackinfo(self): + def copy(self): dupe = TrackInfo() dupe.update(self) return dupe diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index a0a6ea654..038fa809a 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -357,17 +357,13 @@ class DiscogsPlugin(BeetsPlugin): original_year = self.get_master_year(master_id) if master_id else year return AlbumInfo(album=album, album_id=album_id, artist=artist, - artist_id=artist_id, tracks=tracks, asin=None, - albumtype=albumtype, va=va, year=year, month=None, - day=None, label=label, mediums=len(set(mediums)), - artist_sort=None, releasegroup_id=master_id, - catalognum=catalogno, script=None, language=None, + artist_id=artist_id, tracks=tracks, + albumtype=albumtype, va=va, year=year, + label=label, mediums=len(set(mediums)), + releasegroup_id=master_id, catalognum=catalogno, country=country, style=style, genre=genre, - albumstatus=None, media=media, - albumdisambig=None, artist_credit=None, - original_year=original_year, original_month=None, - original_day=None, data_source='Discogs', - data_url=data_url, + media=media, original_year=original_year, + data_source='Discogs', data_url=data_url, discogs_albumid=discogs_albumid, discogs_labelid=labelid, discogs_artistid=artist_id) @@ -570,8 +566,7 @@ class DiscogsPlugin(BeetsPlugin): length = self.get_track_length(track['duration']) return TrackInfo(title=title, track_id=track_id, artist=artist, artist_id=artist_id, length=length, index=index, - medium=medium, medium_index=medium_index, - artist_sort=None, disctitle=None, artist_credit=None) + medium=medium, medium_index=medium_index) def get_track_index(self, position): """Returns the medium, medium index and subtrack index for a discogs diff --git a/test/test_autotag.py b/test/test_autotag.py index cef6364d0..febd1641d 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -350,9 +350,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) self.assertEqual(self._dist(items, info), 0) @@ -364,9 +362,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) dist = self._dist(items, info) self.assertNotEqual(dist, 0) @@ -382,9 +378,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'someone else', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) self.assertNotEqual(self._dist(items, info), 0) @@ -397,9 +391,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'should be ignored', album=u'some album', tracks=_make_trackinfo(), - va=True, - album_id=None, - artist_id=None, + va=True ) self.assertEqual(self._dist(items, info), 0) @@ -413,9 +405,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'should be ignored', album=u'some album', tracks=_make_trackinfo(), - va=True, - album_id=None, - artist_id=None, + va=True ) info.tracks[0].artist = None info.tracks[1].artist = None @@ -431,9 +421,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=True, - album_id=None, - artist_id=None, + va=True ) self.assertNotEqual(self._dist(items, info), 0) @@ -446,9 +434,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) dist = self._dist(items, info) self.assertTrue(0 < dist < 0.2) @@ -462,9 +448,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) info.tracks[0].medium_index = 1 info.tracks[1].medium_index = 2 @@ -481,9 +465,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) info.tracks[0].medium_index = 1 info.tracks[1].medium_index = 2 @@ -505,9 +487,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 2)) items.append(self.item(u'two', 3)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'two', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'two')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -524,9 +506,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 1)) items.append(self.item(u'two', 1)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'two', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'two')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -542,9 +524,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'one', 1)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'two', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'two')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -560,8 +542,8 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'two', 2)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, [items[1]]) @@ -597,7 +579,7 @@ class AssignmentTest(unittest.TestCase): items.append(item(12, 186.45916150485752)) def info(index, title, length): - return TrackInfo(title=title, track_id=None, length=length, + return TrackInfo(title=title, length=length, index=index) trackinfo = [] trackinfo.append(info(1, u'Alone', 238.893)) @@ -752,15 +734,13 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].albumtype, 'album') def test_album_artist_overrides_empty_track_artist(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') def test_album_artist_overridden_by_nonempty_track_artist(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -782,8 +762,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].artist_sort, 'albumArtistSort') def test_full_date_applied(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -798,8 +777,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item(year=1, month=2, day=3)) self.items.append(Item(year=4, month=5, day=6)) - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.year = 2013 self._apply(info=my_info) @@ -819,8 +797,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[0].day, 3) def test_data_source_applied(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -882,8 +859,7 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): self.assertFalse(self.items[1].comp) def test_va_flag_sets_comp(self): - # make a deepcopy of self.info - va_info = self.info.dup_albuminfo() + va_info = self.info.copy() va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) diff --git a/test/test_ui.py b/test/test_ui.py index d0be10060..b1e7e8fad 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1138,10 +1138,7 @@ class SummarizeItemsTest(_common.TestCase): self.assertEqual(summary, u"1 items, F, 4kbps, 10:54, 987.0 B") # make a copy of self.item - i2 = library.Item() - i2.bitrate = 4321 - i2.length = 10 * 60 + 54 - i2.format = "F" + i2 = self.item.copy() summary = commands.summarize_items([self.item, i2], False) self.assertEqual(summary, u"2 items, F, 4kbps, 21:48, 1.9 KiB") From 4933671c10b955a6ce7a732b520d0b6c4a9994a0 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Fri, 8 May 2020 21:36:56 +0200 Subject: [PATCH 484/613] review: updated subsonicplaylist.py --- beetsplug/subsonicplaylist.py | 77 ++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 14cd855b1..d02a8919d 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -15,24 +15,44 @@ from __future__ import absolute_import, division, print_function -from hashlib import md5 -import xml.etree.ElementTree as ET import random import string +import xml.etree.ElementTree as ET +from hashlib import md5 from urllib.parse import urlencode import requests -from beets.dbcore.query import SubstringQuery from beets.dbcore import AndQuery - +from beets.dbcore.query import SubstringQuery +from beets.plugins import BeetsPlugin from beets.ui import Subcommand -from beets.plugins import BeetsPlugin - __author__ = 'https://github.com/MrNuggelz' +def filter_to_be_removed(items, keys): + if len(items) > len(keys): + dont_remove = [] + for artist, album, title in keys: + for item in items: + if artist == item['artist'] and \ + album == item['album'] and \ + title == item['title']: + dont_remove.append(item) + return [item for item in items if item not in dont_remove] + else: + def to_be_removed(item): + for artist, album, title in keys: + if artist == item['artist'] and\ + album == item['album'] and\ + title == item['title']: + return False + return True + + return [item for item in items if to_be_removed(item)] + + class SubsonicPlaylistPlugin(BeetsPlugin): def __init__(self): @@ -80,12 +100,12 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def build_playlist(lib, opts, args): self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() - if len(self.config['playlist_names'].as_str_seq()) > 0: + if self.config['playlist_names'].as_str_seq(): playlists = ET.fromstring(self.send('getPlaylists').text)[ 0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ - ' but no erro message found' + ' but no error message found' self._log.warn( playlists.attrib.get('message', alt_error)) return @@ -93,17 +113,17 @@ class SubsonicPlaylistPlugin(BeetsPlugin): for playlist in playlists: if name == playlist.attrib['name']: ids.append(playlist.attrib['id']) - playlist_dict = dict() - for playlist_id in ids: - name, tracks = self.get_playlist(playlist_id) - for track in tracks: - if track not in playlist_dict: - playlist_dict[track] = ';' - playlist_dict[track] += name + ';' + + playlist_dict = self.get_playlists(ids) + # delete old tags if self.config['delete']: - for item in lib.items('subsonic_playlist:";"'): - item.update({'subsonic_playlist': ''}) + existing = list(lib.items('subsonic_playlist:";"')) + to_be_removed = filter_to_be_removed( + existing, + playlist_dict.keys()) + for item in to_be_removed: + item['subsonic_playlist'] = '' with lib.transaction(): item.try_sync(write=True, move=False) @@ -129,10 +149,21 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def send(self, endpoint, params=None): if params is None: params = dict() - url = '{}/rest/{}?u={}&t={}&s={}&v=1.12.0&c=beets'.format( - self.config['base_url'].get(), - endpoint, - self.config['username'], - *self.generate_token()) - resp = requests.get(url + urlencode(params)) + a, b = self.generate_token() + params['u'] = self.config['username'] + params['t'] = a + params['s'] = b + params['v'] = '1.12.0' + params['c'] = 'beets' + resp = requests.get('{}/rest/{}?{}'.format(self.config['base_url'].get(),endpoint,urlencode(params))) return resp + + def get_playlists(self, ids): + output = dict() + for playlist_id in ids: + name, tracks = self.get_playlist(playlist_id) + for track in tracks: + if track not in output: + output[track] = ';' + output[track] += name + ';' + return output \ No newline at end of file From 4b7f42d21458166ef5eefb9b3068d8ff0447016c Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 9 May 2020 12:29:52 +0200 Subject: [PATCH 485/613] Update AttrDict docstring Co-authored-by: Adrian Sampson --- beets/autotag/hooks.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 6e0cada53..9bdf6b001 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -40,18 +40,8 @@ except AttributeError: # Classes used to represent candidate options. class AttrDict(dict): - """ - Dictionary with flexible attributes - to get an tag value: - value = info.tag - or value = info[tag] - or value = info.get(tag) - or value = getattr(info, tag) - all raise AttributeError when info doesn't have tag - to set a tag value: - info.tag = value - or info[tag] = value - or setattr(info, tag, value) + """A dictionary that supports attribute ("dot") access, so `d.field` + is equivalent to `d['field']`. """ def __getattr__(self, attr): From 8fa103e0de4de0ebb01685345f44e229cf4dab21 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 9 May 2020 12:44:36 +0200 Subject: [PATCH 486/613] changelog entry --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2f1521eb..5434e8b13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -115,6 +115,9 @@ New features: :bug:`3459` * :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. :bug:`3530` +* The classes ``AlbumInfo`` and ``TrackInfo`` now have flexible attributes, + allowing to solve :bug:`1547`. + Thanks to :user:`dosoe`. Fixes: From 5d90296a20b336349261e1d39a95503a605112f9 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 9 May 2020 13:16:56 +0200 Subject: [PATCH 487/613] fixed flake8 issues --- beetsplug/subsonicplaylist.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index d02a8919d..95bdaa886 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -155,7 +155,11 @@ class SubsonicPlaylistPlugin(BeetsPlugin): params['s'] = b params['v'] = '1.12.0' params['c'] = 'beets' - resp = requests.get('{}/rest/{}?{}'.format(self.config['base_url'].get(),endpoint,urlencode(params))) + resp = requests.get('{}/rest/{}?{}'.format( + self.config['base_url'].get(), + endpoint, + urlencode(params)) + ) return resp def get_playlists(self, ids): @@ -166,4 +170,4 @@ class SubsonicPlaylistPlugin(BeetsPlugin): if track not in output: output[track] = ';' output[track] += name + ';' - return output \ No newline at end of file + return output From c948530279f6488964b7715e131946484f0fa5dc Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 13:05:55 +0200 Subject: [PATCH 488/613] Fetching parentwork if changed, even if not force --- beetsplug/parentwork.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index d40254696..e2460e875 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -173,13 +173,17 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return hasparent = hasattr(item, 'parentwork') - if force or not hasparent: + workfalse = True + if hasattr(item, 'mb_workid_current'): + workfalse = item.mb_workid_current != item.mb_workid + if force or (not hasparent) or workfalse: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) + parent_info['mb_workid_current'] = item.mb_workid if 'parent_composer' in parent_info: self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], @@ -202,4 +206,5 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return ui.show_model_changes( item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', - 'parent_composer_sort', 'work_date']) + 'parent_composer_sort', 'work_date', + 'mb_workid_current']) From 42e6fdbc9bddb047f94a26e2ae53987a938c8a6c Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:00:50 +0200 Subject: [PATCH 489/613] Instead of applying all tags from a list, apply all not in a list --- beets/autotag/__init__.py | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index f9e38413e..d3bb48117 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -157,52 +157,52 @@ def apply_metadata(album_info, mapping): # Track alt. item.track_alt = track_info.track_alt - # Miscellaneous/nullable metadata. + # Metadata that has already been set misc_fields = { 'album': ( - 'albumtype', - 'label', - 'asin', - 'catalognum', - 'script', - 'language', - 'country', - 'style', - 'genre', - 'discogs_albumid', - 'discogs_artistid', - 'discogs_labelid', - 'albumstatus', - 'albumdisambig', - 'releasegroupdisambig', - 'data_source', + 'va', + 'releasegroup_id', + 'artist_id', + 'album_id', + 'mediums', + 'tracks', + 'year', + 'month', + 'day', + 'artist', + 'artist_credit', ), 'track': ( - 'disctitle', - 'lyricist', - 'media', - 'composer', - 'composer_sort', - 'arranger', - 'work', - 'mb_workid', - 'work_disambig', - 'bpm', - 'initial_key', - 'genre' + 'track_alt', + 'artist_id', + 'release_track_id', + 'medium', + 'index', + 'medium_index', + 'title', + 'artist_credit', + 'artist_sort', + 'artist', + 'track_id', + 'medium_total', + 'data_url' ) } # Don't overwrite fields with empty values unless the # field is explicitly allowed to be overwritten - for field in misc_fields['album']: + for field in album_info.keys(): + if field in misc_fields['album']: + continue clobber = field in config['overwrite_null']['album'].as_str_seq() value = getattr(album_info, field) if value is None and not clobber: continue item[field] = value - for field in misc_fields['track']: + for field in track_info.keys(): + if field in misc_fields['track']: + continue clobber = field in config['overwrite_null']['track'].as_str_seq() value = getattr(track_info, field) if value is None and not clobber: From f39bc1f894606f0e13e8a09b060d9b6991f80741 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:09:51 +0200 Subject: [PATCH 490/613] forgot a tag in the exclude list --- beets/autotag/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index d3bb48117..9cde24e27 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -171,6 +171,7 @@ def apply_metadata(album_info, mapping): 'day', 'artist', 'artist_credit', + 'artist_sort' ), 'track': ( 'track_alt', From e7d7c72ae151f086070c27b88be926835c9e22ef Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:16:57 +0200 Subject: [PATCH 491/613] forgot another one --- beets/autotag/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 9cde24e27..945dd06e9 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -171,7 +171,8 @@ def apply_metadata(album_info, mapping): 'day', 'artist', 'artist_credit', - 'artist_sort' + 'artist_sort', + 'data_url' ), 'track': ( 'track_alt', From cef8ab65367575b59559795f893b5f957ddf79c1 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:27:52 +0200 Subject: [PATCH 492/613] Same for tracks --- beets/autotag/__init__.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 945dd06e9..f79606aaa 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -52,20 +52,24 @@ def apply_item_metadata(item, track_info): if track_info.data_source: item.data_source = track_info.data_source - if track_info.lyricist is not None: - item.lyricist = track_info.lyricist - if track_info.composer is not None: - item.composer = track_info.composer - if track_info.composer_sort is not None: - item.composer_sort = track_info.composer_sort - if track_info.arranger is not None: - item.arranger = track_info.arranger - if track_info.work is not None: - item.work = track_info.work - if track_info.mb_workid is not None: - item.mb_workid = track_info.mb_workid - if track_info.work_disambig is not None: - item.work_disambig = track_info.work_disambig + misc_fields = ['artist_id', + 'release_track_id', + 'title', + 'artist_credit', + 'artist_sort', + 'artist', + 'track_id', + 'data_url' + ] + + for field in track_info.keys(): + if field in misc_fields: + continue + clobber = field in config['overwrite_null']['track'].as_str_seq() + value = getattr(track_info, field) + if value is None and not clobber: + continue + item[field] = value # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? From 2d15060bf8d4fbb4a677f7cc68e6c7a2fd138676 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 20:02:54 +0200 Subject: [PATCH 493/613] more forgotten tags --- beets/autotag/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index f79606aaa..670d55212 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -59,7 +59,8 @@ def apply_item_metadata(item, track_info): 'artist_sort', 'artist', 'track_id', - 'data_url' + 'data_url', + 'length' ] for field in track_info.keys(): @@ -191,7 +192,8 @@ def apply_metadata(album_info, mapping): 'artist', 'track_id', 'medium_total', - 'data_url' + 'data_url', + 'length' ) } From 1f93ce5a49fc356566b00da165bc4d35d1ca4bfc Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 11 May 2020 16:14:52 +0200 Subject: [PATCH 494/613] small changes, docs --- beetsplug/parentwork.py | 8 ++++---- docs/plugins/parentwork.rst | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index e2460e875..e4978ebf4 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -174,16 +174,16 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) hasparent = hasattr(item, 'parentwork') workfalse = True - if hasattr(item, 'mb_workid_current'): + if hasattr(item, 'parentwork_workid_current'): workfalse = item.mb_workid_current != item.mb_workid - if force or (not hasparent) or workfalse: + if force or not hasparent or workfalse: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) - parent_info['mb_workid_current'] = item.mb_workid + parent_info['parentwork_workid_current'] = item.mb_workid if 'parent_composer' in parent_info: self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], @@ -207,4 +207,4 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date', - 'mb_workid_current']) + 'parentwork_workid_current']) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index 9707650b4..e25f01395 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -18,7 +18,7 @@ example, all the movements of a symphony. This plugin aims to solve this problem by also fetching the parent work, which would be the whole symphony in this example. -This plugin adds five tags: +This plugin adds six tags: - **parentwork**: The title of the parent work. - **mb_parentworkid**: The musicbrainz id of the parent work. @@ -27,6 +27,9 @@ This plugin adds five tags: - **parent_composer_sort**: The sort name of the parent work composer. - **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. +- **parentwork_workid_current**: The musicbrainz id of the work as it was when + the parentwork was retrieved. This tag exists only for internal bookkeeping, + to keep track of recordings whose work have changed. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From dda7e41aebf45b177e471f5f65d5ca91725aa8c0 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 11 May 2020 16:41:20 +0200 Subject: [PATCH 495/613] various small changes --- beets/autotag/__init__.py | 101 +++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 56 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 670d55212..dea1fe756 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -35,6 +35,41 @@ from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') +# metadata that is already hardcoded +MISC_FIELDS = { + 'album': ( + 'va', + 'releasegroup_id', + 'artist_id', + 'album_id', + 'mediums', + 'tracks', + 'year', + 'month', + 'day', + 'artist', + 'artist_credit', + 'artist_sort', + 'data_url' + ), + 'track': ( + 'track_alt', + 'artist_id', + 'release_track_id', + 'medium', + 'index', + 'medium_index', + 'title', + 'artist_credit', + 'artist_sort', + 'artist', + 'track_id', + 'medium_total', + 'data_url', + 'length' + ) +} + # Additional utilities for the main interface. @@ -52,23 +87,11 @@ def apply_item_metadata(item, track_info): if track_info.data_source: item.data_source = track_info.data_source - misc_fields = ['artist_id', - 'release_track_id', - 'title', - 'artist_credit', - 'artist_sort', - 'artist', - 'track_id', - 'data_url', - 'length' - ] - - for field in track_info.keys(): - if field in misc_fields: + for field, value in track_info.items(): + # only overwrite fields that are not already hardcoded + if field in MISC_FIELDS['track']: continue - clobber = field in config['overwrite_null']['track'].as_str_seq() - value = getattr(track_info, field) - if value is None and not clobber: + if value is None: continue item[field] = value @@ -162,54 +185,20 @@ def apply_metadata(album_info, mapping): # Track alt. item.track_alt = track_info.track_alt - # Metadata that has already been set - misc_fields = { - 'album': ( - 'va', - 'releasegroup_id', - 'artist_id', - 'album_id', - 'mediums', - 'tracks', - 'year', - 'month', - 'day', - 'artist', - 'artist_credit', - 'artist_sort', - 'data_url' - ), - 'track': ( - 'track_alt', - 'artist_id', - 'release_track_id', - 'medium', - 'index', - 'medium_index', - 'title', - 'artist_credit', - 'artist_sort', - 'artist', - 'track_id', - 'medium_total', - 'data_url', - 'length' - ) - } - # Don't overwrite fields with empty values unless the # field is explicitly allowed to be overwritten - for field in album_info.keys(): - if field in misc_fields['album']: + for field, value in album_info.items(): + # only overwrite fields that are not already hardcoded + if field in MISC_FIELDS['album']: continue clobber = field in config['overwrite_null']['album'].as_str_seq() - value = getattr(album_info, field) if value is None and not clobber: continue item[field] = value - for field in track_info.keys(): - if field in misc_fields['track']: + for field, value in track_info.items(): + # only overwrite fields that are not already hardcoded + if field in MISC_FIELDS['track']: continue clobber = field in config['overwrite_null']['track'].as_str_seq() value = getattr(track_info, field) From fd1fd2182b3ee3af6aa775f3127ade69594ac1d4 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Mon, 11 May 2020 20:09:04 +0200 Subject: [PATCH 496/613] changed "SubstringQuery" to "MatchQuery" to prevent wrong song selection --- beetsplug/subsonicplaylist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 95bdaa886..c537c4093 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -24,7 +24,7 @@ from urllib.parse import urlencode import requests from beets.dbcore import AndQuery -from beets.dbcore.query import SubstringQuery +from beets.dbcore.query import MatchQuery from beets.plugins import BeetsPlugin from beets.ui import Subcommand @@ -71,9 +71,9 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def update_tags(self, playlist_dict, lib): with lib.transaction(): for query, playlist_tag in playlist_dict.items(): - query = AndQuery([SubstringQuery("artist", query[0]), - SubstringQuery("album", query[1]), - SubstringQuery("title", query[2])]) + query = AndQuery([MatchQuery("artist", query[0]), + MatchQuery("album", query[1]), + MatchQuery("title", query[2])]) items = lib.items(query) if not items: self._log.warn(u"{} | track not found ({})", playlist_tag, From 918fea5ecaa729de335a8da8eb170021559caafb Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 12 May 2020 16:05:08 +0200 Subject: [PATCH 497/613] small changes, documentation --- beetsplug/parentwork.py | 6 +++--- docs/plugins/parentwork.rst | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index e4978ebf4..781485ba4 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -173,10 +173,10 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return hasparent = hasattr(item, 'parentwork') - workfalse = True + work_changed = True if hasattr(item, 'parentwork_workid_current'): - workfalse = item.mb_workid_current != item.mb_workid - if force or not hasparent or workfalse: + work_changed = item.mb_workid_current != item.mb_workid + if force or not hasparent or work_changed: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index e25f01395..a3b203c7b 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -18,6 +18,12 @@ example, all the movements of a symphony. This plugin aims to solve this problem by also fetching the parent work, which would be the whole symphony in this example. +It uses the ``mb_workid`` tag to retrieve the *parent work* and stores the +``mb_workid`` as ``parentwork_workid_current``. When one executes +``beet parentwork`` it checks if ``mb_workid`` has changed by comparing it +with ``parentwork_workid_current``, if it has changed or if ``force`` is +enabled it fetches the *parent work* again. + This plugin adds six tags: - **parentwork**: The title of the parent work. @@ -27,7 +33,7 @@ This plugin adds six tags: - **parent_composer_sort**: The sort name of the parent work composer. - **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. -- **parentwork_workid_current**: The musicbrainz id of the work as it was when +- **parentwork_workid_current**: The Musicbrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, to keep track of recordings whose work have changed. @@ -41,7 +47,8 @@ To configure the plugin, make a ``parentwork:`` section in your configuration file. The available options are: - **force**: As a default, ``parentwork`` only fetches work info for - recordings that do not already have a ``parentwork`` tag. If ``force`` + recordings that do not already have a ``parentwork`` tag or where + ``mb_workid`` differs from ``parentwork_workid_current``. If ``force`` is enabled, it fetches it for all recordings. Default: ``no`` From 2923e90da1f3d0887d582024f2cefee56db5e923 Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 00:16:18 +0200 Subject: [PATCH 498/613] cosmetics --- beets/autotag/__init__.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index dea1fe756..5fc5b1c6c 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -36,7 +36,7 @@ from .match import Recommendation # noqa log = logging.getLogger('beets') # metadata that is already hardcoded -MISC_FIELDS = { +SPECIAL_FIELDS = { 'album': ( 'va', 'releasegroup_id', @@ -84,12 +84,10 @@ def apply_item_metadata(item, track_info): item.mb_releasetrackid = track_info.release_track_id if track_info.artist_id: item.mb_artistid = track_info.artist_id - if track_info.data_source: - item.data_source = track_info.data_source for field, value in track_info.items(): # only overwrite fields that are not already hardcoded - if field in MISC_FIELDS['track']: + if field in SPECIAL_FIELDS['track']: continue if value is None: continue @@ -188,8 +186,7 @@ def apply_metadata(album_info, mapping): # Don't overwrite fields with empty values unless the # field is explicitly allowed to be overwritten for field, value in album_info.items(): - # only overwrite fields that are not already hardcoded - if field in MISC_FIELDS['album']: + if field in SPECIAL_FIELDS['album']: continue clobber = field in config['overwrite_null']['album'].as_str_seq() if value is None and not clobber: @@ -197,8 +194,7 @@ def apply_metadata(album_info, mapping): item[field] = value for field, value in track_info.items(): - # only overwrite fields that are not already hardcoded - if field in MISC_FIELDS['track']: + if field in SPECIAL_FIELDS['track']: continue clobber = field in config['overwrite_null']['track'].as_str_seq() value = getattr(track_info, field) From 5db2b4ccaa7223345b02626df188476aabc838ae Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 13 May 2020 08:20:50 -0400 Subject: [PATCH 499/613] Fix new flake8 warnings --- beetsplug/replaygain.py | 2 +- test/test_player.py | 2 ++ test/test_playlist.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 6f15b454e..0ba04783d 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -613,7 +613,7 @@ class FfmpegBackend(Backend): return float(value) except ValueError: raise ReplayGainError( - u"ffmpeg output: expected float value, found {1}" + u"ffmpeg output: expected float value, found {0}" .format(value) ) diff --git a/test/test_player.py b/test/test_player.py index 021a04fc0..dd47ac62b 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -241,6 +241,8 @@ def implements(commands, expectedFailure=False): # noqa: N803 bluelet_listener = bluelet.Listener + + @mock.patch("beets.util.bluelet.Listener") def start_server(args, assigned_port, listener_patch): """Start the bpd server, writing the port to `assigned_port`. diff --git a/test/test_playlist.py b/test/test_playlist.py index edd98e711..88abb733f 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -120,7 +120,7 @@ class PlaylistQueryTestHelper(PlaylistTestHelper): ])) def test_name_query_with_nonexisting_playlist(self): - q = u'playlist:nonexisting'.format(self.playlist_dir) + q = u'playlist:nonexisting' results = self.lib.items(q) self.assertEqual(set(results), set()) From e8b57dfda6057404906dfffefabf0e562923faf3 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Wed, 13 May 2020 14:55:00 +0200 Subject: [PATCH 500/613] Update docs/plugins/parentwork.rst Co-authored-by: Adrian Sampson --- docs/plugins/parentwork.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index a3b203c7b..1aea9e9ff 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -35,7 +35,7 @@ This plugin adds six tags: that has a composition date. Format: yyyy-mm-dd. - **parentwork_workid_current**: The Musicbrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, - to keep track of recordings whose work have changed. + to keep track of recordings whose works have changed. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From 26a78a0c280c480601258c151e8185e9167a6b3c Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 16:54:04 +0200 Subject: [PATCH 501/613] Cosmetics --- beets/autotag/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 5fc5b1c6c..057daa405 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -35,7 +35,7 @@ from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') -# metadata that is already hardcoded +# Metadata that is already hardcoded. SPECIAL_FIELDS = { 'album': ( 'va', @@ -86,7 +86,7 @@ def apply_item_metadata(item, track_info): item.mb_artistid = track_info.artist_id for field, value in track_info.items(): - # only overwrite fields that are not already hardcoded + # We only overwrite fields that are not already hardcoded. if field in SPECIAL_FIELDS['track']: continue if value is None: From 66085a231f88c1fcec91c7ef1807af91a1f1e084 Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 17:42:49 +0200 Subject: [PATCH 502/613] Cosmetics --- beets/autotag/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 057daa405..7ab0d57fd 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -35,7 +35,7 @@ from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') -# Metadata that is already hardcoded. +# Metadata fields that are already hardcoded, or where the tag name changes. SPECIAL_FIELDS = { 'album': ( 'va', From ed97b984b02cc2ff41da9b1ec73931687b11ce79 Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 18:02:14 +0200 Subject: [PATCH 503/613] Documentation --- docs/plugins/parentwork.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index 1aea9e9ff..cc3f21c0c 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -18,22 +18,24 @@ example, all the movements of a symphony. This plugin aims to solve this problem by also fetching the parent work, which would be the whole symphony in this example. -It uses the ``mb_workid`` tag to retrieve the *parent work* and stores the -``mb_workid`` as ``parentwork_workid_current``. When one executes -``beet parentwork`` it checks if ``mb_workid`` has changed by comparing it -with ``parentwork_workid_current``, if it has changed or if ``force`` is -enabled it fetches the *parent work* again. +The plugin can detect changes in ``mb_workid`` so it knows when to re-fetch +other metadata, such as ``parentwork``. To do this, when it runs, it stores a +copy of ``mb_workid`` in the bookkeeping field ``parentwork_workid_current``. +At any later run of ``beet parentwork`` it will check if the tags +``mb_workid`` and ``parentwork_workid_current`` are still identical. If it is +not the case, it means the work has changed and all the tags need to be +fetched again. This plugin adds six tags: - **parentwork**: The title of the parent work. -- **mb_parentworkid**: The musicbrainz id of the parent work. +- **mb_parentworkid**: The MusicBrainz id of the parent work. - **parentwork_disambig**: The disambiguation of the parent work title. - **parent_composer**: The composer of the parent work. - **parent_composer_sort**: The sort name of the parent work composer. - **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. -- **parentwork_workid_current**: The Musicbrainz id of the work as it was when +- **parentwork_workid_current**: The MusicBrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, to keep track of recordings whose works have changed. @@ -53,5 +55,5 @@ configuration file. The available options are: Default: ``no`` - **auto**: If enabled, automatically fetches works at import. It takes quite - some time, because beets is restricted to one musicbrainz query per second. + some time, because beets is restricted to one MusicBrainz query per second. Default: ``no`` From fe418b946ee8e2c4fd62c86c36ec8059fb5c57c3 Mon Sep 17 00:00:00 2001 From: soergeld Date: Thu, 14 May 2020 22:40:37 +0200 Subject: [PATCH 504/613] forgot one mb_workid_current occurence --- beetsplug/parentwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 781485ba4..dea7ba040 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -175,7 +175,7 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) hasparent = hasattr(item, 'parentwork') work_changed = True if hasattr(item, 'parentwork_workid_current'): - work_changed = item.mb_workid_current != item.mb_workid + work_changed = item.parentwork_workid_current != item.mb_workid if force or not hasparent or work_changed: try: work_info, work_date = find_parentwork_info(item.mb_workid) From 05d202760bb75290477ea7be341cb76a3e58751d Mon Sep 17 00:00:00 2001 From: Thorsten Date: Sat, 16 May 2020 00:34:10 +0200 Subject: [PATCH 505/613] Update genres.txt Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres --- beetsplug/lastgenre/genres.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beetsplug/lastgenre/genres.txt b/beetsplug/lastgenre/genres.txt index 914ee1290..7ccd7ad3b 100644 --- a/beetsplug/lastgenre/genres.txt +++ b/beetsplug/lastgenre/genres.txt @@ -450,6 +450,8 @@ emo rap emocore emotronic enka +epic doom metal +epic metal eremwu eu ethereal pop ethereal wave @@ -1024,6 +1026,7 @@ neo-medieval neo-prog neo-psychedelia neoclassical +neoclassical metal neoclassical music neofolk neotraditional country @@ -1176,8 +1179,10 @@ progressive folk progressive folk music progressive house progressive metal +progressive power metal progressive rock progressive trance +progressive thrash metal protopunk psych folk psychedelic music @@ -1396,6 +1401,7 @@ symphonic metal symphonic poem symphonic rock symphony +synthcore synthpop synthpunk t'ong guitar @@ -1428,6 +1434,7 @@ tejano tejano music tekno tembang sunda +teutonic thrash metal texas blues thai pop thillana @@ -1444,6 +1451,7 @@ toeshey togaku trad jazz traditional bluegrass +traditional heavy metal traditional pop music trallalero trance From 852cdb6dfa2e34c112442a2078c2b85c9f2501d0 Mon Sep 17 00:00:00 2001 From: Thorsten Date: Sat, 16 May 2020 00:34:37 +0200 Subject: [PATCH 506/613] Update genres-tree.yaml Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres --- beetsplug/lastgenre/genres-tree.yaml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/beetsplug/lastgenre/genres-tree.yaml b/beetsplug/lastgenre/genres-tree.yaml index a09f7e6b3..c8ae42478 100644 --- a/beetsplug/lastgenre/genres-tree.yaml +++ b/beetsplug/lastgenre/genres-tree.yaml @@ -648,35 +648,51 @@ - glam rock - hard rock - heavy metal: - - alternative metal + - alternative metal: + - funk metal - black metal: - viking metal - christian metal - death metal: + - death/doom - goregrind - melodic death metal - technical death metal - - doom metal + - doom metal: + - epic doom metal + - funeral doom - drone metal + - epic metal - folk metal: - celtic metal - medieval metal + - pagan metal - funk metal - glam metal - gothic metal + - industrial metal: + - industrial death metal - metalcore: - deathcore - mathcore: - djent - - power metal + - synthcore + - neoclassical metal + - post-metal + - power metal: + - progressive power metal - progressive metal - sludge metal - speed metal - - stoner rock + - stoner rock: + - stoner metal - symphonic metal - thrash metal: - crossover thrash - groove metal + - progressive thrash metal + - teutonic thrash metal + - traditional heavy metal - math rock - new wave: - world fusion @@ -719,6 +735,7 @@ - street punk - thrashcore - horror punk + - oi! - pop punk - psychobilly - riot grrrl From cd4d8c7f9c3d8fa3fd1706befc223dd920bdef65 Mon Sep 17 00:00:00 2001 From: Thorsten Date: Sat, 16 May 2020 00:51:30 +0200 Subject: [PATCH 507/613] Update changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 545145f0d..f00380a2c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. From ac043f9be0094089ed29ec8c0552781994945607 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 13:14:39 +0200 Subject: [PATCH 508/613] When fetching lyrics from Genius, search for title and artist. Searching only for the title and just verifying the artist afterwards leads to songs with very common titles not being found, since Genius limits the amount of returned hits. An example would be 'Saviour' by 'Circa Waves'. --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 20e1c8858..3a56cc738 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -390,7 +390,7 @@ class Genius(Backend): def fetch(self, artist, title): search_url = self.base_url + "/search" - data = {'q': title} + data = {'q': title + " " + artist.lower()} try: response = requests.get(search_url, data=data, headers=self.headers) From 46143d97629c48777e82dc147ecfff08f7c5e683 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 16:28:17 +0200 Subject: [PATCH 509/613] Remove unnecessary intermediate web request to genius.com when fetching lyrics. The search results already include the correct song page url, making it superfluous to do another request via the /song api just to get it. --- beetsplug/lyrics.py | 20 ++++---------------- test/test_lyrics.py | 8 ++++---- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 3a56cc738..71969cab3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -359,14 +359,9 @@ class Genius(Backend): 'User-Agent': USER_AGENT, } - def lyrics_from_song_api_path(self, song_api_path): - song_url = self.base_url + song_api_path - response = requests.get(song_url, headers=self.headers) - json = response.json() - path = json["response"]["song"]["path"] - + def lyrics_from_song_page(self, page_url): # Gotta go regular html scraping... come on Genius. - page_url = "https://genius.com" + path + self._log.debug(u'fetching lyrics from: {0}', page_url) try: page = requests.get(page_url) except requests.RequestException as exc: @@ -404,7 +399,6 @@ class Genius(Backend): self._log.debug(u'Genius API request returned invalid JSON') return None - song_info = None for hit in json["response"]["hits"]: # Genius uses zero-width characters to denote lowercase # artist names. @@ -412,15 +406,9 @@ class Genius(Backend): strip(u'\u200b').lower() if hit_artist == artist.lower(): - song_info = hit - break + return self.lyrics_from_song_page(hit["result"]["url"]) - if song_info: - self._log.debug(u'fetched: {0}', song_info["result"]["url"]) - song_api_path = song_info["result"]["api_path"] - return self.lyrics_from_song_api_path(song_api_path) - else: - self._log.debug(u'genius: no matching artist') + self._log.debug(u'genius: no matching artist') class LyricsWiki(SymbolsReplaced): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 81d62fea9..d0507c16f 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -456,7 +456,7 @@ class LyricsGeniusBaseTest(unittest.TestCase): self.skipTest("Python's built-in HTML parser is not good enough") -class LyricsGeniusScrapTest(LyricsGeniusBaseTest): +class LyricsGeniusScrapeTest(LyricsGeniusBaseTest): """Checks that Genius backend works as intended. """ @@ -469,12 +469,12 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): - """Ensure that `lyrics_from_song_api_path` doesn't crash when the html - for a Genius page contain
+ """Ensure that `lyrics_from_song_page` doesn't crash when the html + for a Genius page doesn't contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None - self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + self.assertEqual(genius.lyrics_from_song_page('https://genius.com/sample'), None) From 15402f6aa7bceceef8f3909b5e5939342bd549ca Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 17:15:45 +0200 Subject: [PATCH 510/613] Add alternative scraping algorithm to deal with Genius's new song page html layout. --- beetsplug/lyrics.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 71969cab3..27a29e149 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,13 +373,34 @@ class Genius(Backend): # Remove script tags that they put in the middle of the lyrics. [h.extract() for h in html('script')] - # At least Genius is nice and has a tag called 'lyrics'! - # Updated css where the lyrics are based in HTML. + # Most of the time, the page contains a div with class="lyrics" where + # all of the lyrics can be found already correctly formatted + # Sometimes, though, it packages the lyrics into separate divs, most + # likely for easier ad placement lyrics_div = html.find("div", class_="lyrics") - if lyrics_div is None: - self._log.debug(u'Genius lyrics for {0} not found', - page_url) - return None + if not lyrics_div: + self._log.debug(u'Received unusual song page html') + verse_div = html.find("div", + class_=re.compile("Lyrics__Container")) + if not verse_div: + with open('instrumental.html', 'w') as text_file: + text_file.write(str(html)) + if html.find("div", + class_=re.compile("LyricsPlaceholder__Message"), + string="This song is an instrumental"): + self._log.debug('Detected instrumental') + return "[Instrumental]" + else: + self._log.debug("Couldn't scrape page using known layouts") + return None + + lyrics_div = verse_div.parent + for br in lyrics_div.find_all("br"): + br.replace_with("\n") + ads = lyrics_div.find_all("div", + class_=re.compile("InreadAd__Container")) + for ad in ads: + ad.replace_with("\n") return lyrics_div.get_text() From 5d306d6fd9b48b51cbf25934cd5ded90a83b1b02 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 17:48:56 +0200 Subject: [PATCH 511/613] Remove debugging statement and conform to line length. --- beetsplug/lyrics.py | 2 -- test/test_lyrics.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 27a29e149..71e1d4ef3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -383,8 +383,6 @@ class Genius(Backend): verse_div = html.find("div", class_=re.compile("Lyrics__Container")) if not verse_div: - with open('instrumental.html', 'w') as text_file: - text_file.write(str(html)) if html.find("div", class_=re.compile("LyricsPlaceholder__Message"), string="This song is an instrumental"): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index d0507c16f..d31116284 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -474,7 +474,8 @@ class LyricsGeniusScrapeTest(LyricsGeniusBaseTest): """ # https://github.com/beetbox/beets/issues/3535 # expected return value None - self.assertEqual(genius.lyrics_from_song_page('https://genius.com/sample'), + song_url = 'https://genius.com/sample' + self.assertEqual(genius.lyrics_from_song_page(song_url), None) From 5298e1a97656ab5c8f33f95a5a537c645e35cca1 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sun, 17 May 2020 18:42:23 +0200 Subject: [PATCH 512/613] Add changelog entry. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f00380a2c..bd1a77973 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -202,6 +202,7 @@ Fixes: The log message has also been rewritten for to improve clarity. Thanks to :user:`autrimpo`. :bug:`3533` +* :doc:`/plugins/lyrics`: Reduce the failure rate of the Genius backend. For plugin developers: From 5c91013d1f291b60b295f0ebb3dc0a17d88fd2f4 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 May 2020 14:46:10 -0400 Subject: [PATCH 513/613] Expand changelog for #3594 --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bd1a77973..3f2890913 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -202,7 +202,9 @@ Fixes: The log message has also been rewritten for to improve clarity. Thanks to :user:`autrimpo`. :bug:`3533` -* :doc:`/plugins/lyrics`: Reduce the failure rate of the Genius backend. +* :doc:`/plugins/lyrics`: Adapt the Genius backend to changes in markup to + reduce the scraping failure rate. + :bug:`3535` :bug:`3594` For plugin developers: From 1374aa2aea1df83b11c77218c354c54de991df59 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 18 May 2020 10:23:26 -0400 Subject: [PATCH 514/613] feat: uses new subsonic format as default also gets rid of warning that port must be `int` when it's default was a `string` --- beetsplug/subsonicupdate.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 469a9d142..29b01986e 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -17,11 +17,9 @@ Your Beets configuration file should contain a "subsonic" section like the following: subsonic: - host: 192.168.x.y (Subsonic server IP) - port: 4040 (default) - user: - pass: - contextpath: /subsonic + url: https://mydomain.com:443/subsonic + user: username + pass: password """ from __future__ import division, absolute_import, print_function @@ -84,11 +82,8 @@ class SubsonicUpdate(BeetsPlugin): # Set default configuration values config['subsonic'].add({ - 'host': 'localhost', - 'port': '4040', 'user': 'admin', 'pass': 'admin', - 'contextpath': '/', 'url': 'http://localhost:4040', }) From d9603788bcc6e991773c4fec07a57c42c157ad6b Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 22 May 2020 16:24:49 -0400 Subject: [PATCH 515/613] fix: use `endwith()` instead of `endWith()` --- beetsplug/subsonicupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 29b01986e..354eb701f 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -61,7 +61,7 @@ def format_url(): """ url = config['subsonic']['url'].as_str() - if url and url.endsWith('/'): + if url and url.endswith('/'): url = url[:-1] # @deprecated("Use url config option instead") From 66e6b11404842585a26c3241a9c76b919a4d9c4c Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 22 May 2020 20:35:18 -0400 Subject: [PATCH 516/613] refactor: moved static methods inside class made them static private class methods --- beetsplug/subsonicupdate.py | 86 ++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 354eb701f..45fc3a8cb 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -35,47 +35,6 @@ from beets.plugins import BeetsPlugin __author__ = 'https://github.com/maffo999' -def create_token(): - """Create salt and token from given password. - - :return: The generated salt and hashed token - """ - password = config['subsonic']['pass'].as_str() - - # Pick the random sequence and salt the password - r = string.ascii_letters + string.digits - salt = "".join([random.choice(r) for _ in range(6)]) - salted_password = password + salt - token = hashlib.md5().update(salted_password.encode('utf-8')).hexdigest() - - # Put together the payload of the request to the server and the URL - return salt, token - - -def format_url(): - """Get the Subsonic URL to trigger a scan. Uses either the url - config option or the deprecated host, port, and context_path config - options together. - - :return: Endpoint for updating Subsonic - """ - - url = config['subsonic']['url'].as_str() - if url and url.endswith('/'): - url = url[:-1] - - # @deprecated("Use url config option instead") - if not url: - host = config['subsonic']['host'].as_str() - port = config['subsonic']['port'].get(int) - context_path = config['subsonic']['contextpath'].as_str() - if context_path == '/': - context_path = '' - url = "http://{}:{}{}".format(host, port, context_path) - - return url + '/rest/startScan' - - class SubsonicUpdate(BeetsPlugin): def __init__(self): super(SubsonicUpdate, self).__init__() @@ -90,10 +49,51 @@ class SubsonicUpdate(BeetsPlugin): config['subsonic']['pass'].redact = True self.register_listener('import', self.start_scan) + @staticmethod + def __create_token(): + """Create salt and token from given password. + + :return: The generated salt and hashed token + """ + password = config['subsonic']['pass'].as_str() + + # Pick the random sequence and salt the password + r = string.ascii_letters + string.digits + salt = "".join([random.choice(r) for _ in range(6)]) + salted_password = password + salt + token = hashlib.md5(salted_password.encode('utf-8')).hexdigest() + + # Put together the payload of the request to the server and the URL + return salt, token + + @staticmethod + def __format_url(): + """Get the Subsonic URL to trigger a scan. Uses either the url + config option or the deprecated host, port, and context_path config + options together. + + :return: Endpoint for updating Subsonic + """ + + url = config['subsonic']['url'].as_str() + if url and url.endswith('/'): + url = url[:-1] + + # @deprecated("Use url config option instead") + if not url: + host = config['subsonic']['host'].as_str() + port = config['subsonic']['port'].get(int) + context_path = config['subsonic']['contextpath'].as_str() + if context_path == '/': + context_path = '' + url = "http://{}:{}{}".format(host, port, context_path) + + return url + '/rest/startScan' + def start_scan(self): user = config['subsonic']['user'].as_str() - url = format_url() - salt, token = create_token() + url = self.__format_url() + salt, token = self.__create_token() payload = { 'u': user, From 10f589d80ad83b54dd86edf1ceb677d762b4be3b Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 22 May 2020 20:35:32 -0400 Subject: [PATCH 517/613] feat: added subsonicupdate tests --- test/test_subsonic.py | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 test/test_subsonic.py diff --git a/test/test_subsonic.py b/test/test_subsonic.py new file mode 100644 index 000000000..6d37cdf4f --- /dev/null +++ b/test/test_subsonic.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +"""Tests for the 'subsonic' plugin""" + +from __future__ import division, absolute_import, print_function + +import requests +import responses +import unittest + +from test import _common +from beets import config +from beetsplug import subsonicupdate +from test.helper import TestHelper +from six.moves.urllib.parse import parse_qs, urlparse + + +class ArgumentsMock(object): + def __init__(self, mode, show_failures): + self.mode = mode + self.show_failures = show_failures + self.verbose = 1 + + +def _params(url): + """Get the query parameters from a URL.""" + return parse_qs(urlparse(url).query) + + +class SubsonicPluginTest(_common.TestCase, TestHelper): + @responses.activate + def setUp(self): + config.clear() + self.setup_beets() + + config["subsonic"]["user"] = "admin" + config["subsonic"]["pass"] = "admin" + config["subsonic"]["url"] = "http://localhost:4040" + + self.subsonicupdate = subsonicupdate.SubsonicUpdate() + + def tearDown(self): + self.teardown_beets() + + @responses.activate + def test_start_scan(self): + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_extra_forward_slash_url(self): + config["subsonic"]["url"] = "http://localhost:4040/contextPath" + + responses.add( + responses.POST, + 'http://localhost:4040/contextPath/rest/startScan', + status=200 + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_context_path(self): + config["subsonic"]["url"] = "http://localhost:4040/" + + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_missing_port(self): + config["subsonic"]["url"] = "http://localhost/airsonic" + + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + with self.assertRaises(requests.exceptions.ConnectionError): + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_missing_schema(self): + config["subsonic"]["url"] = "localhost:4040/airsonic" + + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + with self.assertRaises(requests.exceptions.InvalidSchema): + self.subsonicupdate.start_scan() + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From edb31f69d00868c2c70967e634c41fb54ecbe59e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 23 May 2020 09:03:12 -0400 Subject: [PATCH 518/613] Remove explicit jellyfish dep from Tox This is already installed when installing beets, and it's done in a way that actually gets the right package version for the right Python version. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9dac2d59a..19cb3069b 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,6 @@ deps = rarfile responses>=0.3.0 pyxdg - jellyfish python-mpd2 coverage discogs-client From fc4a61c55768f73b54098cb06465b77887ca4f55 Mon Sep 17 00:00:00 2001 From: Martin Kirchgessner Date: Sat, 23 May 2020 17:09:18 +0200 Subject: [PATCH 519/613] add drop2beets to community plugins --- docs/plugins/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index de2f2930c..aab922fcd 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -318,6 +318,10 @@ Here are a few of the plugins written by the beets community: * `beets-originquery`_ augments MusicBrainz queries with locally-sourced data to improve autotagger results. +* `drop2beets`_ automatically imports singles as soon as they are dropped in a + folder (using Linux's ``inotify``). You can also set a sub-folders + hierarchy to set flexible attributes by the way. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -347,3 +351,4 @@ Here are a few of the plugins written by the beets community: .. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser .. _beets-originquery: https://github.com/x1ppy/beets-originquery +.. _drop2beets: https://github.com/martinkirch/drop2beets From 0c3201930c13190eebade4141d314640dfe2b2f3 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Sun, 24 May 2020 11:20:19 -0400 Subject: [PATCH 520/613] feat: added github actions only working with linux for now. exact replica of travis basically. should probably try to implement the rest of tox or deprecate some functionality. --- .github/workflows/ci.yaml | 77 ++++++++++++++++++++++++++++++++ beetsplug/export.py | 14 +++--- beetsplug/lyrics.py | 4 +- beetsplug/metasync/amarok.py | 4 +- beetsplug/plexupdate.py | 4 +- beetsplug/smartplaylist.py | 8 ++-- beetsplug/subsonicplaylist.py | 9 ++-- beetsplug/thumbnails.py | 2 +- setup.py | 4 +- test/test_export.py | 4 +- tox.ini | 83 +++++++++++++++++++---------------- 11 files changed, 150 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..cbd6381e1 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,77 @@ +name: ci +on: push +jobs: + test-27: + runs-on: ${{ matrix.platform }} + strategy: + matrix: + platform: [ ubuntu-latest ] + env: + NOSE_SHOW_SKIPPED: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up python 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + pip install tox sphinx + - name: Test with tox + run: | + tox -e py27-test + tox -e docs + test-3x: + runs-on: ${{ matrix.platform }} + strategy: + matrix: + platform: [ ubuntu-latest ] + python-version: [ 5, 6, 7, 8 ] + env: + NOSE_SHOW_SKIPPED: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up python 3.${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + pip install tox sphinx + - name: Test with tox + run: | + if [[ ${{ matrix.python-version }} == '8' ]]; then + tox -e py3${{ matrix.python-version }}-test + tox -e py3${{ matrix.python-version }}-cov + tox -e py3${{ matrix.python-version }}-flake8 + else + tox -e py3${{ matrix.python-version }}-test + fi + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python to build docs with sphinx + uses: actions/setup-python@v2 + with: + python-version: 2.7 + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox sphinx + - name: Build and check docs using tox + run: tox -e docs \ No newline at end of file diff --git a/beetsplug/export.py b/beetsplug/export.py index f7e84a570..0b4876bbf 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -21,7 +21,7 @@ import sys import codecs import json import csv -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et from datetime import datetime, date from beets.plugins import BeetsPlugin @@ -188,18 +188,18 @@ class XMLFormat(ExportFormat): def export(self, data, **kwargs): # Creates the XML file structure. - library = ET.Element(u'library') - tracks = ET.SubElement(library, u'tracks') + library = et.Element(u'library') + tracks = et.SubElement(library, u'tracks') if data and isinstance(data[0], dict): for index, item in enumerate(data): - track = ET.SubElement(tracks, u'track') + track = et.SubElement(tracks, u'track') for key, value in item.items(): - track_details = ET.SubElement(track, key) + track_details = et.SubElement(track, key) track_details.text = value # Depending on the version of python the encoding needs to change try: - data = ET.tostring(library, encoding='unicode', **kwargs) + data = et.tostring(library, encoding='unicode', **kwargs) except LookupError: - data = ET.tostring(library, encoding='utf-8', **kwargs) + data = et.tostring(library, encoding='utf-8', **kwargs) self.out_stream.write(data) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 71e1d4ef3..f73d72dfd 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -907,7 +907,7 @@ class LyricsPlugin(plugins.BeetsPlugin): return _scrape_strip_cruft(lyrics, True) def append_translation(self, text, to_lang): - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as et if not self.bing_auth_token: self.bing_auth_token = self.get_bing_access_token() @@ -925,7 +925,7 @@ class LyricsPlugin(plugins.BeetsPlugin): self.bing_auth_token = None return self.append_translation(text, to_lang) return text - lines_translated = ET.fromstring(r.text.encode('utf-8')).text + lines_translated = et.fromstring(r.text.encode('utf-8')).text # Use a translation mapping dict to build resulting lyrics translations = dict(zip(text_lines, lines_translated.split('|'))) result = '' diff --git a/beetsplug/metasync/amarok.py b/beetsplug/metasync/amarok.py index 0622fc17a..1d28eb3c9 100644 --- a/beetsplug/metasync/amarok.py +++ b/beetsplug/metasync/amarok.py @@ -49,7 +49,7 @@ class Amarok(MetaSource): 'amarok_lastplayed': DateType(), } - queryXML = u' \ + query_xml = u' \ \ \ \ @@ -72,7 +72,7 @@ class Amarok(MetaSource): # of the result set. So query for the filename and then try to match # the correct item from the results we get back results = self.collection.Query( - self.queryXML % quoteattr(basename(path)) + self.query_xml % quoteattr(basename(path)) ) for result in results: if result['xesam:url'] != path: diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index 17fd8208d..d3d7caf5c 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -12,7 +12,7 @@ Put something like the following in your config.yaml to configure: from __future__ import division, absolute_import, print_function import requests -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et from six.moves.urllib.parse import urljoin, urlencode from beets import config from beets.plugins import BeetsPlugin @@ -28,7 +28,7 @@ def get_music_section(host, port, token, library_name): r = requests.get(url) # Parse xml tree and extract music section key. - tree = ET.fromstring(r.content) + tree = et.fromstring(r.content) for child in tree.findall('Directory'): if child.get('title') == library_name: return child.get('key') diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 4ffdd21e7..f3157c96f 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -105,17 +105,17 @@ class SmartPlaylistPlugin(BeetsPlugin): playlist_data = (playlist['name'],) try: - for key, Model in (('query', Item), ('album_query', Album)): + for key, model in (('query', Item), ('album_query', Album)): qs = playlist.get(key) if qs is None: query_and_sort = None, None elif isinstance(qs, six.string_types): - query_and_sort = parse_query_string(qs, Model) + query_and_sort = parse_query_string(qs, model) elif len(qs) == 1: - query_and_sort = parse_query_string(qs[0], Model) + query_and_sort = parse_query_string(qs[0], model) else: # multiple queries and sorts - queries, sorts = zip(*(parse_query_string(q, Model) + queries, sorts = zip(*(parse_query_string(q, model) for q in qs)) query = OrQuery(queries) final_sorts = [] diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index c537c4093..a4577278a 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function import random import string -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et from hashlib import md5 from urllib.parse import urlencode @@ -85,7 +85,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def get_playlist(self, playlist_id): xml = self.send('getPlaylist', {'id': playlist_id}).text - playlist = ET.fromstring(xml)[0] + playlist = et.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' self._log.warn(playlist.attrib.get('message', alt_error)) @@ -101,8 +101,9 @@ class SubsonicPlaylistPlugin(BeetsPlugin): self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() if self.config['playlist_names'].as_str_seq(): - playlists = ET.fromstring(self.send('getPlaylists').text)[ - 0] + playlists = et.fromstring( + self.send('getPlaylists').text + )[0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ ' but no error message found' diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index fe36fbd13..01da5afff 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -25,7 +25,7 @@ from hashlib import md5 import os import shutil from itertools import chain -from pathlib import PurePosixPath +from pathlib2 import PurePosixPath import ctypes import ctypes.util diff --git a/setup.py b/setup.py index c50e65bf5..0be671f5d 100755 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ setup( 'requests_oauthlib' ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. - ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] + ['pathlib2'] if (sys.version_info < (3, 4, 0)) else [] ), # Plugin (optional) dependencies: @@ -144,7 +144,7 @@ setup( 'web': ['flask', 'flask-cors'], 'import': ['rarfile'], 'thumbnails': ['pyxdg', 'Pillow'] + - (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), + (['pathlib2'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], 'scrub': ['mutagen>=1.33'], diff --git a/test/test_export.py b/test/test_export.py index 757212a38..18b2b17be 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -23,7 +23,7 @@ from test.helper import TestHelper import re # used to test csv format import json from xml.etree.ElementTree import Element -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et class ExportPluginTest(unittest.TestCase, TestHelper): @@ -85,7 +85,7 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='xml', artist=item1.artist ) - library = ET.fromstring(out) + library = et.fromstring(out) self.assertIsInstance(library, Element) for track in library[0]: for details in track: diff --git a/tox.ini b/tox.ini index 19cb3069b..06f0bda8c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,54 +4,63 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py27-flake8, docs +envlist = + py27-{test} + py38-{test,cov,flake8} + docs -# The exhaustive list of environments is: -# envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs +# exhaustive list +# py{27,35,36,37,38}-{test,cov,flake8}, docs -[_test] -deps = - beautifulsoup4 - flask - mock - nose - nose-show-skipped - pylast - rarfile - responses>=0.3.0 - pyxdg - python-mpd2 - coverage - discogs-client - requests_oauthlib - -[_flake8] -deps = - flake8 - flake8-coding - flake8-future-import - flake8-blind-except - pep8-naming~=0.7.0 -files = beets beetsplug beet test setup.py docs +[gh-actions:env] +PLATFORM = + ubuntu-latest: linux + windows-latest: windows [testenv] passenv = NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. deps = {test,cov}: {[_test]deps} - py27: pathlib - py{27,34,35,36,37,38}-flake8: {[_flake8]deps} + flake8: {[_flake8]deps} + py27: pathlib2 commands = - py27-cov: python -m nose --with-coverage {posargs} py27-test: python -m nose {posargs} - py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py38-cov: python -bb -m nose --with-coverage {posargs} + py38-flake8: flake8 {posargs} {[_flake8]files} py3{4,5,6,7,8}-test: python -bb -m nose {posargs} - py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} - py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} - py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} - py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} - py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} - py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} + +[_test] +deps = + beautifulsoup4 + coverage + flask + discogs-client + mock + nose + nose-show-skipped + pylast + python-mpd2 + pyxdg + rarfile + requests_oauthlib + responses + sphinx + +[_flake8] +deps = + flake8 + flake8-blind-except + flake8-coding + flake8-future-import + pep8-naming +files = + beet + beets + beetsplug + docs + setup.py + test [testenv:docs] basepython = python2.7 From 9f8bd4cd3fb1a9d92a270be39d76e77f1a168647 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 25 May 2020 19:39:40 -0400 Subject: [PATCH 521/613] style: flake8 linting --- beetsplug/export.py | 14 +++++++------- beetsplug/lyrics.py | 5 +++-- beetsplug/plexupdate.py | 4 ++-- beetsplug/subsonicplaylist.py | 9 ++++----- beetsplug/thumbnails.py | 2 +- setup.py | 4 ++-- test/test_export.py | 4 ++-- tox.ini | 2 +- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 0b4876bbf..8d98d0ba2 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -21,7 +21,7 @@ import sys import codecs import json import csv -import xml.etree.ElementTree as et +from xml.etree import ElementTree from datetime import datetime, date from beets.plugins import BeetsPlugin @@ -188,18 +188,18 @@ class XMLFormat(ExportFormat): def export(self, data, **kwargs): # Creates the XML file structure. - library = et.Element(u'library') - tracks = et.SubElement(library, u'tracks') + library = ElementTree.Element(u'library') + tracks = ElementTree.SubElement(library, u'tracks') if data and isinstance(data[0], dict): for index, item in enumerate(data): - track = et.SubElement(tracks, u'track') + track = ElementTree.SubElement(tracks, u'track') for key, value in item.items(): - track_details = et.SubElement(track, key) + track_details = ElementTree.SubElement(track, key) track_details.text = value # Depending on the version of python the encoding needs to change try: - data = et.tostring(library, encoding='unicode', **kwargs) + data = ElementTree.tostring(library, encoding='unicode', **kwargs) except LookupError: - data = et.tostring(library, encoding='utf-8', **kwargs) + data = ElementTree.tostring(library, encoding='utf-8', **kwargs) self.out_stream.write(data) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index f73d72dfd..8807225a9 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -907,7 +907,7 @@ class LyricsPlugin(plugins.BeetsPlugin): return _scrape_strip_cruft(lyrics, True) def append_translation(self, text, to_lang): - import xml.etree.ElementTree as et + from xml.etree import ElementTree if not self.bing_auth_token: self.bing_auth_token = self.get_bing_access_token() @@ -925,7 +925,8 @@ class LyricsPlugin(plugins.BeetsPlugin): self.bing_auth_token = None return self.append_translation(text, to_lang) return text - lines_translated = et.fromstring(r.text.encode('utf-8')).text + lines_translated = ElementTree.fromstring( + r.text.encode('utf-8')).text # Use a translation mapping dict to build resulting lyrics translations = dict(zip(text_lines, lines_translated.split('|'))) result = '' diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index d3d7caf5c..ab3396191 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -12,7 +12,7 @@ Put something like the following in your config.yaml to configure: from __future__ import division, absolute_import, print_function import requests -import xml.etree.ElementTree as et +from xml.etree import ElementTree from six.moves.urllib.parse import urljoin, urlencode from beets import config from beets.plugins import BeetsPlugin @@ -28,7 +28,7 @@ def get_music_section(host, port, token, library_name): r = requests.get(url) # Parse xml tree and extract music section key. - tree = et.fromstring(r.content) + tree = ElementTree.fromstring(r.content) for child in tree.findall('Directory'): if child.get('title') == library_name: return child.get('key') diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index a4577278a..732b51c2f 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function import random import string -import xml.etree.ElementTree as et +from xml.etree import ElementTree from hashlib import md5 from urllib.parse import urlencode @@ -85,7 +85,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def get_playlist(self, playlist_id): xml = self.send('getPlaylist', {'id': playlist_id}).text - playlist = et.fromstring(xml)[0] + playlist = ElementTree.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' self._log.warn(playlist.attrib.get('message', alt_error)) @@ -101,9 +101,8 @@ class SubsonicPlaylistPlugin(BeetsPlugin): self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() if self.config['playlist_names'].as_str_seq(): - playlists = et.fromstring( - self.send('getPlaylists').text - )[0] + playlists = ElementTree.fromstring( + self.send('getPlaylists').text)[0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ ' but no error message found' diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index 01da5afff..fe36fbd13 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -25,7 +25,7 @@ from hashlib import md5 import os import shutil from itertools import chain -from pathlib2 import PurePosixPath +from pathlib import PurePosixPath import ctypes import ctypes.util diff --git a/setup.py b/setup.py index 0be671f5d..c50e65bf5 100755 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ setup( 'requests_oauthlib' ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. - ['pathlib2'] if (sys.version_info < (3, 4, 0)) else [] + ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] ), # Plugin (optional) dependencies: @@ -144,7 +144,7 @@ setup( 'web': ['flask', 'flask-cors'], 'import': ['rarfile'], 'thumbnails': ['pyxdg', 'Pillow'] + - (['pathlib2'] if (sys.version_info < (3, 4, 0)) else []), + (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], 'scrub': ['mutagen>=1.33'], diff --git a/test/test_export.py b/test/test_export.py index 18b2b17be..779e74423 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -23,7 +23,7 @@ from test.helper import TestHelper import re # used to test csv format import json from xml.etree.ElementTree import Element -import xml.etree.ElementTree as et +from xml.etree import ElementTree class ExportPluginTest(unittest.TestCase, TestHelper): @@ -85,7 +85,7 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='xml', artist=item1.artist ) - library = et.fromstring(out) + library = ElementTree.fromstring(out) self.assertIsInstance(library, Element) for track in library[0]: for details in track: diff --git a/tox.ini b/tox.ini index 06f0bda8c..92086b820 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ passenv = deps = {test,cov}: {[_test]deps} flake8: {[_flake8]deps} - py27: pathlib2 + py27: pathlib commands = py27-test: python -m nose {posargs} py38-cov: python -bb -m nose --with-coverage {posargs} From 9a67cee95f442cd831163a6b262929f811ac93a7 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 25 May 2020 21:53:22 -0400 Subject: [PATCH 522/613] ci: added windows --- .github/workflows/ci.yaml | 42 +++++++++++++++++++++++++++------------ tox.ini | 1 + 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cbd6381e1..0f9ebbeba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,8 +5,9 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] env: + PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 @@ -15,26 +16,33 @@ jobs: with: python-version: 2.7 - uses: actions/cache@v1 + if: startsWith(runner.os, 'Linux') with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- + - uses: actions/cache@v1 + if: startsWith(runner.os, 'Windows') + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install base dependencies run: | python -m pip install --upgrade pip pip install tox sphinx - name: Test with tox - run: | - tox -e py27-test - tox -e docs + run: tox -e py27-test test-3x: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: + PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 @@ -43,24 +51,32 @@ jobs: with: python-version: 3.${{ matrix.python-version }} - uses: actions/cache@v1 + if: startsWith(runner.os, 'Linux') with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- + - uses: actions/cache@v1 + if: startsWith(runner.os, 'Windows') + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install base dependencies run: | python -m pip install --upgrade pip pip install tox sphinx - - name: Test with tox + - name: Test, coverage, and flake8 with tox + if: matrix.python-version == '8' run: | - if [[ ${{ matrix.python-version }} == '8' ]]; then - tox -e py3${{ matrix.python-version }}-test - tox -e py3${{ matrix.python-version }}-cov - tox -e py3${{ matrix.python-version }}-flake8 - else - tox -e py3${{ matrix.python-version }}-test - fi + tox -e py3${{ matrix.python-version }}-test + tox -e py3${{ matrix.python-version }}-cov + tox -e py3${{ matrix.python-version }}-flake8 + - name: Test with tox + if: matrix.python-version != '8' + run: tox -e py3${{ matrix.python-version }}-test docs: runs-on: ubuntu-latest steps: diff --git a/tox.ini b/tox.ini index 92086b820..b02e491fb 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ PLATFORM = [testenv] passenv = + LANG # avoids output errors NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. deps = {test,cov}: {[_test]deps} From b0f456b3c3410a9af3a78169801c5f55e39e839a Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 25 May 2020 23:18:59 -0400 Subject: [PATCH 523/613] ci: removed windows for now has to do with UnicodeDecodeError --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0f9ebbeba..f38ce898e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,7 +5,7 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] env: PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 @@ -39,7 +39,7 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: PY_COLOR: 1 From e8693acb6ae41c1824f0258b6b1e02e9d04402e1 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Tue, 26 May 2020 10:40:10 -0400 Subject: [PATCH 524/613] docs: elaborated exhaustive list Co-authored-by: Adrian Sampson --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b02e491fb..055f1e572 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ envlist = py38-{test,cov,flake8} docs -# exhaustive list +# The exhaustive list of environments is: # py{27,35,36,37,38}-{test,cov,flake8}, docs [gh-actions:env] From 0068afcb991a4c911142975f1a33dd3b5ab95c3b Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Tue, 26 May 2020 12:40:23 -0400 Subject: [PATCH 525/613] feat: add more options to tox --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 055f1e572..7c32fc460 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,11 @@ deps = flake8: {[_flake8]deps} py27: pathlib commands = + py27-cov: python -m nose --with-coverage {posargs} + py27-flake8: flake8 {posargs} {[_flake8]files} py27-test: python -m nose {posargs} - py38-cov: python -bb -m nose --with-coverage {posargs} - py38-flake8: flake8 {posargs} {[_flake8]files} + py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py3{4,5,6,7,8}-flake8: flake8 {posargs} {[_flake8]files} py3{4,5,6,7,8}-test: python -bb -m nose {posargs} [_test] From 9c93b4d35652c2b0aa90f1d5b1cc67bfeebb88ef Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Wed, 27 May 2020 16:36:27 -0400 Subject: [PATCH 526/613] docs: update punctuation Co-authored-by: Adrian Sampson --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f38ce898e..5c736e1ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ jobs: NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 - - name: Set up python 2.7 + - name: Set up Python 2.7 uses: actions/setup-python@v2 with: python-version: 2.7 @@ -46,7 +46,7 @@ jobs: NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 - - name: Set up python 3.${{ matrix.python-version }} + - name: Set up Python 3.${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: 3.${{ matrix.python-version }} @@ -81,7 +81,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python to build docs with sphinx + - name: Set up Python to build docs with Sphinx uses: actions/setup-python@v2 with: python-version: 2.7 @@ -90,4 +90,4 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx - name: Build and check docs using tox - run: tox -e docs \ No newline at end of file + run: tox -e docs From d3b759d7ca874695a04e13d4b49e1957515acd26 Mon Sep 17 00:00:00 2001 From: ybnd Date: Fri, 29 May 2020 13:29:31 +0200 Subject: [PATCH 527/613] Fix `list -af` example in documentation Didn't work since $tracktotal is not an Album field --- docs/reference/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index b66920d96..724afc80a 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -212,7 +212,7 @@ The ``-p`` option makes beets print out filenames of matched items, which might be useful for piping into other Unix commands (such as `xargs`_). Similarly, the ``-f`` option lets you specify a specific format with which to print every album or track. This uses the same template syntax as beets' :doc:`path formats -`. For example, the command ``beet ls -af '$album: $tracktotal' +`. For example, the command ``beet ls -af '$album: $albumtotal' beatles`` prints out the number of tracks on each Beatles album. In Unix shells, remember to enclose the template argument in single quotes to avoid environment variable expansion. From c69e96432a4bf9a82d683f7d8885aabb54fe39df Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Sun, 31 May 2020 00:10:32 -0400 Subject: [PATCH 528/613] importer: use tarfile.open to handle compressed archives Call tarfile.open instead of tarfile.TarFile from the importer so that we can import compressed tar archives. Note that tarfile.TarFile does not handle compressed archives: $ python3 Python 3.8.2 (default, Apr 27 2020, 15:53:34) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import tarfile >>> tf = tarfile.TarFile("Lagrimas.tar.bz2") Traceback (most recent call last): [...] tarfile.ReadError: invalid header >>> But tarfile.open does deal with them: $ python3 Python 3.8.2 (default, Apr 27 2020, 15:53:34) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import tarfile >>> tf = tarfile.open("Lagrimas.tar.bz2") >>> Tested: $ ls Lagrimas/*.mp3 | wc -l 11 $ tar cjf Lagrimas.tar.bz2 Lagrimas/ - Before: $ beet import Lagrimas.tar.bz2 extraction failed: invalid header No files imported from /tmp/Lagrimas.tar.bz2 - After: $ beet import Lagrimas.tar.bz2 [works] Fixes #3606. --- beets/importer.py | 4 ++-- docs/changelog.rst | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index e97b0a75c..68d5f3d5d 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -1034,8 +1034,8 @@ class ArchiveImportTask(SentinelImportTask): cls._handlers = [] from zipfile import is_zipfile, ZipFile cls._handlers.append((is_zipfile, ZipFile)) - from tarfile import is_tarfile, TarFile - cls._handlers.append((is_tarfile, TarFile)) + import tarfile + cls._handlers.append((tarfile.is_tarfile, tarfile.open)) try: from rarfile import is_rarfile, RarFile except ImportError: diff --git a/docs/changelog.rst b/docs/changelog.rst index 3f2890913..21cbd1217 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -124,6 +124,8 @@ New features: separators to support path queries. Thanks to :user:`nmeum`. :bug:`3567` +* ``beet import`` now handles tar archives with bzip2 or gzip compression. + :bug:`3606` Fixes: From 18645ffbe4b406fd9d7d21d465a5b21c78672bd7 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 00:26:33 -0400 Subject: [PATCH 529/613] chore(tox): revert changes --- tox.ini | 72 ++++++++++++++++++++++++--------------------------------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/tox.ini b/tox.ini index 7c32fc460..2e56f6107 100644 --- a/tox.ini +++ b/tox.ini @@ -4,68 +4,56 @@ # and then run "tox" from this directory. [tox] -envlist = - py27-{test} - py38-{test,cov,flake8} - docs +envlist = py27-test, py37-test, py27-flake8, docs # The exhaustive list of environments is: -# py{27,35,36,37,38}-{test,cov,flake8}, docs - -[gh-actions:env] -PLATFORM = - ubuntu-latest: linux - windows-latest: windows - -[testenv] -passenv = - LANG # avoids output errors - NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. -deps = - {test,cov}: {[_test]deps} - flake8: {[_flake8]deps} - py27: pathlib -commands = - py27-cov: python -m nose --with-coverage {posargs} - py27-flake8: flake8 {posargs} {[_flake8]files} - py27-test: python -m nose {posargs} - py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} - py3{4,5,6,7,8}-flake8: flake8 {posargs} {[_flake8]files} - py3{4,5,6,7,8}-test: python -bb -m nose {posargs} +# envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs [_test] deps = beautifulsoup4 - coverage flask - discogs-client mock nose nose-show-skipped pylast - python-mpd2 - pyxdg rarfile + responses>=0.3.0 + pyxdg + python-mpd2 + coverage + discogs-client requests_oauthlib - responses - sphinx [_flake8] deps = flake8 - flake8-blind-except flake8-coding flake8-future-import - pep8-naming -files = - beet - beets - beetsplug - docs - setup.py - test + flake8-blind-except + pep8-naming~=0.7.0 +files = beets beetsplug beet test setup.py docs + +[testenv] +passenv = + NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. +deps = + {test,cov}: {[_test]deps} + py27: pathlib + py{27,34,35,36,37,38}-flake8: {[_flake8]deps} +commands = + py27-cov: python -m nose --with-coverage {posargs} + py27-test: python -m nose {posargs} + py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py3{4,5,6,7,8}-test: python -bb -m nose {posargs} + py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} + py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} + py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} + py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} + py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} + py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} [testenv:docs] basepython = python2.7 deps = sphinx -commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} +commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} \ No newline at end of file From e66cfd58ad679f02a37480717b19de9c1077e7e4 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:09:35 -0400 Subject: [PATCH 530/613] feat: add windows into ci --- .github/workflows/ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5c736e1ba..50ab7462a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,10 +5,11 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] env: - PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 + PY_COLOR: 1 + PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 2.7 @@ -39,11 +40,12 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: - PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 + PY_COLOR: 1 + PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 3.${{ matrix.python-version }} From 26caef544f5e2f5996dedaed3e6e20e99f9a5d1f Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:21:46 -0400 Subject: [PATCH 531/613] fix: remove `--min-version` --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 19cb3069b..9c1bc4106 100644 --- a/tox.ini +++ b/tox.ini @@ -46,12 +46,12 @@ commands = py27-test: python -m nose {posargs} py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} py3{4,5,6,7,8}-test: python -bb -m nose {posargs} - py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} - py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} - py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} - py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} - py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} - py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} + py27-flake8: flake8 {posargs} {[_flake8]files} + py34-flake8: flake8 {posargs} {[_flake8]files} + py35-flake8: flake8 {posargs} {[_flake8]files} + py36-flake8: flake8 {posargs} {[_flake8]files} + py37-flake8: flake8 {posargs} {[_flake8]files} + py38-flake8: flake8 {posargs} {[_flake8]files} [testenv:docs] basepython = python2.7 From f0534090c32ec386b8dfc21248b5426a70a271be Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:26:09 -0400 Subject: [PATCH 532/613] fix: revert add windows to ci --- .github/workflows/ci.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 50ab7462a..bec58f700 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,11 +5,10 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] env: NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 - PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 2.7 @@ -40,12 +39,11 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 - PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 3.${{ matrix.python-version }} From b9068e3fbd22409509da61a5a693bfbe174b6b16 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:40:05 -0400 Subject: [PATCH 533/613] fix: flake8 problem --- beets/autotag/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9bdf6b001..9841360ea 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -332,7 +332,7 @@ class Distance(object): self._penalties = {} @LazyClassProperty - def _weights(cls): # noqa + def _weights(self, cls): # noqa """A dictionary from keys to floating-point weights. """ weights_view = config['match']['distance_weights'] From 42e5844564740fd5fc5ba3393e03d96522352ff9 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:48:48 -0400 Subject: [PATCH 534/613] fix: revert flake8 change --- beets/autotag/hooks.py | 2 +- beetsplug/smartplaylist.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9841360ea..9bdf6b001 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -332,7 +332,7 @@ class Distance(object): self._penalties = {} @LazyClassProperty - def _weights(self, cls): # noqa + def _weights(cls): # noqa """A dictionary from keys to floating-point weights. """ weights_view = config['match']['distance_weights'] diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index f3157c96f..700b0c76a 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -105,17 +105,18 @@ class SmartPlaylistPlugin(BeetsPlugin): playlist_data = (playlist['name'],) try: - for key, model in (('query', Item), ('album_query', Album)): + for key, model_cls in (('query', Item), + ('album_query', Album)): qs = playlist.get(key) if qs is None: query_and_sort = None, None elif isinstance(qs, six.string_types): - query_and_sort = parse_query_string(qs, model) + query_and_sort = parse_query_string(qs, model_cls) elif len(qs) == 1: - query_and_sort = parse_query_string(qs[0], model) + query_and_sort = parse_query_string(qs[0], model_cls) else: # multiple queries and sorts - queries, sorts = zip(*(parse_query_string(q, model) + queries, sorts = zip(*(parse_query_string(q, model_cls) for q in qs)) query = OrQuery(queries) final_sorts = [] From b239a0b3d2dcbde0495750903dc2e69067d48ebf Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 4 Jun 2020 11:15:34 +0200 Subject: [PATCH 535/613] Fix item == None issues when writing lyrics ReST * Skip ReST writing & sphinx info messages if query doesn't yield anything * `writerest` into `appendrest` and `writerest`, don't call `writerest(item=None)` to flush state at the end. --- beetsplug/lyrics.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 71e1d4ef3..3e03bd224 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -770,7 +770,8 @@ class LyricsPlugin(plugins.BeetsPlugin): write = ui.should_write() if opts.writerest: self.writerest_indexes(opts.writerest) - for item in lib.items(ui.decargs(args)): + items = lib.items(ui.decargs(args)) + for item in items: if not opts.local_only and not self.config['local']: self.fetch_item_lyrics( lib, item, write, @@ -780,10 +781,10 @@ class LyricsPlugin(plugins.BeetsPlugin): if opts.printlyr: ui.print_(item.lyrics) if opts.writerest: - self.writerest(opts.writerest, item) - if opts.writerest: - # flush last artist - self.writerest(opts.writerest, None) + self.appendrest(opts.writerest, item) + if opts.writerest and len(items) > 0: + # flush last artist & write to ReST + self.writerest(opts.writerest) ui.print_(u'ReST files generated. to build, use one of:') ui.print_(u' sphinx-build -b html %s _build/html' % opts.writerest) @@ -795,26 +796,21 @@ class LyricsPlugin(plugins.BeetsPlugin): cmd.func = func return [cmd] - def writerest(self, directory, item): - """Write the item to an ReST file + def appendrest(self, directory, item): + """Append the item to an ReST file This will keep state (in the `rest` variable) in order to avoid writing continuously to the same files. """ - if item is None or slug(self.artist) != slug(item.albumartist): - if self.rest is not None: - path = os.path.join(directory, 'artists', - slug(self.artist) + u'.rst') - with open(path, 'wb') as output: - output.write(self.rest.encode('utf-8')) - self.rest = None - if item is None: - return + if slug(self.artist) != slug(item.albumartist): + # Write current file and start a new one ~ item.albumartist + self.writerest(directory) self.artist = item.albumartist.strip() self.rest = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ % (self.artist, u'=' * len(self.artist)) + if self.album != item.album: tmpalbum = self.album = item.album.strip() if self.album == '': @@ -826,6 +822,16 @@ class LyricsPlugin(plugins.BeetsPlugin): u'~' * len(title_str), block) + def writerest(self, directory): + """Write self.rest to a ReST file + """ + if self.rest is not None and self.artist is not None: + path = os.path.join(directory, 'artists', + slug(self.artist) + u'.rst') + with open(path, 'wb') as output: + output.write(self.rest.encode('utf-8')) + + def writerest_indexes(self, directory): """Write conf.py and index.rst files necessary for Sphinx From d1f3d664d75edb1f681264a2d578ddec21ea6d4f Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 4 Jun 2020 16:43:42 +0200 Subject: [PATCH 536/613] Implement comments & add to changelog --- beetsplug/lyrics.py | 3 +-- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 3e03bd224..05d3ea53f 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -782,7 +782,7 @@ class LyricsPlugin(plugins.BeetsPlugin): ui.print_(item.lyrics) if opts.writerest: self.appendrest(opts.writerest, item) - if opts.writerest and len(items) > 0: + if opts.writerest and items: # flush last artist & write to ReST self.writerest(opts.writerest) ui.print_(u'ReST files generated. to build, use one of:') @@ -831,7 +831,6 @@ class LyricsPlugin(plugins.BeetsPlugin): with open(path, 'wb') as output: output.write(self.rest.encode('utf-8')) - def writerest_indexes(self, directory): """Write conf.py and index.rst files necessary for Sphinx diff --git a/docs/changelog.rst b/docs/changelog.rst index 21cbd1217..691a42de5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -207,6 +207,9 @@ Fixes: * :doc:`/plugins/lyrics`: Adapt the Genius backend to changes in markup to reduce the scraping failure rate. :bug:`3535` :bug:`3594` +* :doc:`/plugins/lyrics`: Fix crash when writing ReST files for a query without + results or fetched lyrics + :bug:`2805` For plugin developers: From 14dc285b4f6348729e6cb3c7173fa3f786ef75b8 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 21:55:47 -0400 Subject: [PATCH 537/613] chore(flake8): update comment and version --- beets/autotag/hooks.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9bdf6b001..065d88170 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -332,7 +332,7 @@ class Distance(object): self._penalties = {} @LazyClassProperty - def _weights(cls): # noqa + def _weights(cls): # noqa: N805 """A dictionary from keys to floating-point weights. """ weights_view = config['match']['distance_weights'] diff --git a/tox.ini b/tox.ini index 9c1bc4106..047784f8b 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ deps = flake8-coding flake8-future-import flake8-blind-except - pep8-naming~=0.7.0 + pep8-naming files = beets beetsplug beet test setup.py docs [testenv] From c095429123c319fb355d22667a434dd1e64f42e5 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 22:03:16 -0400 Subject: [PATCH 538/613] feat: remove Travis CI in favor of GitHub Actions --- .travis.yml | 87 ----------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b889f698d..000000000 --- a/.travis.yml +++ /dev/null @@ -1,87 +0,0 @@ -dist: trusty -sudo: required -language: python - -env: - global: - # Undocumented feature of nose-show-skipped. - NOSE_SHOW_SKIPPED: 1 - -matrix: - include: - - python: 2.7.13 - env: {TOX_ENV: py27-cov, COVERAGE: 1} - - python: 2.7.13 - env: {TOX_ENV: py27-test} - # - python: 3.4 - # env: {TOX_ENV: py34-test} - # - python: 3.4_with_system_site_packages - # env: {TOX_ENV: py34-test} - - python: 3.5 - env: {TOX_ENV: py35-test} - - python: 3.6 - env: {TOX_ENV: py36-test} - - python: 3.7 - env: {TOX_ENV: py37-test} - dist: xenial - - python: 3.8 - env: {TOX_ENV: py38-test} - dist: xenial - # - python: pypy - # - env: {TOX_ENV: pypy-test} - - python: 3.6 - env: {TOX_ENV: py36-flake8} - - python: 2.7.13 - env: {TOX_ENV: docs} -# Non-Python dependencies. -addons: - apt: - sources: - - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty multiverse" - - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty-updates multiverse" - packages: - - bash-completion - - gir1.2-gst-plugins-base-1.0 - - gir1.2-gstreamer-1.0 - - gstreamer1.0-plugins-good - - gstreamer1.0-plugins-bad - - imagemagick - - python-gi - - python-gst-1.0 - - python3-gi - - python3-gst-1.0 - - unrar - -# To install dependencies, tell tox to do everything but actually running the -# test. -install: - - travis_retry pip install tox sphinx - # upgrade requests to satisfy sphinx linkcheck (for building man pages) - - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then pip install -U requests; fi - - travis_retry tox -e $TOX_ENV --notest - -script: - # prevents "libdc1394 error: Failed to initialize libdc1394" errors - - sudo ln -s /dev/null /dev/raw1394 - - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then SITE_PACKAGES=--sitepackages; fi - # pip in trusty breaks on packages prefixed with "_". See https://github.com/pypa/pip/issues/3681 - - if [[ $TRAVIS_PYTHON_VERSION == 3.4_with_system_site_packages ]]; then sudo rm -rf /usr/lib/python3/dist-packages/_lxc-0.1.egg-info; fi - - tox -e $TOX_ENV $SITE_PACKAGES - -# Report coverage to codecov.io. -before_install: - - "[ ! -z $COVERAGE ] && travis_retry pip install codecov || true" -after_success: - - "[ ! -z $COVERAGE ] && codecov || true" - -cache: - pip: true - -notifications: - irc: - channels: - - "irc.freenode.org#beets" - use_notice: true - skip_join: true - on_success: change - on_failure: always From a96059634ca80a907d2a5f9d7d4302da0445ed9d Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 22:09:18 -0400 Subject: [PATCH 539/613] feat: add `codecov` to GitHub Actions --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bec58f700..0f2c625a3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,6 +74,8 @@ jobs: tox -e py3${{ matrix.python-version }}-test tox -e py3${{ matrix.python-version }}-cov tox -e py3${{ matrix.python-version }}-flake8 + pip install codecov || true + codecov || true - name: Test with tox if: matrix.python-version != '8' run: tox -e py3${{ matrix.python-version }}-test From 1aa35edd3d56b3985284d70b2f0836f334ca52db Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 5 Jun 2020 10:03:37 -0400 Subject: [PATCH 540/613] Switch Travis badge to GitHub Actions --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0f653ac02..418ff0f56 100644 --- a/README.rst +++ b/README.rst @@ -4,8 +4,8 @@ .. image:: https://img.shields.io/codecov/c/github/beetbox/beets.svg :target: https://codecov.io/github/beetbox/beets -.. image:: https://travis-ci.org/beetbox/beets.svg?branch=master - :target: https://travis-ci.org/beetbox/beets +.. image:: https://github.com/beetbox/beets/workflows/ci/badge.svg?branch=master + :target: https://github.com/beetbox/beets/actions .. image:: https://repology.org/badge/tiny-repos/beets.svg :target: https://repology.org/project/beets/versions From 74ad96652883427e5d2d4d8363b21e7705bb6bd6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 5 Jun 2020 10:04:13 -0400 Subject: [PATCH 541/613] Run GitHub Actions CI on pull requests too --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0f2c625a3..5045a1b9c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,5 @@ name: ci -on: push +on: [push, pull_request] jobs: test-27: runs-on: ${{ matrix.platform }} From 2ae0dbbd7adbdf6f7ffe8a7f4ff343192df20721 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Jun 2020 16:29:33 -0400 Subject: [PATCH 542/613] docs: clarify `mbcollection` options --- docs/plugins/mbcollection.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/mbcollection.rst b/docs/plugins/mbcollection.rst index 113855bce..00acd4604 100644 --- a/docs/plugins/mbcollection.rst +++ b/docs/plugins/mbcollection.rst @@ -35,8 +35,8 @@ configuration file. There is one option available: - **auto**: Automatically amend your MusicBrainz collection whenever you import a new album. Default: ``no``. -- **collection**: Which MusicBrainz collection to update. +- **collection**: The MBID of which MusicBrainz collection to update. Default: ``None``. - **remove**: Remove albums from collections which are no longer present in the beets database. - Default: ``None``. + Default: ``no``. From 22d74a3b8a6978c213df922b1ef27bee60527d08 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Jun 2020 16:19:45 -0400 Subject: [PATCH 543/613] feat: support secure Plex connections --- beetsplug/plexupdate.py | 39 ++++++++++++++++++++++++++++--------- docs/plugins/plexupdate.rst | 4 ++++ test/test_plexupdate.py | 12 +++++++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index ab3396191..156d8835d 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -18,14 +18,16 @@ from beets import config from beets.plugins import BeetsPlugin -def get_music_section(host, port, token, library_name): +def get_music_section(host, port, token, library_name, use_secure, + ignore_cert_errors): """Getting the section key for the music library in Plex. """ api_endpoint = append_token('library/sections', token) - url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) + url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + port), api_endpoint) # Sends request. - r = requests.get(url) + r = requests.get(url, verify=not ignore_cert_errors) # Parse xml tree and extract music section key. tree = ElementTree.fromstring(r.content) @@ -34,17 +36,25 @@ def get_music_section(host, port, token, library_name): return child.get('key') -def update_plex(host, port, token, library_name): +def update_plex(host, port, token, library_name, use_secure, + ignore_cert_errors): + """Ignore certificate errors if configured to. + """ + if ignore_cert_errors: + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) """Sends request to the Plex api to start a library refresh. """ # Getting section key and build url. - section_key = get_music_section(host, port, token, library_name) + section_key = get_music_section(host, port, token, library_name, + use_secure, ignore_cert_errors) api_endpoint = 'library/sections/{0}/refresh'.format(section_key) api_endpoint = append_token(api_endpoint, token) - url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) + url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + port), api_endpoint) # Sends request and returns requests object. - r = requests.get(url) + r = requests.get(url, verify=not ignore_cert_errors) return r @@ -56,6 +66,13 @@ def append_token(url, token): return url +def get_protocol(use_secure): + if use_secure: + return 'https' + else: + return 'http' + + class PlexUpdate(BeetsPlugin): def __init__(self): super(PlexUpdate, self).__init__() @@ -65,7 +82,9 @@ class PlexUpdate(BeetsPlugin): u'host': u'localhost', u'port': 32400, u'token': u'', - u'library_name': u'Music'}) + u'library_name': u'Music', + u'use_secure': False, + u'ignore_cert_errors': False}) config['plex']['token'].redact = True self.register_listener('database_change', self.listen_for_db_change) @@ -85,7 +104,9 @@ class PlexUpdate(BeetsPlugin): config['plex']['host'].get(), config['plex']['port'].get(), config['plex']['token'].get(), - config['plex']['library_name'].get()) + config['plex']['library_name'].get(), + config['plex']['use_secure'].get(bool), + config['plex']['ignore_cert_errors'].get(bool)) self._log.info(u'... started.') except requests.exceptions.RequestException: diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index f9312280a..2c876c6fd 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -41,3 +41,7 @@ The available options under the ``plex:`` section are: Default: Empty. - **library_name**: The name of the Plex library to update. Default: ``Music`` +- **use_secure**: Use secure connections to the Plex server. + Default: ``False`` +- **ignore_cert_errors**: Ignore TLS certificate errors when using secure connections. + Default: ``False`` diff --git a/test/test_plexupdate.py b/test/test_plexupdate.py index f8b0bdcd9..862721837 100644 --- a/test/test_plexupdate.py +++ b/test/test_plexupdate.py @@ -93,7 +93,9 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['host'], self.config['plex']['port'], self.config['plex']['token'], - self.config['plex']['library_name'].get()), '2') + self.config['plex']['library_name'].get(), + self.config['plex']['use_secure'], + self.config['plex']['ignore_cert_errors']), '2') @responses.activate def test_get_named_music_section(self): @@ -104,7 +106,9 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['host'], self.config['plex']['port'], self.config['plex']['token'], - 'My Music Library'), '2') + 'My Music Library', + self.config['plex']['use_secure'], + self.config['plex']['ignore_cert_errors']), '2') @responses.activate def test_update_plex(self): @@ -117,7 +121,9 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['host'], self.config['plex']['port'], self.config['plex']['token'], - self.config['plex']['library_name'].get()).status_code, 200) + self.config['plex']['library_name'].get(), + self.config['plex']['use_secure'], + self.config['plex']['ignore_cert_errors']).status_code, 200) def suite(): From 6d41f31309503ff97cd2762b1e86d2abc3e4ee42 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Jun 2020 20:19:19 -0400 Subject: [PATCH 544/613] Rename use_secure to secure and add to changelog --- beetsplug/plexupdate.py | 18 +++++++++--------- docs/changelog.rst | 3 +++ docs/plugins/plexupdate.rst | 2 +- test/test_plexupdate.py | 6 +++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index 156d8835d..860757cfb 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -18,12 +18,12 @@ from beets import config from beets.plugins import BeetsPlugin -def get_music_section(host, port, token, library_name, use_secure, +def get_music_section(host, port, token, library_name, secure, ignore_cert_errors): """Getting the section key for the music library in Plex. """ api_endpoint = append_token('library/sections', token) - url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + url = urljoin('{0}://{1}:{2}'.format(get_protocol(secure), host, port), api_endpoint) # Sends request. @@ -36,7 +36,7 @@ def get_music_section(host, port, token, library_name, use_secure, return child.get('key') -def update_plex(host, port, token, library_name, use_secure, +def update_plex(host, port, token, library_name, secure, ignore_cert_errors): """Ignore certificate errors if configured to. """ @@ -47,10 +47,10 @@ def update_plex(host, port, token, library_name, use_secure, """ # Getting section key and build url. section_key = get_music_section(host, port, token, library_name, - use_secure, ignore_cert_errors) + secure, ignore_cert_errors) api_endpoint = 'library/sections/{0}/refresh'.format(section_key) api_endpoint = append_token(api_endpoint, token) - url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + url = urljoin('{0}://{1}:{2}'.format(get_protocol(secure), host, port), api_endpoint) # Sends request and returns requests object. @@ -66,8 +66,8 @@ def append_token(url, token): return url -def get_protocol(use_secure): - if use_secure: +def get_protocol(secure): + if secure: return 'https' else: return 'http' @@ -83,7 +83,7 @@ class PlexUpdate(BeetsPlugin): u'port': 32400, u'token': u'', u'library_name': u'Music', - u'use_secure': False, + u'secure': False, u'ignore_cert_errors': False}) config['plex']['token'].redact = True @@ -105,7 +105,7 @@ class PlexUpdate(BeetsPlugin): config['plex']['port'].get(), config['plex']['token'].get(), config['plex']['library_name'].get(), - config['plex']['use_secure'].get(bool), + config['plex']['secure'].get(bool), config['plex']['ignore_cert_errors'].get(bool)) self._log.info(u'... started.') diff --git a/docs/changelog.rst b/docs/changelog.rst index 691a42de5..5ecffb8e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -126,6 +126,9 @@ New features: :bug:`3567` * ``beet import`` now handles tar archives with bzip2 or gzip compression. :bug:`3606` +* :doc:`/plugins/plexupdate`: Add option to use secure connection to Plex + server, and to ignore certificate validation errors if necessary. + :bug:`2871` Fixes: diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index 2c876c6fd..92fc949d2 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -41,7 +41,7 @@ The available options under the ``plex:`` section are: Default: Empty. - **library_name**: The name of the Plex library to update. Default: ``Music`` -- **use_secure**: Use secure connections to the Plex server. +- **secure**: Use secure connections to the Plex server. Default: ``False`` - **ignore_cert_errors**: Ignore TLS certificate errors when using secure connections. Default: ``False`` diff --git a/test/test_plexupdate.py b/test/test_plexupdate.py index 862721837..ef89892a4 100644 --- a/test/test_plexupdate.py +++ b/test/test_plexupdate.py @@ -94,7 +94,7 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['port'], self.config['plex']['token'], self.config['plex']['library_name'].get(), - self.config['plex']['use_secure'], + self.config['plex']['secure'], self.config['plex']['ignore_cert_errors']), '2') @responses.activate @@ -107,7 +107,7 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['port'], self.config['plex']['token'], 'My Music Library', - self.config['plex']['use_secure'], + self.config['plex']['secure'], self.config['plex']['ignore_cert_errors']), '2') @responses.activate @@ -122,7 +122,7 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['port'], self.config['plex']['token'], self.config['plex']['library_name'].get(), - self.config['plex']['use_secure'], + self.config['plex']['secure'], self.config['plex']['ignore_cert_errors']).status_code, 200) From dab0c1f9abda5b17cc7488f89a6fe08be7bc56a0 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 9 Jun 2020 19:34:31 +0200 Subject: [PATCH 545/613] compatibility with breaking changes to the ast module new in 3.10, also backported to 3.8 and 3.9: https://github.com/python/cpython/pull/20649 In fact, our generation of some Literals has been invalid since Python 3.4, fix that too. --- beets/util/functemplate.py | 29 ++++++++++++++++++++--------- docs/changelog.rst | 1 + 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index af22b7908..266534a9b 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -73,15 +73,26 @@ def ex_literal(val): """An int, float, long, bool, string, or None literal with the given value. """ - if val is None: - return ast.Name('None', ast.Load()) - elif isinstance(val, six.integer_types): - return ast.Num(val) - elif isinstance(val, bool): - return ast.Name(bytes(val), ast.Load()) - elif isinstance(val, six.string_types): - return ast.Str(val) - raise TypeError(u'no literal for {0}'.format(type(val))) + if sys.version_info[:2] < (3, 4): + if val is None: + return ast.Name('None', ast.Load()) + elif isinstance(val, six.integer_types): + return ast.Num(val) + elif isinstance(val, bool): + return ast.Name(bytes(val), ast.Load()) + elif isinstance(val, six.string_types): + return ast.Str(val) + raise TypeError(u'no literal for {0}'.format(type(val))) + elif sys.version_info[:2] < (3, 6): + if val in [None, True, False]: + return ast.NameConstant(val) + elif isinstance(val, six.integer_types): + return ast.Num(val) + elif isinstance(val, six.string_types): + return ast.Str(val) + raise TypeError(u'no literal for {0}'.format(type(val))) + else: + return ast.Constant(val) def ex_varassign(name, expr): diff --git a/docs/changelog.rst b/docs/changelog.rst index 5ecffb8e3..453d22305 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -213,6 +213,7 @@ Fixes: * :doc:`/plugins/lyrics`: Fix crash when writing ReST files for a query without results or fetched lyrics :bug:`2805` +* Adapt to breaking changes in Python's ``ast`` module in 3.8 For plugin developers: From ae175e156b92f77b593532f371f5a1c8317153d1 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 16:10:33 -0700 Subject: [PATCH 546/613] fix genius lyrics artist matching when artist contains a hyphen --- beetsplug/lyrics.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index f53191d8a..49d2be672 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -419,16 +419,23 @@ class Genius(Backend): return None for hit in json["response"]["hits"]: - # Genius uses zero-width characters to denote lowercase - # artist names. - hit_artist = hit["result"]["primary_artist"]["name"]. \ - strip(u'\u200b').lower() + hit_artist = self._clean(hit["result"]["primary_artist"]["name"]) - if hit_artist == artist.lower(): + if hit_artist.lower() == artist.lower(): return self.lyrics_from_song_page(hit["result"]["url"]) self._log.debug(u'genius: no matching artist') + def _clean(self, artist): + """Cleans genius-isms to help artist matching""" + # Genius uses zero-width characters to denote lowercase artist names + artist = artist.strip(u'\u200b') + + # Genius uses minus-hypen compared to beets hyphen + artist = artist.replace(u'\u002D', u'\u2010') + + return artist + class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" From bb5740d63caaaf230ceaab64e0ec9b52a1d19e1d Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 16:41:20 -0700 Subject: [PATCH 547/613] better debug message for no genius artist match --- beetsplug/lyrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 49d2be672..090b99278 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -424,7 +424,8 @@ class Genius(Backend): if hit_artist.lower() == artist.lower(): return self.lyrics_from_song_page(hit["result"]["url"]) - self._log.debug(u'genius: no matching artist') + self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', + artist) def _clean(self, artist): """Cleans genius-isms to help artist matching""" From 7220c91d2abc3c2bb565765ab512f20c11946c12 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 18:56:20 -0700 Subject: [PATCH 548/613] generalize genius artist matching to use slug() --- beetsplug/lyrics.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 090b99278..8e45d25b4 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -419,24 +419,14 @@ class Genius(Backend): return None for hit in json["response"]["hits"]: - hit_artist = self._clean(hit["result"]["primary_artist"]["name"]) + hit_artist = hit["result"]["primary_artist"]["name"] - if hit_artist.lower() == artist.lower(): + if slug(hit_artist) == slug(artist): return self.lyrics_from_song_page(hit["result"]["url"]) self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', artist) - def _clean(self, artist): - """Cleans genius-isms to help artist matching""" - # Genius uses zero-width characters to denote lowercase artist names - artist = artist.strip(u'\u200b') - - # Genius uses minus-hypen compared to beets hyphen - artist = artist.replace(u'\u002D', u'\u2010') - - return artist - class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" From 17d9477106afd319b3a861b09366e0a32953de74 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 19:33:08 -0700 Subject: [PATCH 549/613] tests for applicable genius artist slugging --- test/test_lyrics.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index d31116284..22d5f2f0c 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function +import itertools import os import re import six @@ -485,18 +486,28 @@ class SlugTests(unittest.TestCase): # plain ascii passthrough text = u"test" self.assertEqual(lyrics.slug(text), 'test') + # german unicode and capitals text = u"Mørdag" self.assertEqual(lyrics.slug(text), 'mordag') + # more accents and quotes text = u"l'été c'est fait pour jouer" self.assertEqual(lyrics.slug(text), 'l-ete-c-est-fait-pour-jouer') + # accents, parens and spaces text = u"\xe7afe au lait (boisson)" self.assertEqual(lyrics.slug(text), 'cafe-au-lait-boisson') text = u"Multiple spaces -- and symbols! -- merged" self.assertEqual(lyrics.slug(text), 'multiple-spaces-and-symbols-merged') + text = u"\u200Bno-width-space" + self.assertEqual(lyrics.slug(text), 'no-width-space') + + # variations of dashes should get standardized + dashes = [u'\u200D', u'\u2010'] + for dash1, dash2 in itertools.combinations(dashes, 2): + self.assertEqual(lyrics.slug(dash1), lyrics.slug(dash2)) def suite(): From 2131f6f7d95a1c3bee17f878298ba217ea508c39 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 11:50:52 -0400 Subject: [PATCH 550/613] fetchart: Use Cover Art Archive thumbnails The Cover Art Archive API offers pre-resized thumbnails of cover art. If the `maxwidth` option of `fetchart` matches one of the supported Cover Art Archive thumbnail sizes fetch it directly instead of fetching the full size image then resizing it. --- beetsplug/fetchart.py | 20 +++++++++++++++++--- docs/changelog.rst | 3 +++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 70477a624..ad90b380c 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -317,12 +317,26 @@ class CoverArtArchive(RemoteArtSource): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ + + # Cover Art Archive API offers pre-resized thumbnails at several sizes. + # If the maxwidth config matches one of the already available sizes + # fetch it directly intead of fetching the full sized image and + # resizing it. + release_url = self.URL + group_url = self.GROUP_URL + valid_thumbnail_sizes = [250, 500, 1200] + if plugin.maxwidth in valid_thumbnail_sizes: + size_suffix = "-" + str(plugin.maxwidth) + release_url += size_suffix + group_url += size_suffix + if 'release' in self.match_by and album.mb_albumid: - yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT) + yield self._candidate( + url=release_url.format(mbid=album.mb_albumid), + match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( - url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), + url=group_url.format(mbid=album.mb_releasegroupid), match=Candidate.MATCH_FALLBACK) diff --git a/docs/changelog.rst b/docs/changelog.rst index 453d22305..5bc5c59cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -214,6 +214,9 @@ Fixes: results or fetched lyrics :bug:`2805` * Adapt to breaking changes in Python's ``ast`` module in 3.8 +* :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive + if the ``maxwidth`` option matches one of the sizes supported by the Cover + Art Archive API. For plugin developers: From 1e527ed4e52bad174f56cc2223bbb5ba4dc59ff1 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Jun 2020 12:49:32 -0400 Subject: [PATCH 551/613] Indentation fix --- beetsplug/fetchart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad90b380c..ebe343e82 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -333,7 +333,8 @@ class CoverArtArchive(RemoteArtSource): if 'release' in self.match_by and album.mb_albumid: yield self._candidate( url=release_url.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT) + match=Candidate.MATCH_EXACT, + ) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( url=group_url.format(mbid=album.mb_releasegroupid), From db181d35fb83339f1d24eeacba0a44f1edeb4740 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Jun 2020 12:50:13 -0400 Subject: [PATCH 552/613] Thanks & bug link for #3637 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bc5c59cc..112fdf23d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,6 +217,8 @@ Fixes: * :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive if the ``maxwidth`` option matches one of the sizes supported by the Cover Art Archive API. + Thanks to :user:`trolley`. + :bug:`3637` For plugin developers: From 71af5c04fc62a955712bc732b68c10c562d118c4 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 16:26:05 -0400 Subject: [PATCH 553/613] Revert "Thanks & bug link for #3637" This reverts commit db181d35fb83339f1d24eeacba0a44f1edeb4740. --- docs/changelog.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 112fdf23d..5bc5c59cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,8 +217,6 @@ Fixes: * :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive if the ``maxwidth`` option matches one of the sizes supported by the Cover Art Archive API. - Thanks to :user:`trolley`. - :bug:`3637` For plugin developers: From 3232a5b8b74134447f5dbaaefa7be5bd05147755 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 16:26:20 -0400 Subject: [PATCH 554/613] Revert "Indentation fix" This reverts commit 1e527ed4e52bad174f56cc2223bbb5ba4dc59ff1. --- beetsplug/fetchart.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ebe343e82..ad90b380c 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -333,8 +333,7 @@ class CoverArtArchive(RemoteArtSource): if 'release' in self.match_by and album.mb_albumid: yield self._candidate( url=release_url.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT, - ) + match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( url=group_url.format(mbid=album.mb_releasegroupid), From a4c647be461b56458a54fc1d7c74376a393cb60f Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 16:26:52 -0400 Subject: [PATCH 555/613] Revert "fetchart: Use Cover Art Archive thumbnails" This reverts commit 2131f6f7d95a1c3bee17f878298ba217ea508c39. --- beetsplug/fetchart.py | 20 +++----------------- docs/changelog.rst | 3 --- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad90b380c..70477a624 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -317,26 +317,12 @@ class CoverArtArchive(RemoteArtSource): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ - - # Cover Art Archive API offers pre-resized thumbnails at several sizes. - # If the maxwidth config matches one of the already available sizes - # fetch it directly intead of fetching the full sized image and - # resizing it. - release_url = self.URL - group_url = self.GROUP_URL - valid_thumbnail_sizes = [250, 500, 1200] - if plugin.maxwidth in valid_thumbnail_sizes: - size_suffix = "-" + str(plugin.maxwidth) - release_url += size_suffix - group_url += size_suffix - if 'release' in self.match_by and album.mb_albumid: - yield self._candidate( - url=release_url.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT) + yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), + match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( - url=group_url.format(mbid=album.mb_releasegroupid), + url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), match=Candidate.MATCH_FALLBACK) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bc5c59cc..453d22305 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -214,9 +214,6 @@ Fixes: results or fetched lyrics :bug:`2805` * Adapt to breaking changes in Python's ``ast`` module in 3.8 -* :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive - if the ``maxwidth`` option matches one of the sizes supported by the Cover - Art Archive API. For plugin developers: From 685d21a8fb907ff84559d8c98712ab196da579aa Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 27 Jun 2020 14:10:03 -0700 Subject: [PATCH 556/613] changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 453d22305..73cd89fb8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -129,6 +129,9 @@ New features: * :doc:`/plugins/plexupdate`: Add option to use secure connection to Plex server, and to ignore certificate validation errors if necessary. :bug:`2871` +* :doc:`/plugins/lyrics`: Improved searching Genius backend when artist + contained special characters. + :bug:`3634` Fixes: From 0f767ffa668bed98b5be9249f9b5e970b3245397 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sun, 28 Jun 2020 16:04:29 -0400 Subject: [PATCH 557/613] fetchart: Use Cover Art Archive thumbnails The Cover Art Archive API offers pre-resized thumbnails of cover art. If the `maxwidth` option of `fetchart` matches one of the supported Cover Art Archive thumbnail sizes, and a thumbnail of that size exists in the Cover Art Archive, fetch it directly instead of fetching the full size image then resizing it. --- beetsplug/fetchart.py | 27 +++++++++++++++++++++++---- docs/changelog.rst | 3 +++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 70477a624..86c5b958f 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -305,6 +305,7 @@ class RemoteArtSource(ArtSource): class CoverArtArchive(RemoteArtSource): NAME = u"Cover Art Archive" VALID_MATCHING_CRITERIA = ['release', 'releasegroup'] + VALID_THUMBNAIL_SIZES = [250, 500, 1200] if util.SNI_SUPPORTED: URL = 'https://coverartarchive.org/release/{mbid}/front' @@ -317,13 +318,31 @@ class CoverArtArchive(RemoteArtSource): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ + release_url = self.URL.format(mbid=album.mb_albumid) + release_group_url = self.GROUP_URL.format(mbid=album.mb_releasegroupid) + + # Cover Art Archive API offers pre-resized thumbnails at several sizes. + # If the maxwidth config matches one of the already available sizes + # fetch it directly intead of fetching the full sized image and + # resizing it. + size_suffix = None + if plugin.maxwidth in self.VALID_THUMBNAIL_SIZES: + size_suffix = "-" + str(plugin.maxwidth) + if 'release' in self.match_by and album.mb_albumid: - yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), + if size_suffix: + release_thumbnail_url = release_url + size_suffix + yield self._candidate(url=release_thumbnail_url, + match=Candidate.MATCH_EXACT) + yield self._candidate(url=release_url, match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: - yield self._candidate( - url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), - match=Candidate.MATCH_FALLBACK) + if size_suffix: + release_group_thumbnail_url = release_group_url + size_suffix + yield self._candidate(url=release_group_thumbnail_url, + match=Candidate.MATCH_FALLBACK) + yield self._candidate(url=release_group_url, + match=Candidate.MATCH_FALLBACK) class Amazon(RemoteArtSource): diff --git a/docs/changelog.rst b/docs/changelog.rst index 73cd89fb8..b6904bb71 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,6 +217,9 @@ Fixes: results or fetched lyrics :bug:`2805` * Adapt to breaking changes in Python's ``ast`` module in 3.8 +* :doc:`/plugins/fetchart`: Attempt to fetch pre-resized thumbnails from Cover + Art Archive if the ``maxwidth`` option matches one of the sizes supported by + the Cover Art Archive API. For plugin developers: From c9519c1ef53b9ddeb725dd3e12a97383998060d8 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:32:37 -0500 Subject: [PATCH 558/613] Create pull_request_template.md --- .github/pull_request_template.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ef2679b50 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,9 @@ +## Description +(...) + +Fixes #(insert issue number here) + +## TODO: +- [ ] docs - If you've added a new command flag, for example, find the appropraite page under docs/ where it needs to be listed +- [ ] Add a **changelog entry** to docs/changelog.rst near the top of the document. +- [ ] Add tests (suggested, but not a hard requirement) From 3282f1565ea80733523ca5d9b323ddabfe1ce1df Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:33:42 -0500 Subject: [PATCH 559/613] Update pull_request_template.md --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ef2679b50..1d6217540 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,5 +5,5 @@ Fixes #(insert issue number here) ## TODO: - [ ] docs - If you've added a new command flag, for example, find the appropraite page under docs/ where it needs to be listed -- [ ] Add a **changelog entry** to docs/changelog.rst near the top of the document. +- [ ] Add a changelog entry to docs/changelog.rst near the top of the document. - [ ] Add tests (suggested, but not a hard requirement) From b458509ba18ba5da6191e2207b4f32d0486c8645 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:35:08 -0500 Subject: [PATCH 560/613] spelling --- .github/pull_request_template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1d6217540..49982230b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,9 @@ ## Description (...) -Fixes #(insert issue number here) +Fixes #(insert issue number here if applicable) ## TODO: -- [ ] docs - If you've added a new command flag, for example, find the appropraite page under docs/ where it needs to be listed +- [ ] docs - If you've added a new command flag, for example, find the appropriate page under docs/ where it needs to be listed - [ ] Add a changelog entry to docs/changelog.rst near the top of the document. - [ ] Add tests (suggested, but not a hard requirement) From def44e6f88fcec0d4bb6586feef3acd96050e9d2 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:36:40 -0500 Subject: [PATCH 561/613] Update pull_request_template.md --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 49982230b..9abf2cdbd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -3,7 +3,7 @@ Fixes #(insert issue number here if applicable) -## TODO: +## TODO - [ ] docs - If you've added a new command flag, for example, find the appropriate page under docs/ where it needs to be listed - [ ] Add a changelog entry to docs/changelog.rst near the top of the document. - [ ] Add tests (suggested, but not a hard requirement) From aed81da2fada53a3921f0152a4c870ed5def0c6b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 28 Jun 2020 21:19:35 -0400 Subject: [PATCH 562/613] Thanks & bug link for #3637 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b6904bb71..1b80f5091 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -220,6 +220,8 @@ Fixes: * :doc:`/plugins/fetchart`: Attempt to fetch pre-resized thumbnails from Cover Art Archive if the ``maxwidth`` option matches one of the sizes supported by the Cover Art Archive API. + Thanks to :user:`trolley`. + :bug:`3637` For plugin developers: From 6411a58c80cfc755af744aae23637720bad632ef Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 29 Jun 2020 09:08:25 -0400 Subject: [PATCH 563/613] Slight formatting/wording improvements --- .github/pull_request_template.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9abf2cdbd..42cd8169b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,11 @@ ## Description + +Fixes #X. + (...) -Fixes #(insert issue number here if applicable) +## To Do -## TODO -- [ ] docs - If you've added a new command flag, for example, find the appropriate page under docs/ where it needs to be listed -- [ ] Add a changelog entry to docs/changelog.rst near the top of the document. -- [ ] Add tests (suggested, but not a hard requirement) +- [ ] Documentation. (If you've add a new command-line flag, for example, find the appropriate page under `docs/` to describe it.) +- [ ] Changelog. (Add an entry to `docs/changelog.rst` near the top of the document.) +- [ ] Tests. (Encouraged but not strictly required.) From 27199abac8f2d82afc93993c1901a4edc6bb34b0 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Fri, 3 Jul 2020 09:27:55 -0700 Subject: [PATCH 564/613] Get the date of the parentwork, which may be different from the date of the child work. Also add a changelog entry and docs. --- beetsplug/parentwork.py | 4 +++- docs/changelog.rst | 4 ++++ docs/plugins/parentwork.rst | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index dea7ba040..b3e464e60 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -131,6 +131,8 @@ class ParentWorkPlugin(BeetsPlugin): if artist['type'] == 'composer': parent_composer.append(artist['artist']['name']) parent_composer_sort.append(artist['artist']['sort-name']) + if 'end' in artist.keys(): + parentwork_info["parentwork_date"] = artist['end'] parentwork_info['parent_composer'] = u', '.join(parent_composer) parentwork_info['parent_composer_sort'] = u', '.join( @@ -207,4 +209,4 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date', - 'parentwork_workid_current']) + 'parentwork_workid_current', 'parentwork_date']) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1b80f5091..6ea5ae123 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -132,6 +132,10 @@ New features: * :doc:`/plugins/lyrics`: Improved searching Genius backend when artist contained special characters. :bug:`3634` +* :doc:`/plugins/parentwork`: Also get the composition date of the parent work, + instead of just the child work. + Thanks to :user:`aereaux`. + :bug:`3650` Fixes: diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index cc3f21c0c..fb15af9f1 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -26,7 +26,7 @@ At any later run of ``beet parentwork`` it will check if the tags not the case, it means the work has changed and all the tags need to be fetched again. -This plugin adds six tags: +This plugin adds seven tags: - **parentwork**: The title of the parent work. - **mb_parentworkid**: The MusicBrainz id of the parent work. @@ -38,6 +38,7 @@ This plugin adds six tags: - **parentwork_workid_current**: The MusicBrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, to keep track of recordings whose works have changed. +- **parentwork_date**: The composition date of the parent work. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From c05ba5c814301d01f56099ce62d7e489d6f03478 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sun, 5 Jul 2020 21:41:22 -0700 Subject: [PATCH 565/613] add integration test github action --- .github/workflows/integration_test.yaml | 32 +++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 33 insertions(+) create mode 100644 .github/workflows/integration_test.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml new file mode 100644 index 000000000..11085223f --- /dev/null +++ b/.github/workflows/integration_test.yaml @@ -0,0 +1,32 @@ +name: integration tests +on: + schedule: + - cron: '0 0 * * SUN' # run every Sunday at midnight +jobs: + test integration: + runs-on: ubuntu-latest + strategy: + python-version: 3.8 + env: + NOSE_SHOW_SKIPPED: 1 + PY_COLOR: 1 + INTEGRATION_TEST: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + pip install tox + - name: Test with tox + run: | + tox -e py38-test \ No newline at end of file diff --git a/tox.ini b/tox.ini index 047784f8b..a62981ec8 100644 --- a/tox.ini +++ b/tox.ini @@ -37,6 +37,7 @@ files = beets beetsplug beet test setup.py docs [testenv] passenv = NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. + INTEGRATION_TEST # set to 1 for integration tests deps = {test,cov}: {[_test]deps} py27: pathlib From 06553cc37cf1113648070e3039c3cec878188e1d Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sun, 5 Jul 2020 21:41:31 -0700 Subject: [PATCH 566/613] adjust test_lyrics.py to work with integration tests --- test/test_lyrics.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 22d5f2f0c..ed221fbb1 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -298,8 +298,8 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): DEFAULT_SOURCES = [ dict(DEFAULT_SONG, backend=lyrics.LyricsWiki), - dict(artist=u'Santana', title=u'Black magic woman', - backend=lyrics.MusiXmatch), + # dict(artist=u'Santana', title=u'Black magic woman', + # backend=lyrics.MusiXmatch), dict(DEFAULT_SONG, backend=lyrics.Genius), ] @@ -313,9 +313,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(DEFAULT_SONG, url=u'http://www.chartlyrics.com', path=u'/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx'), - dict(DEFAULT_SONG, - url=u'http://www.elyricsworld.com', - path=u'/lady_madonna_lyrics_beatles.html'), + # dict(DEFAULT_SONG, + # url=u'http://www.elyricsworld.com', + # path=u'/lady_madonna_lyrics_beatles.html'), dict(url=u'http://www.lacoccinelle.net', artist=u'Jacques Brel', title=u"Amsterdam", path=u'/paroles-officielles/275679.html'), @@ -332,11 +332,11 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(url=u'http://www.lyricsontop.com', artist=u'Amy Winehouse', title=u"Jazz'n'blues", path=u'/amy-winehouse-songs/jazz-n-blues-lyrics.html'), - dict(DEFAULT_SONG, - url='http://www.metrolyrics.com/', - path='lady-madonna-lyrics-beatles.html'), - dict(url='http://www.musica.com/', path='letras.asp?letra=2738', - artist=u'Santana', title=u'Black magic woman'), + # dict(DEFAULT_SONG, + # url='http://www.metrolyrics.com/', + # path='lady-madonna-lyrics-beatles.html'), + # dict(url='http://www.musica.com/', path='letras.asp?letra=2738', + # artist=u'Santana', title=u'Black magic woman'), dict(url=u'http://www.paroles.net/', artist=u'Lilly Wood & the prick', title=u"Hey it's ok", path=u'lilly-wood-the-prick/paroles-hey-it-s-ok'), @@ -352,9 +352,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() - @unittest.skipUnless(os.environ.get( - 'BEETS_TEST_LYRICS_SOURCES', '0') == '1', - 'lyrics sources testing not enabled') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_backend_sources_ok(self): """Test default backends with songs known to exist in respective databases. """ @@ -366,9 +366,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): errors.append(s['backend'].__name__) self.assertFalse(errors) - @unittest.skipUnless(os.environ.get( - 'BEETS_TEST_LYRICS_SOURCES', '0') == '1', - 'lyrics sources testing not enabled') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_google_sources_ok(self): """Test if lyrics present on websites registered in beets google custom search engine are correctly scraped. From 45abc9ed7a4b75d2a723bf33ff6285729b0ec8f6 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 6 Jul 2020 08:41:27 -0500 Subject: [PATCH 567/613] Clean up code and tests for Genius Lyrics plugin backend (#3641) * clean-up code & add tests for genius lyrics backend * add genius fetch tests * organize imports: standard lib -> pip -> local * check in sample genius lyrics page * fix mock import * force utf-8 encoding for opened files * use io.open to force utf-8 encoding w/ python2.7 --- .gitignore | 1 - beetsplug/lyrics.py | 81 +- .../geniuscom/Wutangclancreamlyrics.txt | 2227 +++++++++++++++++ test/test_lyrics.py | 129 +- 4 files changed, 2356 insertions(+), 82 deletions(-) create mode 100644 test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt diff --git a/.gitignore b/.gitignore index 64f08abe5..241202b88 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ # Project Specific patterns man -test/rsrc/lyrics/* # The rest is from https://www.gitignore.io/api/python diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 8e45d25b4..81040af3a 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -359,16 +359,54 @@ class Genius(Backend): 'User-Agent': USER_AGENT, } - def lyrics_from_song_page(self, page_url): - # Gotta go regular html scraping... come on Genius. - self._log.debug(u'fetching lyrics from: {0}', page_url) - try: - page = requests.get(page_url) - except requests.RequestException as exc: - self._log.debug(u'Genius page request for {0} failed: {1}', - page_url, exc) + def fetch(self, artist, title): + """Fetch lyrics from genius.com + + Because genius doesn't allow accesssing lyrics via the api, + we first query the api for a url matching our artist & title, + then attempt to scrape that url for the lyrics. + """ + json = self._search(artist, title) + if not json: + self._log.debug(u'Genius API request returned invalid JSON') return None - html = BeautifulSoup(page.text, "html.parser") + + # find a matching artist in the json + for hit in json["response"]["hits"]: + hit_artist = hit["result"]["primary_artist"]["name"] + + if slug(hit_artist) == slug(artist): + return self._scrape_lyrics_from_html( + self.fetch_url(hit["result"]["url"])) + + self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', + artist) + + def _search(self, artist, title): + """Searches the genius api for a given artist and title + + https://docs.genius.com/#search-h2 + + :returns: json response + """ + search_url = self.base_url + "/search" + data = {'q': title + " " + artist.lower()} + try: + response = requests.get( + search_url, data=data, headers=self.headers) + except requests.RequestException as exc: + self._log.debug(u'Genius API request failed: {0}', exc) + return None + + try: + return response.json() + except ValueError: + return None + + def _scrape_lyrics_from_html(self, html): + """Scrape lyrics from a given genius.com html""" + + html = BeautifulSoup(html, "html.parser") # Remove script tags that they put in the middle of the lyrics. [h.extract() for h in html('script')] @@ -402,31 +440,6 @@ class Genius(Backend): return lyrics_div.get_text() - def fetch(self, artist, title): - search_url = self.base_url + "/search" - data = {'q': title + " " + artist.lower()} - try: - response = requests.get(search_url, data=data, - headers=self.headers) - except requests.RequestException as exc: - self._log.debug(u'Genius API request failed: {0}', exc) - return None - - try: - json = response.json() - except ValueError: - self._log.debug(u'Genius API request returned invalid JSON') - return None - - for hit in json["response"]["hits"]: - hit_artist = hit["result"]["primary_artist"]["name"] - - if slug(hit_artist) == slug(artist): - return self.lyrics_from_song_page(hit["result"]["url"]) - - self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', - artist) - class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" diff --git a/test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt b/test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt new file mode 100644 index 000000000..08518f8ee --- /dev/null +++ b/test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt @@ -0,0 +1,2227 @@ + + + + + + + +Wu-Tang Clan – C.R.E.A.M. Lyrics | Genius Lyrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + {{:: 'cloud_flare_always_on_short_message' | i18n }} +
Check @genius for updates. We'll have things fixed soon. + +
+
+ + +
+ GENIUS +
+ + + +
+ + + + + + + + + + + + + +
+ + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+
+
+ Https%3a%2f%2fimages +
+
+ +
+
+ +

C.R.E.A.M.

+

+ Wu-Tang Clan +

+ +

+ + +

+

+ +
+ Produced by + + + RZA + +
+ + +

+

+ +
+ Album + Enter the Wu-Tang (36 Chambers) +
+ + +

+ +
+
+
+
+
+
+
+ + +
+
+
+ +
+
+ Beets is awesome! +
+
+ + +
+
+ +
+
+ + + + +
+ +

+ About “C.R.E.A.M.” +

+ + +
+
+

Arguably one of the most iconic songs in hip-hop, the underlying idea of “C.R.E.A.M.” is found in its title—cash rules everything. The timeless piano riffs and background vocals come from a chopped up sample of The Charmels‘ 1967 record, “As Long As I’ve Got You,” that make up the entire track.

+ +

Although it was released as an official single in 1994, “C.R.E.A.M.” was first recorded in 1991, around the same time as RZA’s assault case, and featured himself and Ghostface Killah. The track went through several revisions and was later re-recorded by Raekwon and Inspectah Deck in 1993—an early title of the song was “Lifestyles of the Mega-Rich.”

+ +

In 2017, RZA explained to Power 106 how the final version of the track came together:

+ +

Once we got to the studio, I decided that this track had to be on the Wu-Tang album. I reminded Rae and Deck of their verses—their verses were long. […] Method Man, the master of hooks at the time, came in with this hook right here: ‘cash rules everything around me, cream, get the money.’ Once he added that element, I knew it was going to be a smash.

+ +

Since its release, the song and chorus have been referenced countless times by several artists. It has also been featured in movies such as Eminem’s 8 Mile and the N.W.A biopic, Straight Outta Compton.

+
+
+
+ +
+
+
    + +
  • +
    +

    What has RZA said about the song?

    +
    + +
    +

    + +
    +
    + + +
    • + + +
  • +
    +

    What has Raekwon said about the song?

    +
    + +
    +
    +

    ‘C.R.E.A.M.’ did a lot for my career personally. It gave me an opportunity to revisit the times where that cream meant that much to us. So, yeah, when I think of this record it just automatically puts me back into ‘87/’88 where we were standing in front of the building. It’s cold outside. We didn’t care. We’re out there, all black on trying to make dollars. Just trying to make some money and trying to eat. Survive.

    + +

    This song, I remember writing to the beat a long time ago before we actually came out. That beat is old. That was probably like a ‘89 beat. RZA had it that long because he had a bunch of breaks. He had all kind of things and he was making beats back then, but we was just picking and that beat happened to always sit around and I would be like, ‘I want that beat, so don’t give that beat to nobody.’ And he kept his word and let me have it.

    + +

    Meth came up with the hook but our dude named Raider Ruckus, this was like Meth’s homeboy back then, like they was real close, he came up with the phrase ‘cash rules everything around me.’ So when he showed Meth what it was and was like, ‘Cash rules everything around me,’ Meth was like, ‘Word, you right!’ And turned it into a movie, and I came in later that day and heard it and co-signed it.

    +
    + +

    via Complex

    +
    + + +
    • + + +
  • +
    +

    What has U-God said about the song?

    +
    + +
    +

    “C.R.E.A.M.” is a true song. Everything Inspectah Deck and Raekwon said is 100 percent true. Not one line in that entire song is a lie, or even a slight exaggeration. Deck did sell base, and he did go to jail at the age of fifteen. Rae was sticking up white boys on ball courts, rocking the same damn ’Lo sweater. And of “course, Meth on the hook was like butter on the popcorn. Meth knew the hard times, too, being out there smoking woolies and pumping crack, etc. That raspy shit he was kicking just echoed in everyone’s head long after the song was done playing.

    The realism on “C.R.E.A.M.” is what resonates with so many people all over the world. People everywhere know that sentiment of being slaves to the dollar. Cash is king, and we are its lowly subjects. That’s pretty much the case in every nation around the world, the desperation to put your life and your freedom on the line to make a couple dollars. Whether you’re working, stripping, hustling, or slinging, whether you’re a business owner or homeless, cash rules everything around us.

    + +

    Source: Raw:My Journey into Wu-Tang

    +
    + + +
    • + + +
  • +
    +

    What songs were sampled on the beat for “C.R.E.A.M.?”

    +
    + +
    +

    The vocals and background sample that can be heard on the song’s intro were taken taken from The Charmels’ 1967 song “As Long as I’ve Got You”:

    + +

    + +
    + +

    The classic keys sample that can be heard throughout the beat was also taken from the previously mentioned song:

    + +

    + +
    +
    + + +
    • + + +
  • +
    +

    What has Method Man said about the song?

    +
    + +
    +

    Meth told Complex,

    + +
    +

    ‘C.R.E.A.M.’ was the one that really put us on the map if you wanna be technical. I wasn’t there when they recorded ‘C.R.E.A.M.’ I came in after the fact. RZA was like, ‘Put a hook on this song’ and I put a hook on it. That’s how it always went. I liked doing hooks.

    + +

    The hook for that was done by my man Raider Ruckus. We used to work at the Statue of Liberty and when we were coming home we used to come up with all these made-up words that were acronyms.

    + +

    We had words like ‘BIBWAM’ which meant, ‘Bitches Is Busted Without A Man’ and all this other crazy shit. Raider Ruckus was so ill with the way he put the words together. We would call money ‘cream’ so he took each letter and made a word out of it and killed it the way he did it.

    + +

    Something like that had never been done before as far as a hook or even a way of speaking. This is just showing and proving that we paid attention in class when we was kids. You can’t do shit like that unless you got a brain in your fucking head! You got to have some level of intelligence to do something like that.

    + +

    The best acronym for a word that I heard was ‘P.R.O.J.E.C.T.S.’ by Killah Priest. He said ‘People Relying On Just Enough Cash To Survive.’ And he’s the one that came up with ‘Basic Instructions Before Leaving Earth,’ the acronym for B.I.B.L.E. This ain’t no fluke shit man.

    + +

    There’s a reason you got millions upon millions of fucking kids running around with Wu-Tang tattoos. You don’t just put something on your body permanently unless it’s official. At that time, when you’re coming out brand new and representing where you come from, everybody from that area wants you to win because they win. That’s what it was like for us.

    + +

    We were the only dudes from Staten Island doing it so everybody from Staten Island wanted us to win. Not just dudes from Staten Island, but dudes from Brooklyn too because they had peoples in the group too. Then it was just grimy niggas who loved to see real shit, saying, ‘We riding with them Wu-Tang niggas. Fuck all that shiny suit shit!’ That ain’t no take on Puff, a lot of niggas was wearing suits and shit man, but that ain’t us.

    +
    +
    + + +
    • + + + +
+
+ + +
+
+

"C.R.E.A.M." Track Info

+ + +
+ Written By + + + Method Man, Inspectah Deck & Raekwon + +
+ + + + + +
+ Programmer + + + RZA + +
+ + + + +
+ Mixing Engineer + + + RZA + +
+ + + + +
+ Label + + + RCA Records, Loud Records & Bertlesmann Music Group + +
+ + + + +
+ Mastering Engineer + + + Chris Gehringer + +
+ + + + +
+ Engineer + + + Ethan Ryman + +
+ + + + + +
+ Recorded At + Firehouse Studios (Brooklyn, New York City) +
+ + + +
+ Release Date + January 31, 1994 +
+ + + + + + +
+ Samples + + +
+ As Long As I've Got You by The Charmels +
+ + +
+ + + + + +
+ Sampled In + + +
+ Inspector Deck by A-Wax (Ft. Bandana Tha Rag) +
+ +
+ Cash Rules by ​iyla (Ft. Method Man) +
+ +
+ C.R.E.A.M by Fler +
+ +
+ Brain Cell by CunninLynguists +
+ +
+ Watch Your Mouth by Ghostface Killah (Ft. GZA, Inspectah Deck, Masta Killa, Method Man, Raekwon, RZA, U-God & Wu-Tang Clan) +
+ +
+ Misère by IAM +
+ +
+ Petit Frère by IAM +
+ +
+ Lift-Off by Luizão 911 +
+ +
+ It's Me Bitches (Remix) by Swizz Beatz (Ft. Jadakiss, Lil Wayne & R. Kelly) +
+ +
+ Rules by Wu-Tang Clan +
+ +
+ Cold World by Lupe Fiasco +
+ +
+ The Rep Growz Bigga by Gang Starr +
+ +
+ IAM - Petit Frère (English Translation) by Genius English Translations +
+ +
+ I Grew Up on Wu-Tang by The Game (Ft. Lorine Chia) +
+ +
+ Terrorist's Cell by 7L & Esoteric +
+ +
+ Flaschenpost by Eins Zwo +
+ +
+ Facts of Life by L.I.F.E. Long +
+ +
+ Respect Mine by PMD +
+ +
+ Heartbeat by Jon Murdock +
+ +
+ Anything (Old Skool Radio Version) by SWV (Ft. Wu-Tang Clan) +
+ +
+ Ghost Dog by Kyle Kallgren (Ft. Rap Critic) +
+ +
+ Deathbed Doctrine by Jedi Mind Tricks +
+ +
+ Mathematics by Yasiin Bey +
+ +
+ Cash Rule$ by King Lil G +
+ +
+ What It's All About by Girl Talk +
+ +
+ Il Mondo Che Non C'è by Chief & Soci +
+ +
+ Parasiet by Mertens +
+ +
+ Freestyle #13 by Funkmaster Flex (Ft. The LOX & The Notorious B.I.G.) +
+ +
+ T.O.N.Y. C.R.E.A.M. by DJ Unknown & DJ Mekalek (Ft. Percee P & Rok One) +
+ +
+ The Polorican by Thirstin Howl III +
+ +
+ 8 Mile: Cheddar Bob Freestyle by Eminem +
+ +
+ Ποτέ δε θα Φύγω (Wu-Tang Special) by Rappas +
+ +
+ Welcome to Killmington by Triigga +
+ +
+ Queens Get The Money by Oscar O'Neill +
+ +
+ My Crew by Saigon +
+ +
+ Ends by Everlast +
+ +
+ K.R.E.A.M. by U-N-I (Rap Group) +
+ +
+ C'est mort by Big Budha Cheez (Ft. Cenza) +
+ +
+ Look Up / Save The Day by Michael Christmas +
+ +
+ I Hustle Everyday by Deez Nuts +
+ +
+ Leave It All Behind by Hex One +
+ +
+ Papoose Freestyles on Flex | Freestyle #076 by Papoose +
+ +
+ One Twenty by Eyedea & Abilities (Ft. Eyedea) +
+ +
+ Fuego by Cypress Hill (Ft. La Bruja) +
+ +
+ History of Rap 6 by Jimmy Fallon (Ft. Justin Timberlake) +
+ +
+ Life as a Shorty by Shyheim +
+ +
+ Brass Knuckles by Money Making Jam Boys (Ft. DJ Omega) +
+ +
+ Põe na Balança by Rap Plus Size +
+ +
+ Everywhere We Go by French Montana (Ft. Wale) +
+ +
+ Still Strugglin' by Raekwon (Ft. The Notorious B.I.G.) +
+ +
+ 4 Tha OG'z by MC Eiht (Ft. Freddie Foxxx) +
+ +
+ Yessaï (Sur la carte) by Caballero & JeanJass +
+ +
+ Respect Mine by Pete Rock (Ft. O.C.) +
+ +
+ Stop The Breaks by DJ Ron G (Ft. Killa Sin, KRS-One, The Notorious B.I.G., O.C. & Raekwon) +
+ +
+ Half Man Half Amazin by Pete Rock (Ft. Method Man) +
+ +
+ The Wake Up Call by Smiley the Ghetto Child +
+ +
+ Tras este Escenario by Nach (Ft. Juaninacka) +
+ +
+ Reality by Da Youngsta's +
+ +
+ Pascal by Hocus Pocus +
+ +
+ Good Mourning by Reflection Eternal +
+ +
+ Cream by Lil Wayne (Ft. Euro) +
+ +
+ Ride On by Snowgoons (Ft. Defari, Maylay Sparks & Sondro Castro) +
+ +
+ Dollar Bill by Count Bass D +
+ +
+ Elevate by Jigmastas +
+ +
+ Carta Viola by Duke Montana (Ft. Club Dogo) +
+ +
+ Birds by Killah Priest (Ft. Jay Notes & Redd Mudd) +
+ +
+ Foul Child / City Soldier (performed by Tee Productions by Warlocks +
+ +
+ Bomber by Oxon +
+ +
+ Comin from the Lower Level by CHOPS (Ft. Phil Da Agony, Ras Kass & Talib Kweli) +
+ +
+ Get the money by PANTyRAiD +
+ +
+ Grown Ass Man by Outerspace +
+ +
+ Escuchame by Braintax +
+ +
+ Front Steps Pt II (Tough Love) by Akrobatik +
+ +
+ Amor veneno by Dealema +
+ +
+ Release Me by Datsik +
+ +
+ Cash Rules by Snow Tha Product +
+ +
+ Homage to Premier by Focus… (Ft. Phonte, Royce Da 5'9'’ & Stat Quo) +
+ +
+ Survival Skills by KRS-ONE & Buckshot +
+ +
+ The Truth by Smif N Wessun +
+ +
+ Days Of My Youth by Rashad & Confidence +
+ +
+ Je suis en vie by Akhenaton (Ft. Tyler Woods) +
+ +
+ Cream Freestyle [Wu Tang Killa Deez] by Mista Deezy +
+ +
+ Tricher by ATK +
+ +
+ Lettre aux hirondelles by Akhenaton +
+ +
+ Intro by The X-Ecutioners (Ft. DJ Apollo, DJ Shortkut & DJ Vinroc) +
+ +
+ 40 Years of Hip-Hop by The Hood Internet +
+ +
+ Live From Shaolin by Jae Millz +
+ +
+ Spare Change by Ea$y Money +
+ +
+ Blutgrueneaugen by Die Beleidiger +
+ +
+ Intro by Inoki +
+ +
+ Supreme by Young Fla$h (Ft. Kwame Badu & Mike Holmes (Rapper)) +
+ +
+ Firing Squad by Heath McNease +
+ +
+ Kream by Iggy Azalea (Ft. Tyga) +
+ +
+ Dinheiro by Mãolee (Ft. BK' & Filipe Ret) +
+ +
+ C.R.E.A.M. by Heath McNease (Ft. Krum) +
+ +
+ Cream Remix by Djflipmobeatz +
+ +
+ If You Don't Know, Now You Know by Rap Genius +
+ +
+ Lucid by Internet Strangers +
+ +
+ You by Nicole Bus +
+ +
+ Zωή Mικρή by RNS (GR) +
+ +
+ Dinheiro (Remix) by Mãolee (Ft. Akira Presidente) +
+ +
+ Torcida by Big Fish (Ft. Chadia Rodriguez, Emis Killa, Fabri Fibra & Jake La Furia) +
+ +
+ Livin' Proof by Group Home +
+ +
+ Pound Cake Freestyle by JAG +
+ +
+ Barrionalistas by Los Chikos del Maíz +
+ +
+ No hay Empate by Cidtronyck (Ft. Freeman) +
+ +
+ D.R.E.A.M. by Miley Cyrus (Ft. Ghostface Killah) +
+ +
+ Wu by ​nafla +
+ +
+ INTRO by Nolan The Ninja +
+ +
+ Quake by BROCKBEATS +
+ +
+ Gospel by Rodrigo Zin (Ft. Bruno Nix & Nanda Oliveira N/O) +
+ +
+ DENTI D'ORO by Ketama126 +
+ +
+ (Unreleased) Intro by Grown Ass Child +
+ +
+ C.R.E.A.M. by Martin Matys +
+ +
+ Drei Tage wach by Plusmacher +
+ +
+ C.R.E.A.M. Team by Alemán (Ft. Muelas De Gallo) +
+ +
+ Ends [Woodstock 99] by Everlast +
+ +
+ Full Immersion by DJ Fede (Ft. Ghali) +
+ +
+ OASE by RAMO +
+ +
+ C.R.E.A.M. (Remix) by Serious P +
+ +
+ Verlaine by Rockin' Squat +
+ +
+ ​chain on / hold me by BROCKHAMPTON (Ft. JPEGMAFIA) +
+ +
+ Coppola by Cartel MCs (Ft. Ajota) +
+ + +
+ + + + + +
+ Interpolates + + +
+ Money (Dollar Bill, Y'all) by Jimmy Spicer +
+ + +
+ + + + + +
+ Interpolated By + + +
+ Denaro by CHS (Ft. BK') +
+ +
+ YOUTH* by GOTBARS +
+ +
+ MONEY* by GOTBARS +
+ +
+ Finesse (C.R.E.A.M.) by BMB Entertainment (Ft. K-$krilla & TrapKamo) +
+ +
+ Notorious Thugs by The Notorious B.I.G. (Ft. Bone Thugs-N-Harmony) +
+ +
+ Pound Cake / Paris Morton Music 2 by Drake (Ft. JAY-Z) +
+ +
+ We Takin' Over by DJ Khaled (Ft. Akon, Birdman, Fat Joe, Lil Wayne, Rick Ross & T.I.) +
+ +
+ Paper Trail$ by Joey Bada$$ +
+ +
+ Ballin' by Fat Joe (Ft. Teyana Taylor & Wiz Khalifa) +
+ +
+ New World Water by Yasiin Bey +
+ +
+ Game's Pain by The Game (Ft. Keyshia Cole) +
+ +
+ Hip Hop by Yasiin Bey +
+ +
+ Money Man by A$AP Mob (Ft. A$AP Bari, A$AP NAST & A$AP Rocky) +
+ +
+ Pound Cake (Remix) by Chamillionaire +
+ +
+ Childish Gambino Sway In The Morning Freestyle by Sway in the Morning (Ft. Childish Gambino) +
+ +
+ Pound Cake Freestyle by Meek Mill (Ft. Spade) +
+ +
+ Pound Cake (Freestyle) by King Los +
+ +
+ Pound Cake (Freestyle) by Migos +
+ +
+ Pound Cake by OTF Nunu +
+ +
+ Pound Cake (Remix) by Chip +
+ +
+ Pound Cake by Papoose +
+ +
+ Runaway Love (Remix) by Justin Bieber (Ft. Kanye West & Raekwon) +
+ +
+ Money Make the World Go Round by G-Unit +
+ +
+ Wild Thots by A Boogie wit da Hoodie (Ft. Fabolous) +
+ +
+ Ain't The Same by MIST +
+ +
+ Rise by Lecrae +
+ +
+ N64 by Denzel Curry +
+ +
+ I Am The Narrator by Plan B +
+ +
+ Return of the Hustle by Fabolous (Ft. Swizz Beatz) +
+ +
+ R.R.E.A.M. by Kitty +
+ +
+ D.R.E.A.M. by Transplants (Ft. Danny Diablo) +
+ +
+ Why Why Why by RZA (Ft. Swnkah) +
+ +
+ Life as a Shorty by Fashawn +
+ +
+ Sweetest Girl (Dollar Bill) by Wyclef Jean (Ft. Akon, Lil Wayne & Niia) +
+ +
+ The Ass by Pharoahe Monch (Ft. Apani B Fly) +
+ +
+ Pro Cakes 2 by Dirty Sanchez (Ft. Dyemond Lewis & Nyck Caution) +
+ +
+ Ice Cream by Audrey Rose (Ft. Fetty Wap & Remy Ma) +
+ +
+ The Sidewalks by J-Live +
+ +
+ Krølla 50-lapp y'all by Karpe +
+ +
+ The Last Huzzah (Remix) by Mr. Muthafuckin eXquire (Ft. Danny Brown, Das Racist, Despot & El-P) +
+ +
+ Respect Mine by Fat Joe (Ft. Raekwon) +
+ +
+ M.O.N.E.Y. by Lords of the Underground +
+ +
+ Sausage by Lil' Mama +
+ +
+ Sho Nuff by Juelz Santana +
+ +
+ Show You by Lil Debbie +
+ +
+ This Can't Be Real by Freeway (Ft. Marsha Ambrosius) +
+ +
+ Perfect by Princess Superstar +
+ +
+ Double Cross by Ghostface Killah (Ft. AZ) +
+ +
+ Living My Dream by King Lil G (Ft. Carolyn Rodriguez, Isuppose & Self-Provoked) +
+ +
+ I Know Why The Caged Bird Sings (Part 2) by Binary Star +
+ +
+ Loyalty by Ghostface Killah (Ft. Kool G Rap & Nems) +
+ +
+ Scars & Memories by MF Grimm +
+ +
+ D.R.E.A.M. by Pharoahe Monch (Ft. Talib Kweli) +
+ +
+ Ay Yah by DeJ Loaf +
+ +
+ I Hustle Everyday by Deez Nuts +
+ +
+ High Society by Kottonmouth Kings +
+ +
+ Live from the Underground by Big K.R.I.T. +
+ +
+ Scorn by HS87 (Ft. Audio Push, Hit-Boy, Kent M$ney & Kid Cudi) +
+ +
+ Get Money by Redlight (Ft. Raekwon) +
+ +
+ Young Nigga by Young Dolph (Ft. Fiend) +
+ +
+ Longevity by Inspectah Deck (Ft. U-God) +
+ +
+ Life Is Hell by Apathy (Ft. Hayze) +
+ +
+ DILIGENCE by Inga Copeland +
+ +
+ La Petite Leçon by Oxmo Puccino +
+ +
+ When I Flow (It’s Gospel) by The Cross Movement +
+ +
+ Democracy by Killah Priest (Ft. Canibus) +
+ +
+ How Big Is Your World by Median +
+ +
+ Cash Rules by James Wade (US) (Ft. Kassidy Henderson) +
+ +
+ Welcome To The Darkside by Fat Joe (Ft. French Montana) +
+ +
+ Notes by Verbal Kent +
+ +
+ Tash Rules by Tash (Alkaholiks) (Ft. L.V. (South Central Cartel)) +
+ +
+ Do It All Over Again by Danny Swain (Ft. Joi & ?uestlove) +
+ +
+ Money, Power & Women by G-Spot-Geez +
+ +
+ Camouflage by C-Rayz Walz +
+ +
+ Catalina by Raekwon (Ft. Lyfe Jennings) +
+ +
+ MILK by BROCKHAMPTON +
+ +
+ Login by GIRIBOY (Ft. Swings & Young B (KR)) +
+ +
+ C.R.E.A.M. by Heath McNease (Ft. Krum) +
+ +
+ I'll Make U Famous by Da Youngsta's +
+ +
+ ​infinity (888) by XXXTENTACION (Ft. Joey Bada$$) +
+ +
+ FFM Freestyle: Big Baby Tape by Fast Food Music (Ft. Big Baby Tape) +
+ +
+ Gottaknow by Royce Da 5'9'’ +
+ +
+ MOOO! by Doja Cat +
+ +
+ Kream Get the Money by Maxo Kream +
+ +
+ Ciao Ciao by Colle Der Fomento (Ft. Kaos One & Piotta) +
+ +
+ Still by Loyle Carner +
+ +
+ Energy by Trizzy Trae +
+ +
+ Sorry by Future +
+ +
+ Killa Killa Hill by Cappadonna (Ft. Raekwon) +
+ +
+ Kream by KEZ +
+ +
+ Make My by The Roots (Ft. Big K.R.I.T. & Dice Raw) +
+ +
+ The Cave: Episode 12 by Kenny Beats (Ft. Denzel Curry) +
+ +
+ Second Round Knockout by Shaquille O'Neal +
+ +
+ Represent'96 by TVETH +
+ +
+ 7.O.D. by 2nd Generation Wu +
+ +
+ R.E.M.E.D.Y. by Remedy (NY) +
+ +
+ 5 Bucks by KIL +
+ +
+ Back In Time by E SENS +
+ +
+ Hey, Listen! by Kaisson +
+ +
+ EVIL TWIN by Denzel Curry (Ft. ZillaKami) +
+ +
+ C.R.E.A.M by Hester Shawty & BabySlugga +
+ +
+ Right Now by Malz Monday +
+ +
+ CREAM by Loccie Shmula +
+ +
+ Cream by VI Seconds +
+ +
+ Onun Adı V by ViO (Ft. DJ Artz) +
+ +
+ Coppola by Cartel MCs (Ft. Ajota) +
+ +
+ Dans la ville by Darryl Zeuja +
+ + +
+ + + + + + + + + +
+ Cover By + + +
+ C.R.E.A.M. by El Michels Affair +
+ +
+ C.R.E.A.M. by Ramin Djawadi +
+ + +
+ + + + + + + + + +
+ Remixed By + + +
+ C.R.E.A.M. by Remy Ma (Ft. Curren$y) +
+ +
+ Sleep Rules Everything Around Me by Wugazi (Ft. Fugazi, Ian MacKaye, Inspectah Deck, Raekwon & Wu-Tang Clan) +
+ +
+ D.R.E.A.M. by Reservoir Dogs +
+ + +
+ + + + + + + + + + + +
+
+ + +
+
+
+ + Https%3a%2f%2fimages + +
+ + Enter the Wu-Tang (36 Chambers) + + + Wu-Tang Clan +
+
+ +
+ +
+
+ + 1.   + + + + Bring Da Ruckus + + + + +
+
+ +
+
+ + 2.   + + + + Shame On A Nigga + + + + +
+
+ +
+
+ + 3.   + + + + Clan in Da Front + + + + +
+
+ +
+
+ + 4.   + + + + Wu-Tang: 7th Chamber + + + + +
+
+ + + +
+
+ + 6.   + + + + Da Mystery of Chessboxin’ + + + + +
+
+ + + +
+
+ + 8.   + + + + C.R.E.A.M. + + + + +
+
+ +
+
+ + 9.   + + + + Method Man + + + + +
+
+ +
+
+ + 10.   + + + + Protect Ya Neck + + + + +
+
+ +
+
+ + 11.   + + + + Tearz + + + + +
+
+ +
+
+ + 12.   + + + + Wu-Tang: 7th Chamber - Pt. II + + + + +
+
+ +
+
+ + 13.   + + + + Conclusion + + + + +
+
+ +
+ +
+
+ +
+
+
+ + + + + + + + + +
+
+ + + + +
+ © 2020 Genius Media Group Inc. +
+
+ +
+ Verified Artists + + All Artists: +
    + +
  • + A +
    • + +
  • + B +
    • + +
  • + C +
    • + +
  • + D +
    • + +
  • + E +
    • + +
  • + F +
    • + +
  • + G +
    • + +
  • + H +
    • + +
  • + I +
    • + +
  • + J +
    • + +
  • + K +
    • + +
  • + L +
    • + +
  • + M +
    • + +
  • + N +
    • + +
  • + O +
    • + +
  • + P +
    • + +
  • + Q +
    • + +
  • + R +
    • + +
  • + S +
    • + +
  • + T +
    • + +
  • + U +
    • + +
  • + V +
    • + +
  • + W +
    • + +
  • + X +
    • + +
  • + Y +
    • + +
  • + Z +
    • + +
  • + # +
    • + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 22d5f2f0c..1355ec779 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -18,23 +18,21 @@ from __future__ import absolute_import, division, print_function import itertools +from io import open import os import re import six import sys import unittest -from mock import patch -from test import _common +import confuse +from mock import MagicMock, patch from beets import logging from beets.library import Item from beets.util import bytestring_path -import confuse - from beetsplug import lyrics - -from mock import MagicMock +from test import _common log = logging.getLogger('beets.test_lyrics') @@ -232,38 +230,11 @@ class MockFetchUrl(object): def __call__(self, url, filename=None): self.fetched = url fn = url_to_filename(url) - with open(fn, 'r') as f: + with open(fn, 'r', encoding="utf8") as f: content = f.read() return content -class GeniusMockGet(object): - - def __init__(self, pathval='fetched_path'): - self.pathval = pathval - self.fetched = None - - def __call__(self, url, headers=False): - from requests.models import Response - # for the first requests.get() return a path - if headers: - response = Response() - response.status_code = 200 - response._content = b'{"meta":{"status":200},\ - "response":{"song":{"path":"/lyrics/sample"}}}' - return response - # for the second requests.get() return the genius page - else: - from mock import PropertyMock - self.fetched = url - fn = url_to_filename(url) - with open(fn, 'r') as f: - content = f.read() - response = Response() - type(response).text = PropertyMock(return_value=content) - return response - - def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" if not text: @@ -445,8 +416,9 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): google.is_page_candidate(url, url_title, s['title'], u'Sunn O)))') -class LyricsGeniusBaseTest(unittest.TestCase): +# test Genius backend +class GeniusBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: @@ -457,28 +429,91 @@ class LyricsGeniusBaseTest(unittest.TestCase): self.skipTest("Python's built-in HTML parser is not good enough") -class LyricsGeniusScrapeTest(LyricsGeniusBaseTest): - - """Checks that Genius backend works as intended. - """ - import requests +class GeniusScrapeLyricsFromHtmlTest(GeniusBaseTest): + """tests Genius._scrape_lyrics_from_html()""" def setUp(self): """Set up configuration""" - LyricsGeniusBaseTest.setUp(self) + GeniusBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() - @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): - """Ensure that `lyrics_from_song_page` doesn't crash when the html - for a Genius page doesn't contain
+ """Ensure we don't crash when the scraping the html for a genius page + doesn't contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None - song_url = 'https://genius.com/sample' - self.assertEqual(genius.lyrics_from_song_page(song_url), - None) + url = 'https://genius.com/sample' + mock = MockFetchUrl() + self.assertEqual(genius._scrape_lyrics_from_html(mock(url)), None) + def test_good_lyrics(self): + """Ensure we are able to scrape a page with lyrics""" + url = 'https://genius.com/Wu-tang-clan-cream-lyrics' + mock = MockFetchUrl() + self.assertIsNotNone(genius._scrape_lyrics_from_html(mock(url))) + + # TODO: find an example of a lyrics page with multiple divs and test it + + +class GeniusFetchTest(GeniusBaseTest): + """tests Genius.fetch()""" + + def setUp(self): + """Set up configuration""" + GeniusBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @patch.object(lyrics.Genius, '_scrape_lyrics_from_html') + @patch.object(lyrics.Backend, 'fetch_url', return_value=True) + def test_json(self, mock_fetch_url, mock_scrape): + """Ensure we're finding artist matches""" + with patch.object( + lyrics.Genius, '_search', return_value={ + "response": { + "hits": [ + { + "result": { + "primary_artist": { + "name": u"\u200Bblackbear", + }, + "url": "blackbear_url" + } + }, + { + "result": { + "primary_artist": { + "name": u"El\u002Dp" + }, + "url": "El-p_url" + } + } + ] + } + } + ) as mock_json: + # genius uses zero-width-spaces (\u200B) for lowercase + # artists so we make sure we can match those + self.assertIsNotNone(genius.fetch('blackbear', 'Idfc')) + mock_fetch_url.assert_called_once_with("blackbear_url") + mock_scrape.assert_called_once_with(True) + + # genius uses the hypen minus (\u002D) as their dash + self.assertIsNotNone(genius.fetch('El-p', 'Idfc')) + mock_fetch_url.assert_called_with('El-p_url') + mock_scrape.assert_called_with(True) + + # test no matching artist + self.assertIsNone(genius.fetch('doesntexist', 'none')) + + # test invalid json + mock_json.return_value = None + self.assertIsNone(genius.fetch('blackbear', 'Idfc')) + + # TODO: add integration test hitting real api + + +# test utilties class SlugTests(unittest.TestCase): From 723e04ebb3e2c6d5efbbc3b2d216eb68bb588e93 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 6 Jul 2020 17:02:23 -0700 Subject: [PATCH 568/613] deprecate `tests_require` in favor of extras --- setup.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index c50e65bf5..0e2cb332a 100755 --- a/setup.py +++ b/setup.py @@ -109,24 +109,33 @@ setup( ['colorama'] if (sys.platform == 'win32') else [] ), - tests_require=[ - 'beautifulsoup4', - 'flask', - 'mock', - 'pylast', - 'rarfile', - 'responses', - 'pyxdg', - 'python-mpd2', - 'discogs-client', - 'requests_oauthlib' - ] + ( - # Tests for the thumbnails plugin need pathlib on Python 2 too. - ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] - ), - - # Plugin (optional) dependencies: extras_require={ + 'test': [ + 'beautifulsoup4', + 'coverage', + 'discogs-client', + 'flask', + 'mock', + 'pylast', + 'pytest', + 'python-mpd2', + 'pyxdg', + 'rarfile', + 'responses>=0.3.0', + 'requests_oauthlib', + ] + ( + # Tests for the thumbnails plugin need pathlib on Python 2 too. + ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] + ), + 'lint': [ + 'flake8', + 'flake8-blind-except', + 'flake8-coding', + 'flake8-future-import', + 'pep8-naming', + ], + + # Plugin (optional) dependencies: 'absubmit': ['requests'], 'fetchart': ['requests', 'Pillow'], 'embedart': ['Pillow'], From dabde74683bbaf7a16520c19a908d2fbb600fae0 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 6 Jul 2020 17:03:11 -0700 Subject: [PATCH 569/613] nose -> pytest and clean-up tox --- .github/workflows/ci.yaml | 5 +-- .github/workflows/integration_test.yaml | 3 +- setup.cfg | 4 -- tox.ini | 54 +++++++------------------ 4 files changed, 18 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5045a1b9c..56596f83b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,9 +40,8 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - python-version: [ 5, 6, 7, 8 ] + python-version: [ 4, 5, 6, 7, 8 ] env: - NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 steps: - uses: actions/checkout@v2 @@ -73,7 +72,7 @@ jobs: run: | tox -e py3${{ matrix.python-version }}-test tox -e py3${{ matrix.python-version }}-cov - tox -e py3${{ matrix.python-version }}-flake8 + tox -e py3${{ matrix.python-version }}-lint pip install codecov || true codecov || true - name: Test with tox diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 11085223f..9a4469e14 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -10,7 +10,6 @@ jobs: env: NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 - INTEGRATION_TEST: 1 steps: - uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -29,4 +28,4 @@ jobs: pip install tox - name: Test with tox run: | - tox -e py38-test \ No newline at end of file + tox -e int \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 0660b2721..89649ed11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,3 @@ -[nosetests] -verbosity=1 -logging-clear-handlers=1 - [flake8] min-version=2.7 accept-encodings=utf-8 diff --git a/tox.ini b/tox.ini index a62981ec8..13531e556 100644 --- a/tox.ini +++ b/tox.ini @@ -4,57 +4,33 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py27-flake8, docs - -# The exhaustive list of environments is: -# envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs +envlist = py27-test, py38-{cov,lint}, docs [_test] -deps = - beautifulsoup4 - flask - mock - nose - nose-show-skipped - pylast - rarfile - responses>=0.3.0 - pyxdg - python-mpd2 - coverage - discogs-client - requests_oauthlib +deps = .[test] # defined in setup.py -[_flake8] -deps = - flake8 - flake8-coding - flake8-future-import - flake8-blind-except - pep8-naming +[_lint] +deps = .[lint] # defined in setup.py files = beets beetsplug beet test setup.py docs [testenv] -passenv = - NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. - INTEGRATION_TEST # set to 1 for integration tests deps = {test,cov}: {[_test]deps} + lint: {[_lint]deps} py27: pathlib - py{27,34,35,36,37,38}-flake8: {[_flake8]deps} commands = - py27-cov: python -m nose --with-coverage {posargs} - py27-test: python -m nose {posargs} - py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} - py3{4,5,6,7,8}-test: python -bb -m nose {posargs} - py27-flake8: flake8 {posargs} {[_flake8]files} - py34-flake8: flake8 {posargs} {[_flake8]files} - py35-flake8: flake8 {posargs} {[_flake8]files} - py36-flake8: flake8 {posargs} {[_flake8]files} - py37-flake8: flake8 {posargs} {[_flake8]files} - py38-flake8: flake8 {posargs} {[_flake8]files} + py27-test: python -m pytest {posargs} + py3{4,5,6,7,8}-test: python -bb -m pytest {posargs} + cov: coverage run -m pytest {posargs} + lint: python -m flake8 {posargs} {[_lint]files} [testenv:docs] basepython = python2.7 deps = sphinx commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} + +[testenv:int] +basepython = python3.8 +deps = {[_test]deps} +setenv = INTEGRATION_TEST = 1 +commands = python -m pytest {posargs} \ No newline at end of file From e4284cff55397c209a9f894771107f412547f784 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 6 Jul 2020 18:51:36 -0700 Subject: [PATCH 570/613] reverting change to ci for python3.4 seems github actions doesn't support python3.4 --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56596f83b..86a9fd45a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - python-version: [ 4, 5, 6, 7, 8 ] + python-version: [ 5, 6, 7, 8 ] env: PY_COLOR: 1 steps: From 917c7def4e2d2caf20263dcde0fdde60700db39b Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 09:37:22 -0700 Subject: [PATCH 571/613] test commit for windows ci windows seems to be having issues with tox installing depencies via `.[test]` this is a test to see if removing the inline comment changes anything --- .github/workflows/ci.yaml | 1 - tox.ini | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 86a9fd45a..9d2d8880f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,6 @@ jobs: matrix: platform: [ ubuntu-latest ] env: - NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 steps: - uses: actions/checkout@v2 diff --git a/tox.ini b/tox.ini index 13531e556..cb673b872 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,10 @@ envlist = py27-test, py38-{cov,lint}, docs [_test] -deps = .[test] # defined in setup.py +deps = .[test] [_lint] -deps = .[lint] # defined in setup.py +deps = .[lint] files = beets beetsplug beet test setup.py docs [testenv] From 25c6843c284ad7402b22467a18f743bbf58191e4 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 15:05:21 -0700 Subject: [PATCH 572/613] classify integration tests in `test_parentwork.py` --- test/test_parentwork.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_parentwork.py b/test/test_parentwork.py index aa25c7f51..df6a98d79 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -17,6 +17,7 @@ from __future__ import division, absolute_import, print_function +import os import unittest from test.helper import TestHelper @@ -34,6 +35,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.unload_plugins() self.teardown_beets() + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_normal_case(self): item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-b8ebc18e8c53') @@ -45,6 +49,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.assertEqual(item['mb_parentworkid'], u'32c8943f-1b27-3a23-8660-4567f4847c94') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_force(self): self.config['parentwork']['force'] = True item = Item(path='/file', @@ -58,6 +65,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.assertEqual(item['mb_parentworkid'], u'32c8943f-1b27-3a23-8660-4567f4847c94') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_no_force(self): self.config['parentwork']['force'] = True item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-\ @@ -72,6 +82,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): # test different cases, still with Matthew Passion Ouverture or Mozart # requiem + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_direct_parent_work(self): mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', From d5e5be5c8e640d7d2f113c959a226bbad32e42ea Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 15:05:40 -0700 Subject: [PATCH 573/613] simplify `tox.ini` --- tox.ini | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index cb673b872..e3476db1c 100644 --- a/tox.ini +++ b/tox.ini @@ -17,10 +17,8 @@ files = beets beetsplug beet test setup.py docs deps = {test,cov}: {[_test]deps} lint: {[_lint]deps} - py27: pathlib commands = - py27-test: python -m pytest {posargs} - py3{4,5,6,7,8}-test: python -bb -m pytest {posargs} + test: python -bb -m pytest {posargs} cov: coverage run -m pytest {posargs} lint: python -m flake8 {posargs} {[_lint]files} @@ -30,7 +28,6 @@ deps = sphinx commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} [testenv:int] -basepython = python3.8 deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 -commands = python -m pytest {posargs} \ No newline at end of file +commands = python -bb -m pytest {posargs} \ No newline at end of file From f2e7b3fdd7c103d8910020bc5b72279a80998b62 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 15:28:38 -0700 Subject: [PATCH 574/613] remove references to nose --- .coveragerc | 1 - .github/workflows/integration_test.yaml | 1 - appveyor.yml | 3 --- test/_common.py | 2 +- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7d111e74c..bcc99c18f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,6 @@ omit = */pyshared/* */python?.?/* - */site-packages/nose/* */test/* exclude_lines = assert False diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 9a4469e14..7d930bb72 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -8,7 +8,6 @@ jobs: strategy: python-version: 3.8 env: - NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 steps: - uses: actions/checkout@v2 diff --git a/appveyor.yml b/appveyor.yml index 00a3eb189..5a0f32135 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,9 +6,6 @@ skip_commits: message: /\[appveyor skip\]/ environment: - # Undocumented feature of nose-show-skipped. - NOSE_SHOW_SKIPPED: 1 - matrix: - PYTHON: C:\Python27 TOX_ENV: py27-test diff --git a/test/_common.py b/test/_common.py index 5412ab650..8e3b1dd18 100644 --- a/test/_common.py +++ b/test/_common.py @@ -44,7 +44,7 @@ beetsplug.__path__ = [os.path.abspath( RSRC = util.bytestring_path(os.path.join(os.path.dirname(__file__), 'rsrc')) PLUGINPATH = os.path.join(os.path.dirname(__file__), 'rsrc', 'beetsplug') -# Propagate to root logger so nosetest can capture it +# Propagate to root logger so the test runner can capture it log = logging.getLogger('beets') log.propagate = True log.setLevel(logging.DEBUG) From 7427bfbc582851e19b64dd4030c26cbe18b37953 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 19:54:01 -0700 Subject: [PATCH 575/613] clean github workflows --- .github/workflows/ci.yaml | 137 +++++++++++------------- .github/workflows/integration_test.yaml | 16 +-- 2 files changed, 66 insertions(+), 87 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9d2d8880f..c0b24d9c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,93 +1,78 @@ name: ci on: [push, pull_request] jobs: - test-27: + test: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ubuntu-latest] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] env: - PY_COLOR: 1 + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 - - name: Set up Python 2.7 + + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 2.7 - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Linux') - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Windows') - with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install base dependencies - run: | - python -m pip install --upgrade pip - pip install tox sphinx - - name: Test with tox - run: tox -e py27-test - test-3x: - runs-on: ${{ matrix.platform }} - strategy: - matrix: - platform: [ ubuntu-latest ] - python-version: [ 5, 6, 7, 8 ] - env: - PY_COLOR: 1 - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: 3.${{ matrix.python-version }} - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Linux') - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Windows') - with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install base dependencies - run: | - python -m pip install --upgrade pip - pip install tox sphinx - - name: Test, coverage, and flake8 with tox - if: matrix.python-version == '8' - run: | - tox -e py3${{ matrix.python-version }}-test - tox -e py3${{ matrix.python-version }}-cov - tox -e py3${{ matrix.python-version }}-lint - pip install codecov || true - codecov || true - - name: Test with tox - if: matrix.python-version != '8' - run: tox -e py3${{ matrix.python-version }}-test - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python to build docs with Sphinx - uses: actions/setup-python@v2 - with: - python-version: 2.7 + python-version: ${{ matrix.python-version }} + - name: Install base dependencies run: | python -m pip install --upgrade pip python -m pip install tox sphinx + + - name: Test with tox + if: matrix.python-version != '3.8' + run: | + tox -e py-test + + - name: Test with tox and get coverage + if: matrix.python-version == '3.8' + run: | + tox -vv -e py-cov + + - name: Upload code coverage + if: matrix.python-version == '3.8' + run: | + pip install codecov || true + codecov || true + + test-docs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox sphinx + - name: Build and check docs using tox run: tox -e docs + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox sphinx + + - name: Lint with flake8 + run: tox -e py-lint \ No newline at end of file diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 7d930bb72..6afda7e23 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -5,26 +5,20 @@ on: jobs: test integration: runs-on: ubuntu-latest - strategy: - python-version: 3.8 - env: - PY_COLOR: 1 + steps: - uses: actions/checkout@v2 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: python-version: 3.8 - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + - name: Install base dependencies run: | python -m pip install --upgrade pip - pip install tox + python -m pip install tox sphinx + - name: Test with tox run: | tox -e int \ No newline at end of file From 7280ab2be0c72e1dd47cf769df201beaa16d1cb7 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 19:54:12 -0700 Subject: [PATCH 576/613] remove nosetest.xml from .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 241202b88..3a0340a3f 100644 --- a/.gitignore +++ b/.gitignore @@ -53,7 +53,6 @@ htmlcov/ .coverage .coverage.* .cache -nosetests.xml coverage.xml *,cover .hypothesis/ From f6855cb28418e0c96600e9f5d803a6519f50a39e Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 19:58:27 -0700 Subject: [PATCH 577/613] add py_colors to all github workflows --- .github/workflows/ci.yaml | 7 +++++++ .github/workflows/integration_test.yaml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0b24d9c5..f9cce8d23 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,6 +7,7 @@ jobs: matrix: platform: [ubuntu-latest] python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + env: PY_COLORS: 1 @@ -42,6 +43,9 @@ jobs: test-docs: runs-on: ubuntu-latest + env: + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 @@ -61,6 +65,9 @@ jobs: lint: runs-on: ubuntu-latest + env: + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 6afda7e23..fd67bb395 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -6,6 +6,9 @@ jobs: test integration: runs-on: ubuntu-latest + env: + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 From d230b3b87024e230e28a879f20ec1b9c2270307e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 11 Jul 2020 10:04:00 -0400 Subject: [PATCH 578/613] Roll back logging changes --- beets/library.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 3ad875991..e22d4edc0 100644 --- a/beets/library.py +++ b/beets/library.py @@ -419,13 +419,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - new_value = self._get('albumartist') - log.debug('No artist, using album artist {0}'.format(new_value)) - return new_value + return self._get('albumartist') elif key == 'albumartist' and not value: - new_value = self._get('artist') - log.debug('No albumartist, using artist {0}'.format(new_value)) - return new_value + return self._get('artist') else: return value From cc9f4ab25123690c25ccd4f01b111fb1ea614ff0 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 11 Jul 2020 15:59:34 -0500 Subject: [PATCH 579/613] Add a stale-bot (#3659) will apply the `stale` label to any issues and pull requests with >60/120 days inactivity respectively --- .github/stale.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/stale.yml diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 000000000..850446d81 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,26 @@ +# Configuration for probot-stale - https://github.com/probot/stale + +daysUntilClose: 7 +staleLabel: stale + +issues: + daysUntilStale: 60 + onlyLabels: + - needinfo + markComment: > + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + +pulls: + daysUntilStale: 120 + markComment: > + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. From 16fddbaef6a1407898b160efbdafac55591c772b Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 11 Jul 2020 17:27:37 -0500 Subject: [PATCH 580/613] add line break to stale-bot comments --- .github/stale.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 850446d81..2f3dbe041 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -8,8 +8,9 @@ issues: onlyLabels: - needinfo markComment: > - Is this still relevant? If so, what is blocking it? - Is there anything you can do to help move it forward? + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you @@ -18,8 +19,9 @@ issues: pulls: daysUntilStale: 120 markComment: > - Is this still relevant? If so, what is blocking it? - Is there anything you can do to help move it forward? + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you From 1db46dfeb6607c164afb247d8da82443677795c1 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 18:15:41 -0700 Subject: [PATCH 581/613] reformat flake8 errors --- setup.cfg | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/setup.cfg b/setup.cfg index 89649ed11..ff9af3fd0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,19 +1,32 @@ [flake8] -min-version=2.7 -accept-encodings=utf-8 -# Errors we ignore: -# - E121,E123,E126,E24,E704,W503,W504 flake8 default ignores, excluding E226 (have to be listed here to not be overridden) -# - E221: multiple spaces before operator (used to align visually) -# - E731: do not assign a lambda expression, use a def -# - F405 object may be undefined, or defined from star imports -# - C901: function/method complexity -# - E305: spacing after a declaration (might be nice to change eventually) -# `flake8-future-import` errors we ignore: -# - FI50: `__future__` import "division" present -# - FI51: `__future__` import "absolute_import" present -# - FI12: `__future__` import "with_statement" missing -# - FI53: `__future__` import "print_function" present -# - FI14: `__future__` import "unicode_literals" missing -# - FI15: `__future__` import "generator_stop" missing -# - E741: ambiguous variable name -ignore=E121,E123,E126,E24,E704,W503,W504,E305,C901,E221,E731,F405,FI50,FI51,FI12,FI53,FI14,FI15,E741 +min-version = 2.7 +accept-encodings = utf-8 +docstring-convention = numpy +# flake8 errors we ignore. See https://www.flake8rules.com/ +ignore = + # pycodestyle errors + E121, # continuation line under-indented for hanging indent + E123, # closing bracket does not match indentation of opening bracket's line + E126, # continuation line over-indented for hangin indent + E221, # multiple spaces before operator (for vertical alignment) + E226, # missing whitespace around arithmetic operator + E241, # multiple spaces after non-arithmetic operators (for vertical alignment) + E242, # tabs after ',' + E305, # expected 2 blank lines after end of function or class + E704, # multiple statements on one line + E731, # do not assign a lamba expression, use a def + E741, # do not use variables name 'I', 'O', or 'l' + # pycodestyle warnings + W503, # line break occurred before a binary operator + W504, # line break occurred after a binary operator + # pyflakes errors + F405, # name be undefined, or defined from star imports: module + # mccabe error + C901, # function is too complex + # future-import errors + FI12, # `__future__` import "with_statement" missing + FI14, # `__future__` import "unicode_literals" missing + FI15, # `__future__` import "generator_stop" missing + FI50, # `__future__` import "division" present + FI51, # `__future__` import "absolute_import" present + FI53, # `__future__` import "print_function" present \ No newline at end of file From abad0f9c08edb97e8332ea091636ac2f9fd2affb Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 18:22:43 -0700 Subject: [PATCH 582/613] un-ignore flake8 E226, E242, E704 none of these required any code changes --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index ff9af3fd0..71da3bcc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,11 +9,8 @@ ignore = E123, # closing bracket does not match indentation of opening bracket's line E126, # continuation line over-indented for hangin indent E221, # multiple spaces before operator (for vertical alignment) - E226, # missing whitespace around arithmetic operator E241, # multiple spaces after non-arithmetic operators (for vertical alignment) - E242, # tabs after ',' E305, # expected 2 blank lines after end of function or class - E704, # multiple statements on one line E731, # do not assign a lamba expression, use a def E741, # do not use variables name 'I', 'O', or 'l' # pycodestyle warnings From 124c882b4d850b08ba3c52ff05bef219878165ad Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 18:33:08 -0700 Subject: [PATCH 583/613] un-ignore E221 --- beetsplug/discogs.py | 2 +- beetsplug/playlist.py | 2 +- setup.cfg | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 038fa809a..3b2595585 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -38,7 +38,7 @@ from string import ascii_lowercase USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) -API_KEY = 'rAzVUQYRaoFjeBjyWuWZ' +API_KEY = 'rAzVUQYRaoFjeBjyWuWZ' API_SECRET = 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy' # Exceptions that discogs_client should really handle but does not. diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 302ddb56d..3f8ba2b0e 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -71,7 +71,7 @@ class PlaylistQuery(beets.dbcore.Query): if not self.paths: # Playlist is empty return '0', () - clause = 'path IN ({0})'.format(', '.join('?' for path in self.paths)) + clause = 'path IN ({0})'.format(', '.join('?' for path in self.paths)) return clause, (beets.library.BLOB_TYPE(p) for p in self.paths) def match(self, item): diff --git a/setup.cfg b/setup.cfg index 71da3bcc6..1b61dfae8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,6 @@ ignore = E121, # continuation line under-indented for hanging indent E123, # closing bracket does not match indentation of opening bracket's line E126, # continuation line over-indented for hangin indent - E221, # multiple spaces before operator (for vertical alignment) E241, # multiple spaces after non-arithmetic operators (for vertical alignment) E305, # expected 2 blank lines after end of function or class E731, # do not assign a lamba expression, use a def From 52afd717aabc2955d10a0389f4b77e6f98245881 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 19:10:04 -0700 Subject: [PATCH 584/613] remove docstring-convention and update comment --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1b61dfae8..3f5fb0c57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,7 @@ [flake8] min-version = 2.7 accept-encodings = utf-8 -docstring-convention = numpy -# flake8 errors we ignore. See https://www.flake8rules.com/ +# errors we ignore; see https://www.flake8rules.com/ for more info ignore = # pycodestyle errors E121, # continuation line under-indented for hanging indent From 44e4d558c97271244091e8541336454129b0622c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 12 Jul 2020 22:18:30 +0200 Subject: [PATCH 585/613] lyrics: fix bad_triggers in the google plugin regression from 117d16f --- beetsplug/lyrics.py | 2 +- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 81040af3a..16696d425 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -563,7 +563,7 @@ class Google(Backend): bad_triggers = ['lyrics', 'copyright', 'property', 'links'] if artist: - bad_triggers_occ += [artist] + bad_triggers += [artist] for item in bad_triggers: bad_triggers_occ += [item] * len(re.findall(r'\W%s\W' % item, diff --git a/docs/changelog.rst b/docs/changelog.rst index 6ea5ae123..0268aa2f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -136,6 +136,9 @@ New features: instead of just the child work. Thanks to :user:`aereaux`. :bug:`3650` +* :doc:`/plugins/lyrics`: Fix a bug in the heuristic for detecting valid + lyrics in the Google source of the lyrics plugin + :bug:`2969` Fixes: From d9272fcb88a617a36a1bc1f7c6405b6df8da8c4f Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 13 Jul 2020 14:20:20 -0500 Subject: [PATCH 586/613] fix integration test job name --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index fd67bb395..05623bf20 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -3,7 +3,7 @@ on: schedule: - cron: '0 0 * * SUN' # run every Sunday at midnight jobs: - test integration: + test_integration: runs-on: ubuntu-latest env: @@ -24,4 +24,4 @@ jobs: - name: Test with tox run: | - tox -e int \ No newline at end of file + tox -e int From 05516f9503bdb6c23015d844269ea809114f8db2 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 14 Jul 2020 06:59:48 -0400 Subject: [PATCH 587/613] Thanks & bug number for #2554 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 90e48de6b..167eba953 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -230,6 +230,8 @@ Fixes: Thanks to :user:`trolley`. :bug:`3637` * :doc:`/plugins/ipfs`: Fix Python 3 compatibility. + Thanks to :user:`musoke`. + :bug:`2554` For plugin developers: From dc7e4337680f7166167e5f08814fb2aee99ae155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Albornoz?= Date: Wed, 15 Jul 2020 22:00:50 -0400 Subject: [PATCH 588/613] Fix thumbnail plugin pathlib bug (#3360) PathlibURI.uri() would fail if given a path as bytes instead of as string. Co-authored-by: Jacob Pavlock --- beetsplug/thumbnails.py | 2 +- docs/changelog.rst | 3 +++ test/test_thumbnails.py | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index fe36fbd13..1b262eca5 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -224,7 +224,7 @@ class PathlibURI(URIGetter): name = "Python Pathlib" def uri(self, path): - return PurePosixPath(path).as_uri() + return PurePosixPath(util.py3_path(path)).as_uri() def copy_c_string(c_string): diff --git a/docs/changelog.rst b/docs/changelog.rst index 167eba953..0f41c38ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -139,6 +139,9 @@ New features: * :doc:`/plugins/lyrics`: Fix a bug in the heuristic for detecting valid lyrics in the Google source of the lyrics plugin :bug:`2969` +* :doc:`/plugins/thumbnails`: Fix a bug where pathlib expected a string instead + of bytes for a path. + :bug:`3360` Fixes: diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index dc03f06f7..dbbc032f7 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -284,6 +284,15 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): u'file:///music/%EC%8B%B8%EC%9D%B4') +class TestPathlibURI(): + """Test PathlibURI class""" + def test_uri(self): + test_uri = PathlibURI() + + # test it won't break if we pass it bytes for a path + test_uri.uri(b'/') + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 441cfebff19bdc08ffaab8590db94b32d1ae2c0a Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 19 Jul 2020 13:25:55 -0500 Subject: [PATCH 589/613] add .venv/ to gitignore .venv/ is recommended via https://docs.python.org/3/library/venv.html --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3a0340a3f..370776197 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,7 @@ target/ # virtualenv venv/ +.venv/ ENV/ # Spyder project settings From 62ce5d15dbde671c9c64864d696cecb3546bfa19 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 08:12:34 -0700 Subject: [PATCH 590/613] add contributing.md & update readme (#3643) * add contributing.md & update readme * add for developers section of docs to contributing.md * remove relative wiki links * readme spelling fix * clarify discourse/docs idea for user 'recipes' to get featured * remove coveralls from testing section * fix api links & clarify docstring standard * Contributing -> Contribute * remove manual pip module installation instructions * add link to testing wiki page on "how to contribute" * add databse access examples * added voting clarification * Rename contributing.md to CONTRIBUTING.md * remove out of date info Includes references to gitter, the old 'good first issue' label and also some updates to testing. * No need for backticks around beets We usually just typeset it as an ordinary noun, not as code. :) * Whitespace * End-to-end CONTRIBUTING pass Co-authored-by: Adrian Sampson --- CONTRIBUTING.md | 165 ++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 15 +++-- 2 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..4e2c8964d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,165 @@ +# Thank you! + +First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. + +These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team. By following them, you indicate your respect for the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. + +# Types of Contributions + +We love to get contributions from our community—you! There are many ways to contribute, whether you're a programmer or not. + +## Non-Programming + +* Promote beets! Help get the word out by telling your friends, writing a blog + post, or discussing it on a forum you frequent. +* Improve the [documentation][docs]. It's incredibly easy to contribute here: + just find a page you want to modify and hit the "Edit on GitHub" button in + the upper-right. You can automatically send us a pull request for your + changes. +* GUI design. For the time being, beets is a command-line-only affair. But + that's mostly because we don't have any great ideas for what a good GUI + should look like. If you have those great ideas, please get in touch. +* Benchmarks. We'd like to have a consistent way of measuring speed + improvements in beets' tagger and other functionality as well as a way of + comparing beets' performance to other tools. You can help by compiling a + library of freely-licensed music files (preferably with incorrect metadata) + for testing and measurement. +* Think you have a nice config or cool use-case for beets? We'd love to hear about it! Submit a post to our [our forums][forum] under the "Show and Tell" category for a chance to get featured in [the docs][advanced]. +* Consider helping out in [our forums][forum] by responding to support requests or driving some new discussions. + +[docs]: http://beets.readthedocs.org/ +[forum]: https://discourse.beets.io/ +[advanced]: https://beets.readthedocs.io/en/stable/guides/advanced.html + +## Programming + +* As a programmer (even if you're just a beginner!), you have a ton of opportunities to get your feet wet with beets. +* For developing plugins, or hacking away at beets, there's some good information in the ["For Developers" section of the docs][dev-docs]. + +[dev-docs]: https://beets.readthedocs.io/en/stable/dev/ + +### Getting the Source + +The easiest way to get started with the latest beets source is to use [pip][] to install an "editable" package. This can be done with one command: + + $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets + +Or, equivalently: + + $ git clone https://github.com/beetbox/beets.git + $ cd beets + $ pip install -e . + +If you already have a released version of beets installed, you may need to +remove it first by typing `pip uninstall beets`. The pip command above will put +the beets source in a `src/beets` directory and install the `beet` CLI script to +a standard location on your system. You may want to use the `--src` option to specify +the parent directory where the source will be checked out and the `--user` option +such that the package will be installed to your home directory (compare with the output of +`pip install --help`). + +[pip]: https://pip.pypa.io/ + +### Code Contribution Ideas + +* We maintain a set of [issues marked as "bite-sized"](https://github.com/beetbox/beets/labels/bitesize). These are issues that would serve as a good introduction to the codebase. Claim one and start exploring! +* Like testing? Our [test coverage](https://codecov.io/github/beetbox/beets) is somewhat low. You can help out by finding low-coverage modules or checking out other [testing-related issues](https://github.com/beetbox/beets/labels/testing). +* There are several ways to improve the tests in general (see [Testing](https://github.com/beetbox/beets/wiki/Testing)) and some places to think about performance optimization (see [Optimization](https://github.com/beetbox/beets/wiki/Optimization)). +* Not all of our code is up to our coding conventions. In particular, the [API documentation](https://beets.readthedocs.io/en/stable/dev/api.html) are currently quite sparse. You can help by adding to the docstrings in the code and to the documentation pages themselves. beets follows [PEP-257](https://www.python.org/dev/peps/pep-0257/) for docstrings and in some places, we also sometimes use [ReST autodoc syntax for Sphinx](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to, for example, refer to a class name. + +## Your First Contribution + +If this is your first time contributing to an open source project, welcome! If you are confused at all about how to contribute or what to contribute, take a look at [this great tutorial](http://makeapullrequest.com/), or stop by our [forums](https://discourse.beets.io/) if you have any questions. + +We maintain a list of issues we reserved for those new to open source labeled ["first timers only"](https://github.com/beetbox/beets/issues?q=is%3Aopen+is%3Aissue+label%3A%22first+timers+only%22). Since the goal of these issues is to get users comfortable with contributing to an open source project, please do not hesitate to ask any questions. + +## How to Submit Your Work + +Do you have a great bug fix, new feature, or documentation expansion you'd like to contribute? Follow these steps to create a GitHub pull request and your code will ship in no time. + +1. Fork the beets repository and clone it (see above) to create a workspace. +2. Make your changes. +3. Add tests. If you've fixed a bug, write a test to ensure that you've actually fixed it. If there's a new feature or plugin, please contribute tests that show that your code does what it says. +4. Add documentation. If you've added a new command flag, for example, find the appropriate page under `docs/` where it needs to be listed. +5. Add a changelog entry to `docs/changelog.rst` near the top of the document. +6. Run the tests and style checker. The easiest way to run the tests is to use [tox](https://tox.readthedocs.org/en/latest/). For more information on running tests, see our [Testing wiki page](https://github.com/beetbox/beets/wiki/Testing). +7. Push to your fork and open a pull request! We'll be in touch shortly. +8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn't automatically notify us when commits are added. + +Remember, code contributions have four parts: the code, the tests, the documentation, and the changelog entry. Thank you for contributing! + + +# The Code + +The documentation has an [API section](https://beets.readthedocs.io/en/stable/dev/api.html) that serves as an introduction to beets' design. + +## Coding Conventions + +There are a few coding conventions we use in beets: + +* Whenever you access the library database, do so through the provided Library + methods or via a Transaction object. Never call `lib.conn.*` directly. + For example, do this: + ``` + with g.lib.transaction() as tx: + rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' + .format(field, model._table, sort_field)) + ``` + To fetch Item objects from the database, use lib.items(...) and supply a query as an argument. Resist the urge to write raw SQL for your query. If you must use lower-level queries into the database, do this: + ``` + with lib.transaction() as tx: + rows = tx.query('SELECT …') + ``` + Transaction objects help control concurrent access to the database and assist in debugging conflicting accesses. +* Always use the [future imports][] `print_function`, `division`, and + `absolute_import`, but *not* `unicode_literals`. These help keep your code + modern and will help in the eventual move to Python 3. +* `str.format()` should be used instead of the `%` operator +* Never `print` informational messages; use the [logging][] module instead. In + particular, we have our own logging shim, so you'll see `from beets import + logging` in most files. + * Always log Unicode strings (e.g., `log.debug(u"hello world")`). + * The loggers use [str.format][]-style logging instead of ``%``-style, so + you can type `log.debug(u"{0}", obj)` to do your formatting. +* Exception handlers must use `except A as B:` instead of `except A, B:`. + +[future imports]: http://docs.python.org/library/__future__.html +[logging]: http://docs.python.org/library/logging.html +[str.format]: http://docs.python.org/library/stdtypes.html#str.format +[modformat]: http://docs.python.org/library/stdtypes.html#string-formatting-operations + +We follow [PEP 8](http://www.python.org/dev/peps/pep-0008/) for style. You can use `tox -e lint` to check your code for any style errors. + +## Handling Paths + +A great deal of convention deals with the handling of **paths**. Paths are +stored internally—in the database, for instance—as byte strings (i.e., `bytes` instead of `str` in Python 3). This is because POSIX operating systems' path names are only +reliably usable as byte strings—operating systems typically recommend but do not require that filenames use a given encoding, so violations of any reported encoding are inevitable. +On Windows, the strings are always encoded with UTF-8; on +Unix, the encoding is controlled by the filesystem. Here are some guidelines to +follow: + +* If you have a Unicode path or you're not sure whether something is Unicode or + not, pass it through `bytestring_path` function in the `beets.util` module to + convert it to bytes. +* Pass every path name trough the `syspath` function (also in `beets.util`) + before sending it to any *operating system* file operation (`open`, for + example). This is necessary to use long filenames (which, maddeningly, must + be Unicode) on Windows. This allows us to consistently store bytes in the + database but use the native encoding rule on both POSIX and Windows. +* Similarly, the `displayable_path` utility function converts bytestring paths + to a Unicode string for displaying to the user. Every time you want to print + out a string to the terminal or log it with the `logging` module, feed it + through this function. + +## Editor Settings + +Personally, I work on beets with [vim](http://www.vim.org/). Here are some +`.vimrc` lines that might help with PEP 8-compliant Python coding: + + filetype indent on + autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent + +Consider installing [this alternative Python indentation +plugin](https://github.com/mitsuhiko/vim-python-combined). I also like +[neomake](https://github.com/neomake/neomake) with its flake8 checker. diff --git a/README.rst b/README.rst index 418ff0f56..7f509efdc 100644 --- a/README.rst +++ b/README.rst @@ -90,11 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Check out the `Hacking`_ page on the wiki for tips on how to help out. -You might also be interested in the `For Developers`_ section in the docs. +Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `Contributing.md`_. -.. _Hacking: https://github.com/beetbox/beets/wiki/Hacking -.. _For Developers: https://beets.readthedocs.io/en/stable/dev/ +.. _Contributing.md: https://github.com/beetbox/beets/blob/contributing.md/contributing.md Read More --------- @@ -105,6 +103,15 @@ news and updates. .. _its Web site: https://beets.io/ .. _@b33ts: https://twitter.com/b33ts/ +Contact +------- +* Encountered a bug you'd like to report or have an idea for a new feature? Check out our `issue tracker`_! If your issue or feature hasn't already been reported, please `open a new ticket`_ and we'll be in touch with you shortly. If you'd like to vote on a feature/bug, simply give a :+1: on issues you'd like to see prioritized over others. +* Need help/support, would like to start a discussion, or would just like to introduce yourself to the team? Check out our `forums`_! + +.. _issue tracker: https://github.com/beetbox/beets/issues +.. _open a new ticket: https://github.com/beetbox/beets/issues/new/choose +.. _forums: https://discourse.beets.io/ + Authors ------- From 426cb0800305f4b88854c17aeed9d742e3b57551 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:02:38 -0500 Subject: [PATCH 591/613] fix CONTRIBUTING.md link and spelling --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7f509efdc..4cf66a29a 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `Contributing.md`_. +Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.md`_. -.. _Contributing.md: https://github.com/beetbox/beets/blob/contributing.md/contributing.md +.. _CONTRIBUTING.md: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.md Read More --------- From 5544c7a8813fe68ce89e2e65aecfc8c9b6f70f73 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:04:44 -0500 Subject: [PATCH 592/613] Fix wording --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e2c8964d..4ed28746e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. -These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team. By following them, you indicate your respect for the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. +These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team as it indicates you respect the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. # Types of Contributions From c1b9e99722291cd267ffc5ca985302d80c99d445 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:07:32 -0500 Subject: [PATCH 593/613] maintainers *will* reciprocate respect :) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ed28746e..e16e10cb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. -These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team as it indicates you respect the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. +These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team as it indicates you respect the maintainers' time; in return, the maintainers will reciprocate by helping to address your issue, review changes, and finalize pull requests. # Types of Contributions From b309ed95c2fdf0613b522db0fa2048a75e3ee3ab Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:17:11 -0500 Subject: [PATCH 594/613] Remove duplicate forum mention Forums are already mentioned in the 'Contact' section --- README.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4cf66a29a..d57afa933 100644 --- a/README.rst +++ b/README.rst @@ -115,8 +115,6 @@ Contact Authors ------- -Beets is by `Adrian Sampson`_ with a supporting cast of thousands. For help, -please visit our `forum`_. +Beets is by `Adrian Sampson`_ with a supporting cast of thousands. -.. _forum: https://discourse.beets.io .. _Adrian Sampson: https://www.cs.cornell.edu/~asampson/ From faa55534680f816fa81e71ea886b54dfee5bfa0f Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 18:55:27 -0700 Subject: [PATCH 595/613] convert CONTRIBUTING.md to rst --- CONTRIBUTING.md | 165 ------------------------------ CONTRIBUTING.rst | 255 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+), 165 deletions(-) delete mode 100644 CONTRIBUTING.md create mode 100644 CONTRIBUTING.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 4e2c8964d..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,165 +0,0 @@ -# Thank you! - -First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. - -These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team. By following them, you indicate your respect for the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. - -# Types of Contributions - -We love to get contributions from our community—you! There are many ways to contribute, whether you're a programmer or not. - -## Non-Programming - -* Promote beets! Help get the word out by telling your friends, writing a blog - post, or discussing it on a forum you frequent. -* Improve the [documentation][docs]. It's incredibly easy to contribute here: - just find a page you want to modify and hit the "Edit on GitHub" button in - the upper-right. You can automatically send us a pull request for your - changes. -* GUI design. For the time being, beets is a command-line-only affair. But - that's mostly because we don't have any great ideas for what a good GUI - should look like. If you have those great ideas, please get in touch. -* Benchmarks. We'd like to have a consistent way of measuring speed - improvements in beets' tagger and other functionality as well as a way of - comparing beets' performance to other tools. You can help by compiling a - library of freely-licensed music files (preferably with incorrect metadata) - for testing and measurement. -* Think you have a nice config or cool use-case for beets? We'd love to hear about it! Submit a post to our [our forums][forum] under the "Show and Tell" category for a chance to get featured in [the docs][advanced]. -* Consider helping out in [our forums][forum] by responding to support requests or driving some new discussions. - -[docs]: http://beets.readthedocs.org/ -[forum]: https://discourse.beets.io/ -[advanced]: https://beets.readthedocs.io/en/stable/guides/advanced.html - -## Programming - -* As a programmer (even if you're just a beginner!), you have a ton of opportunities to get your feet wet with beets. -* For developing plugins, or hacking away at beets, there's some good information in the ["For Developers" section of the docs][dev-docs]. - -[dev-docs]: https://beets.readthedocs.io/en/stable/dev/ - -### Getting the Source - -The easiest way to get started with the latest beets source is to use [pip][] to install an "editable" package. This can be done with one command: - - $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets - -Or, equivalently: - - $ git clone https://github.com/beetbox/beets.git - $ cd beets - $ pip install -e . - -If you already have a released version of beets installed, you may need to -remove it first by typing `pip uninstall beets`. The pip command above will put -the beets source in a `src/beets` directory and install the `beet` CLI script to -a standard location on your system. You may want to use the `--src` option to specify -the parent directory where the source will be checked out and the `--user` option -such that the package will be installed to your home directory (compare with the output of -`pip install --help`). - -[pip]: https://pip.pypa.io/ - -### Code Contribution Ideas - -* We maintain a set of [issues marked as "bite-sized"](https://github.com/beetbox/beets/labels/bitesize). These are issues that would serve as a good introduction to the codebase. Claim one and start exploring! -* Like testing? Our [test coverage](https://codecov.io/github/beetbox/beets) is somewhat low. You can help out by finding low-coverage modules or checking out other [testing-related issues](https://github.com/beetbox/beets/labels/testing). -* There are several ways to improve the tests in general (see [Testing](https://github.com/beetbox/beets/wiki/Testing)) and some places to think about performance optimization (see [Optimization](https://github.com/beetbox/beets/wiki/Optimization)). -* Not all of our code is up to our coding conventions. In particular, the [API documentation](https://beets.readthedocs.io/en/stable/dev/api.html) are currently quite sparse. You can help by adding to the docstrings in the code and to the documentation pages themselves. beets follows [PEP-257](https://www.python.org/dev/peps/pep-0257/) for docstrings and in some places, we also sometimes use [ReST autodoc syntax for Sphinx](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to, for example, refer to a class name. - -## Your First Contribution - -If this is your first time contributing to an open source project, welcome! If you are confused at all about how to contribute or what to contribute, take a look at [this great tutorial](http://makeapullrequest.com/), or stop by our [forums](https://discourse.beets.io/) if you have any questions. - -We maintain a list of issues we reserved for those new to open source labeled ["first timers only"](https://github.com/beetbox/beets/issues?q=is%3Aopen+is%3Aissue+label%3A%22first+timers+only%22). Since the goal of these issues is to get users comfortable with contributing to an open source project, please do not hesitate to ask any questions. - -## How to Submit Your Work - -Do you have a great bug fix, new feature, or documentation expansion you'd like to contribute? Follow these steps to create a GitHub pull request and your code will ship in no time. - -1. Fork the beets repository and clone it (see above) to create a workspace. -2. Make your changes. -3. Add tests. If you've fixed a bug, write a test to ensure that you've actually fixed it. If there's a new feature or plugin, please contribute tests that show that your code does what it says. -4. Add documentation. If you've added a new command flag, for example, find the appropriate page under `docs/` where it needs to be listed. -5. Add a changelog entry to `docs/changelog.rst` near the top of the document. -6. Run the tests and style checker. The easiest way to run the tests is to use [tox](https://tox.readthedocs.org/en/latest/). For more information on running tests, see our [Testing wiki page](https://github.com/beetbox/beets/wiki/Testing). -7. Push to your fork and open a pull request! We'll be in touch shortly. -8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn't automatically notify us when commits are added. - -Remember, code contributions have four parts: the code, the tests, the documentation, and the changelog entry. Thank you for contributing! - - -# The Code - -The documentation has an [API section](https://beets.readthedocs.io/en/stable/dev/api.html) that serves as an introduction to beets' design. - -## Coding Conventions - -There are a few coding conventions we use in beets: - -* Whenever you access the library database, do so through the provided Library - methods or via a Transaction object. Never call `lib.conn.*` directly. - For example, do this: - ``` - with g.lib.transaction() as tx: - rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' - .format(field, model._table, sort_field)) - ``` - To fetch Item objects from the database, use lib.items(...) and supply a query as an argument. Resist the urge to write raw SQL for your query. If you must use lower-level queries into the database, do this: - ``` - with lib.transaction() as tx: - rows = tx.query('SELECT …') - ``` - Transaction objects help control concurrent access to the database and assist in debugging conflicting accesses. -* Always use the [future imports][] `print_function`, `division`, and - `absolute_import`, but *not* `unicode_literals`. These help keep your code - modern and will help in the eventual move to Python 3. -* `str.format()` should be used instead of the `%` operator -* Never `print` informational messages; use the [logging][] module instead. In - particular, we have our own logging shim, so you'll see `from beets import - logging` in most files. - * Always log Unicode strings (e.g., `log.debug(u"hello world")`). - * The loggers use [str.format][]-style logging instead of ``%``-style, so - you can type `log.debug(u"{0}", obj)` to do your formatting. -* Exception handlers must use `except A as B:` instead of `except A, B:`. - -[future imports]: http://docs.python.org/library/__future__.html -[logging]: http://docs.python.org/library/logging.html -[str.format]: http://docs.python.org/library/stdtypes.html#str.format -[modformat]: http://docs.python.org/library/stdtypes.html#string-formatting-operations - -We follow [PEP 8](http://www.python.org/dev/peps/pep-0008/) for style. You can use `tox -e lint` to check your code for any style errors. - -## Handling Paths - -A great deal of convention deals with the handling of **paths**. Paths are -stored internally—in the database, for instance—as byte strings (i.e., `bytes` instead of `str` in Python 3). This is because POSIX operating systems' path names are only -reliably usable as byte strings—operating systems typically recommend but do not require that filenames use a given encoding, so violations of any reported encoding are inevitable. -On Windows, the strings are always encoded with UTF-8; on -Unix, the encoding is controlled by the filesystem. Here are some guidelines to -follow: - -* If you have a Unicode path or you're not sure whether something is Unicode or - not, pass it through `bytestring_path` function in the `beets.util` module to - convert it to bytes. -* Pass every path name trough the `syspath` function (also in `beets.util`) - before sending it to any *operating system* file operation (`open`, for - example). This is necessary to use long filenames (which, maddeningly, must - be Unicode) on Windows. This allows us to consistently store bytes in the - database but use the native encoding rule on both POSIX and Windows. -* Similarly, the `displayable_path` utility function converts bytestring paths - to a Unicode string for displaying to the user. Every time you want to print - out a string to the terminal or log it with the `logging` module, feed it - through this function. - -## Editor Settings - -Personally, I work on beets with [vim](http://www.vim.org/). Here are some -`.vimrc` lines that might help with PEP 8-compliant Python coding: - - filetype indent on - autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent - -Consider installing [this alternative Python indentation -plugin](https://github.com/mitsuhiko/vim-python-combined). I also like -[neomake](https://github.com/neomake/neomake) with its flake8 checker. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 000000000..2596fd135 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,255 @@ +Thank you! +========== + +First off, thank you for considering contributing to beets! It’s people +like you that make beets continue to succeed. + +These guidelines describe how you can help most effectively. By +following these guidelines, you can make life easier for the development +team as it indicates you respect the maintainers’ time; in return, the +maintainers will reciprocate by helping to address your issue, review +changes, and finalize pull requests. + +Types of Contributions +====================== + +We love to get contributions from our community—you! There are many ways +to contribute, whether you’re a programmer or not. + +Non-Programming +--------------- + +- Promote beets! Help get the word out by telling your friends, writing + a blog post, or discussing it on a forum you frequent. +- Improve the `documentation `__. It’s + incredibly easy to contribute here: just find a page you want to + modify and hit the “Edit on GitHub” button in the upper-right. You + can automatically send us a pull request for your changes. +- GUI design. For the time being, beets is a command-line-only affair. + But that’s mostly because we don’t have any great ideas for what a + good GUI should look like. If you have those great ideas, please get + in touch. +- Benchmarks. We’d like to have a consistent way of measuring speed + improvements in beets’ tagger and other functionality as well as a + way of comparing beets’ performance to other tools. You can help by + compiling a library of freely-licensed music files (preferably with + incorrect metadata) for testing and measurement. +- Think you have a nice config or cool use-case for beets? We’d love to + hear about it! Submit a post to our `our + forums `__ under the “Show and Tell” + category for a chance to get featured in `the + docs `__. +- Consider helping out in `our forums `__ + by responding to support requests or driving some new discussions. + +Programming +----------- + +- As a programmer (even if you’re just a beginner!), you have a ton of + opportunities to get your feet wet with beets. +- For developing plugins, or hacking away at beets, there’s some good + information in the `“For Developers” section of the + docs `__. + +Getting the Source +~~~~~~~~~~~~~~~~~~ + +The easiest way to get started with the latest beets source is to use +`pip `__ to install an “editable” package. This +can be done with one command: + +:: + + $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets + +Or, equivalently: + +:: + + $ git clone https://github.com/beetbox/beets.git + $ cd beets + $ pip install -e . + +If you already have a released version of beets installed, you may need +to remove it first by typing ``pip uninstall beets``. The pip command +above will put the beets source in a ``src/beets`` directory and install +the ``beet`` CLI script to a standard location on your system. You may +want to use the ``--src`` option to specify the parent directory where +the source will be checked out and the ``--user`` option such that the +package will be installed to your home directory (compare with the +output of ``pip install --help``). + +Code Contribution Ideas +~~~~~~~~~~~~~~~~~~~~~~~ + +- We maintain a set of `issues marked as + “bite-sized” `__. + These are issues that would serve as a good introduction to the + codebase. Claim one and start exploring! +- Like testing? Our `test + coverage `__ is somewhat + low. You can help out by finding low-coverage modules or checking out + other `testing-related + issues `__. +- There are several ways to improve the tests in general (see + `Testing `__) and some + places to think about performance optimization (see + `Optimization `__). +- Not all of our code is up to our coding conventions. In particular, + the `API + documentation `__ + are currently quite sparse. You can help by adding to the docstrings + in the code and to the documentation pages themselves. beets follows + `PEP-257 `__ for + docstrings and in some places, we also sometimes use `ReST autodoc + syntax for + Sphinx `__ + to, for example, refer to a class name. + +Your First Contribution +----------------------- + +If this is your first time contributing to an open source project, +welcome! If you are confused at all about how to contribute or what to +contribute, take a look at `this great +tutorial `__, or stop by our +`forums `__ if you have any questions. + +We maintain a list of issues we reserved for those new to open source +labeled `“first timers +only” `__. +Since the goal of these issues is to get users comfortable with +contributing to an open source project, please do not hesitate to ask +any questions. + +How to Submit Your Work +----------------------- + +Do you have a great bug fix, new feature, or documentation expansion +you’d like to contribute? Follow these steps to create a GitHub pull +request and your code will ship in no time. + +1. Fork the beets repository and clone it (see above) to create a + workspace. +2. Make your changes. +3. Add tests. If you’ve fixed a bug, write a test to ensure that you’ve + actually fixed it. If there’s a new feature or plugin, please + contribute tests that show that your code does what it says. +4. Add documentation. If you’ve added a new command flag, for example, + find the appropriate page under ``docs/`` where it needs to be + listed. +5. Add a changelog entry to ``docs/changelog.rst`` near the top of the + document. +6. Run the tests and style checker. The easiest way to run the tests is + to use `tox `__. For more + information on running tests, see our `Testing wiki + page `__. +7. Push to your fork and open a pull request! We’ll be in touch shortly. +8. If you add commits to a pull request, please add a comment or + re-request a review after you push them since GitHub doesn’t + automatically notify us when commits are added. + +Remember, code contributions have four parts: the code, the tests, the +documentation, and the changelog entry. Thank you for contributing! + +The Code +======== + +The documentation has an `API +section `__ that +serves as an introduction to beets’ design. + +Coding Conventions +------------------ + +There are a few coding conventions we use in beets: + +- Whenever you access the library database, do so through the provided + Library methods or via a Transaction object. Never call + ``lib.conn.*`` directly. For example, do this: + + :: + + with g.lib.transaction() as tx: + rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' + .format(field, model._table, sort_field)) + + To fetch Item objects from the database, use lib.items(…) and supply + a query as an argument. Resist the urge to write raw SQL for your + query. If you must use lower-level queries into the database, do + this: + + :: + + with lib.transaction() as tx: + rows = tx.query('SELECT …') + + Transaction objects help control concurrent access to the database + and assist in debugging conflicting accesses. +- Always use the `future + imports `__ + ``print_function``, ``division``, and ``absolute_import``, but *not* + ``unicode_literals``. These help keep your code modern and will help + in the eventual move to Python 3. +- ``str.format()`` should be used instead of the ``%`` operator +- Never ``print`` informational messages; use the + `logging `__ module + instead. In particular, we have our own logging shim, so you’ll see + ``from beets import logging`` in most files. + + - Always log Unicode strings (e.g., ``log.debug(u"hello world")``). + - The loggers use + `str.format `__-style + logging instead of ``%``-style, so you can type + ``log.debug(u"{0}", obj)`` to do your formatting. + +- Exception handlers must use ``except A as B:`` instead of + ``except A, B:``. + +We follow `PEP 8 `__ for +style. You can use ``tox -e lint`` to check your code for any style +errors. + +Handling Paths +-------------- + +A great deal of convention deals with the handling of **paths**. Paths +are stored internally—in the database, for instance—as byte strings +(i.e., ``bytes`` instead of ``str`` in Python 3). This is because POSIX +operating systems’ path names are only reliably usable as byte +strings—operating systems typically recommend but do not require that +filenames use a given encoding, so violations of any reported encoding +are inevitable. On Windows, the strings are always encoded with UTF-8; +on Unix, the encoding is controlled by the filesystem. Here are some +guidelines to follow: + +- If you have a Unicode path or you’re not sure whether something is + Unicode or not, pass it through ``bytestring_path`` function in the + ``beets.util`` module to convert it to bytes. +- Pass every path name trough the ``syspath`` function (also in + ``beets.util``) before sending it to any *operating system* file + operation (``open``, for example). This is necessary to use long + filenames (which, maddeningly, must be Unicode) on Windows. This + allows us to consistently store bytes in the database but use the + native encoding rule on both POSIX and Windows. +- Similarly, the ``displayable_path`` utility function converts + bytestring paths to a Unicode string for displaying to the user. + Every time you want to print out a string to the terminal or log it + with the ``logging`` module, feed it through this function. + +Editor Settings +--------------- + +Personally, I work on beets with `vim `__. Here are +some ``.vimrc`` lines that might help with PEP 8-compliant Python +coding: + +:: + + filetype indent on + autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent + +Consider installing `this alternative Python indentation +plugin `__. I also +like `neomake `__ with its flake8 +checker. From 2982f245e1a8a41758bfb8038aa9229e3330facd Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 18:55:46 -0700 Subject: [PATCH 596/613] add CONTRIBUTING.rst to the docs --- docs/contributing.rst | 3 +++ docs/index.rst | 1 + 2 files changed, 4 insertions(+) create mode 100644 docs/contributing.rst diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 000000000..6af7deaef --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,3 @@ +.. contributing: + +.. include:: ../CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst index 62c87461b..1d03a0f27 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents reference/index plugins/index faq + contributing dev/index .. toctree:: From 66e6af48be5b09bba5b71cbfbfa4e53521a9a887 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 19:38:55 -0700 Subject: [PATCH 597/613] add table of contents to contributing.rst --- CONTRIBUTING.rst | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2596fd135..66298c199 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,3 +1,10 @@ +############ +Contributing +############ + +.. contents:: + :depth: 3 + Thank you! ========== @@ -52,19 +59,19 @@ Programming docs `__. Getting the Source -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ The easiest way to get started with the latest beets source is to use `pip `__ to install an “editable” package. This can be done with one command: -:: +.. code-block:: bash $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets Or, equivalently: -:: +.. code-block:: bash $ git clone https://github.com/beetbox/beets.git $ cd beets @@ -80,7 +87,7 @@ package will be installed to your home directory (compare with the output of ``pip install --help``). Code Contribution Ideas -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ - We maintain a set of `issues marked as “bite-sized” `__. @@ -107,7 +114,7 @@ Code Contribution Ideas to, for example, refer to a class name. Your First Contribution ------------------------ +======================= If this is your first time contributing to an open source project, welcome! If you are confused at all about how to contribute or what to @@ -123,7 +130,7 @@ contributing to an open source project, please do not hesitate to ask any questions. How to Submit Your Work ------------------------ +======================= Do you have a great bug fix, new feature, or documentation expansion you’d like to contribute? Follow these steps to create a GitHub pull @@ -160,15 +167,17 @@ section `__ that serves as an introduction to beets’ design. Coding Conventions ------------------- +================== +General +------- There are a few coding conventions we use in beets: - Whenever you access the library database, do so through the provided Library methods or via a Transaction object. Never call ``lib.conn.*`` directly. For example, do this: - :: + .. code-block:: python with g.lib.transaction() as tx: rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' @@ -179,7 +188,7 @@ There are a few coding conventions we use in beets: query. If you must use lower-level queries into the database, do this: - :: + .. code-block:: python with lib.transaction() as tx: rows = tx.query('SELECT …') @@ -206,6 +215,8 @@ There are a few coding conventions we use in beets: - Exception handlers must use ``except A as B:`` instead of ``except A, B:``. +Style +----- We follow `PEP 8 `__ for style. You can use ``tox -e lint`` to check your code for any style errors. @@ -238,7 +249,7 @@ guidelines to follow: with the ``logging`` module, feed it through this function. Editor Settings ---------------- +^^^^^^^^^^^^^^^ Personally, I work on beets with `vim `__. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python From f677fd24078a634271c6b20e6c3bdca48142cdc8 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 19:39:06 -0700 Subject: [PATCH 598/613] add testing wiki to contributing.rst --- CONTRIBUTING.rst | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 66298c199..8f6ab54b2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -264,3 +264,95 @@ Consider installing `this alternative Python indentation plugin `__. I also like `neomake `__ with its flake8 checker. + +Testing +======= + +Running the Tests +----------------- + +To run the tests for multiple Python versions, compile the docs, and +check style, use `tox`_. Just type ``tox`` or use something like +``tox -e py27`` to test a specific configuration. `detox`_ makes this go +faster. + +You can disable a hand-selected set of "slow" tests by setting the +environment variable SKIP_SLOW_TESTS before running them. + +Other ways to run the tests: + +- ``python testall.py`` (ditto) +- ``python -m unittest discover -p 'test_*'`` (ditto) +- `pytest`_ + +You can also see the latest test results on `Linux`_ and on `Windows`_. + +Test Dependencies +^^^^^^^^^^^^^^^^^ + +The tests have a few more dependencies than beets itself. (The +additional dependencies consist of testing utilities and dependencies of +non-default plugins exercised by the test suite.) The dependencies are +listed under 'test' in ``extras_require`` in `setup.py`_. +To install the test dependencies, run ``python -m pip install .[test]``. +Or, just run a test suite with ``tox`` which will install them +automatically. + +.. _setup.py: https://github.com/beetbox/beets/blob/master/setup.py#L99` + +Writing Tests +------------- + +Writing tests is done by adding or modifying files in folder `test`_. +Take a look at +`https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`_ +to get a basic view on how tests are written. Despite using ``pytest`` +as a test runner, we prefer to write tests using the standard +`unittest`_ testing framework. + +Any tests that involve sending out network traffic e.g. an external API +call, should be skipped normally and run under our weekly `integration +test`_ suite. These tests can be useful in detecting external changes +that would affect ``beets``. In order to do this, simply add the +following snippet before the applicable test case: + +:: + + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') + +If you do this, it is also advised to create a similar test that 'mocks' +the network call and can be run under normal circumstances by our CI and +others. See `unittest.mock`_ for more info. + +Basics +^^^^^^ + +- Your file should contain a class derived from unittest.TestCase +- Each method in this class which name starts with the letters *test* + will be executed to test functionality +- Errors are raised with these methods: + + - ``self.assertEqual`` + - ``self.assertTrue`` + - ``self.assertFalse`` + - ``self.assertRaises`` + +- For detailed information see `Python unittest`_ +- **AVOID** using the ``start()`` and ``stop()`` methods of + ``mock.patch``, as they require manual cleanup. Use the annotation or + context mana + +.. _tox: http://tox.readthedocs.org +.. _detox: https://pypi.python.org/pypi/detox/ +.. _pytest: http://pytest.org +.. _Linux: https://github.com/beetbox/beets/actions +.. _Windows: https://ci.appveyor.com/project/beetbox/beets/ +.. _`https://github.com/beetbox/beets/blob/master/setup.py#L99`: https://github.com/beetbox/beets/blob/master/setup.py#L99 +.. _test: https://github.com/beetbox/beets/tree/master/test +.. _`https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`: https://github.com/beetbox/beets/blob/master/test/test_template.py#L224 +.. _unittest: https://docs.python.org/3.8/library/unittest.html +.. _integration test: https://github.com/beetbox/beets/actions?query=workflow%3A%22integration+tests%22 +.. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html +.. _Python unittest: https://docs.python.org/2/library/unittest.html From cb4bab175a4b5348a058654a527c04b3a2f9c233 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 19:51:48 -0700 Subject: [PATCH 599/613] fix link to contributing page in readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d57afa933..4a2d8cb10 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.md`_. +Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.rst`_. -.. _CONTRIBUTING.md: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.md +.. _CONTRIBUTING.rst: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.rst Read More --------- From bf868be10ea19955dd5dd9b28d8206665374e9ec Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 21 Jul 2020 14:41:01 -0700 Subject: [PATCH 600/613] fix-up adding testing wiki to contributing.rst --- CONTRIBUTING.rst | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8f6ab54b2..5e3a155cf 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -98,8 +98,7 @@ Code Contribution Ideas low. You can help out by finding low-coverage modules or checking out other `testing-related issues `__. -- There are several ways to improve the tests in general (see - `Testing `__) and some +- There are several ways to improve the tests in general (see :ref:`testing` and some places to think about performance optimization (see `Optimization `__). - Not all of our code is up to our coding conventions. In particular, @@ -149,8 +148,7 @@ request and your code will ship in no time. document. 6. Run the tests and style checker. The easiest way to run the tests is to use `tox `__. For more - information on running tests, see our `Testing wiki - page `__. + information on running tests, see :ref:`testing`. 7. Push to your fork and open a pull request! We’ll be in touch shortly. 8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn’t @@ -265,6 +263,8 @@ plugin `__. I also like `neomake `__ with its flake8 checker. +.. _testing: + Testing ======= @@ -287,6 +287,19 @@ Other ways to run the tests: You can also see the latest test results on `Linux`_ and on `Windows`_. +Coverage +^^^^^^^^ + +``tox -e cov`` will add coverage info for tests: Coverage is pretty low +still -- see the current status on `Codecov`_. + +Red Flags +^^^^^^^^^ + +The `pytest-random`_ plugin makes it easy to randomize the order of +tests. ``py.test test --random`` will occasionally turn up failing tests +that reveal ordering dependencies—which are bad news! + Test Dependencies ^^^^^^^^^^^^^^^^^ @@ -342,8 +355,11 @@ Basics - For detailed information see `Python unittest`_ - **AVOID** using the ``start()`` and ``stop()`` methods of ``mock.patch``, as they require manual cleanup. Use the annotation or - context mana + context manager forms instead. +.. _Python unittest: https://docs.python.org/2/library/unittest.html +.. _Codecov: https://codecov.io/github/beetbox/beets +.. _pytest-random: https://github.com/klrmn/pytest-random .. _tox: http://tox.readthedocs.org .. _detox: https://pypi.python.org/pypi/detox/ .. _pytest: http://pytest.org From 9690e4d51fe91081921fff2dc0e370d7c61aeec5 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 21 Jul 2020 14:49:51 -0700 Subject: [PATCH 601/613] improve code blocks --- CONTRIBUTING.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5e3a155cf..6c024e3d9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -251,9 +251,7 @@ Editor Settings Personally, I work on beets with `vim `__. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python -coding: - -:: +coding:: filetype indent on autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent @@ -329,11 +327,11 @@ test`_ suite. These tests can be useful in detecting external changes that would affect ``beets``. In order to do this, simply add the following snippet before the applicable test case: -:: +.. code-block:: python - @unittest.skipUnless( - os.environ.get('INTEGRATION_TEST', '0') == '1', - 'integration testing not enabled') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') If you do this, it is also advised to create a similar test that 'mocks' the network call and can be run under normal circumstances by our CI and From 6febcf11903c31b35b644ca82584ea187d32a155 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:14:43 +0200 Subject: [PATCH 602/613] fix #3685 --- beets/config_default.yaml | 1 + beets/dbcore/db.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 0fd6eb592..c75778b80 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -44,6 +44,7 @@ replace: '^\s+': '' '^-': _ path_sep_replace: _ +drive_sep_replace: _ asciify_paths: false art_filename: cover max_filename_length: 0 diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index b13f2638a..abcbdd029 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -19,6 +19,7 @@ from __future__ import division, absolute_import, print_function import time import os +import re from collections import defaultdict import threading import sqlite3 @@ -83,6 +84,8 @@ class FormattedMapping(Mapping): value = value.decode('utf-8', 'ignore') if self.for_path: + if re.match(r'^\w:', value): + value = value.replace(':', beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() for sep in (os.path.sep, os.path.altsep): if sep: From dfdb068b51288b77ccb238edb48d1d082d8d705e Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:24:39 +0200 Subject: [PATCH 603/613] fix linting error --- beets/dbcore/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index abcbdd029..595078c42 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -85,7 +85,8 @@ class FormattedMapping(Mapping): if self.for_path: if re.match(r'^\w:', value): - value = value.replace(':', beets.config['drive_sep_replace'].as_str()) + value = value.replace(':', + beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() for sep in (os.path.sep, os.path.altsep): if sep: From 8dad7fa67fc19dae6e33101fcafa9f4bb3458cd3 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:26:21 +0200 Subject: [PATCH 604/613] fix linting error again --- beets/dbcore/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 595078c42..38fbae3bc 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -85,8 +85,8 @@ class FormattedMapping(Mapping): if self.for_path: if re.match(r'^\w:', value): - value = value.replace(':', - beets.config['drive_sep_replace'].as_str()) + value = value.replace(':', + beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() for sep in (os.path.sep, os.path.altsep): if sep: From b91efb88f484dad47b4b4abe333447cabfabf66e Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:30:13 +0200 Subject: [PATCH 605/613] extract config to variable --- beets/dbcore/db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 38fbae3bc..9dbe9316b 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -84,10 +84,11 @@ class FormattedMapping(Mapping): value = value.decode('utf-8', 'ignore') if self.for_path: - if re.match(r'^\w:', value): - value = value.replace(':', - beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() + sep_drive = beets.config['drive_sep_replace'].as_str() + if re.match(r'^\w:', value): + value = value.replace(':', sep_drive) + for sep in (os.path.sep, os.path.altsep): if sep: value = value.replace(sep, sep_repl) From 1e6db940450a561d110fb3e716bd6f0c11149905 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:32:11 +0200 Subject: [PATCH 606/613] remove whitespace --- beets/dbcore/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 9dbe9316b..0f43f4bc5 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -86,9 +86,9 @@ class FormattedMapping(Mapping): if self.for_path: sep_repl = beets.config['path_sep_replace'].as_str() sep_drive = beets.config['drive_sep_replace'].as_str() - if re.match(r'^\w:', value): + if re.match(r'^\w:', value): value = value.replace(':', sep_drive) - + for sep in (os.path.sep, os.path.altsep): if sep: value = value.replace(sep, sep_repl) From 0a9a801b0538bff872cd18666988840ae1ab7b84 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 21:50:33 +0200 Subject: [PATCH 607/613] add unit tests --- test/test_files.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_files.py b/test/test_files.py index f31779672..ee50e583e 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -102,6 +102,19 @@ class MoveTest(_common.TestCase): self.i.move() self.assertEqual(self.i.path, old_path) + def test_move_file_with_colon(self): + self.i.artist = u'C:DOS' + self.i.move() + self.assertRegex(self.i.path.decode(), r'C_DOS') + + def test_move_file_with_colon_alt_separator(self): + old = beets.config['drive_sep_replace'] + beets.config["drive_sep_replace"] = '0' + self.i.artist = u'C:DOS' + self.i.move() + self.assertRegex(self.i.path.decode(), r"C0DOS") + beets.config["drive_sep_replace"] = old + def test_read_only_file_copied_writable(self): # Make the source file read-only. os.chmod(self.path, 0o444) From f73a438dce79cdf48d6d14c730a1cb8d7b166408 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 25 Jul 2020 13:02:19 -0700 Subject: [PATCH 608/613] Add python 3.9-dev to CI tests (#3687) * Add python 3.9-dev to ci tests * test integration tests on latest versions of python --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/integration_test.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f9cce8d23..08e7548f2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,7 +6,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9-dev] env: PY_COLORS: 1 @@ -82,4 +82,4 @@ jobs: python -m pip install tox sphinx - name: Lint with flake8 - run: tox -e py-lint \ No newline at end of file + run: tox -e py-lint diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 05623bf20..1e8b3a773 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up latest Python version uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9-dev - name: Install base dependencies run: | From 043fa10f5566c2cab0b7d6c15590cd8f60e6be4e Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 25 Jul 2020 13:58:57 -0500 Subject: [PATCH 609/613] Add tox windows issue to CONTRIBUTING.rst --- CONTRIBUTING.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6c024e3d9..b1bd22169 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -285,6 +285,11 @@ Other ways to run the tests: You can also see the latest test results on `Linux`_ and on `Windows`_. +Note, if you are on Windows and are seeing errors running tox, it may be related to `this issue`_, +in which case you may have to install tox v3.8.3 e.g. ``python -m pip install tox=3.8.3`` + +.. _this issue: https://github.com/tox-dev/tox/issues/1550 + Coverage ^^^^^^^^ From 1eda0d39e6dcb8f75741e50e9379ef7da0ccdd28 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 22:11:07 +0200 Subject: [PATCH 610/613] better regex for drive letter; hopefully fixed tests for 2.7; added test to show only drive-letter like colons are affected --- beets/dbcore/db.py | 3 ++- test/test_files.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 0f43f4bc5..46b47a2e1 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -86,8 +86,9 @@ class FormattedMapping(Mapping): if self.for_path: sep_repl = beets.config['path_sep_replace'].as_str() sep_drive = beets.config['drive_sep_replace'].as_str() + if re.match(r'^\w:', value): - value = value.replace(':', sep_drive) + value = re.sub(r'(?<=^\w):', sep_drive, value) for sep in (os.path.sep, os.path.altsep): if sep: diff --git a/test/test_files.py b/test/test_files.py index ee50e583e..13a8b4407 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -105,14 +105,20 @@ class MoveTest(_common.TestCase): def test_move_file_with_colon(self): self.i.artist = u'C:DOS' self.i.move() - self.assertRegex(self.i.path.decode(), r'C_DOS') + self.assertIn('C_DOS', self.i.path.decode()) + + def test_move_file_with_multiple_colons(self): + print(beets.config['replace']) + self.i.artist = u'COM:DOS' + self.i.move() + self.assertIn('COM_DOS', self.i.path.decode()) def test_move_file_with_colon_alt_separator(self): old = beets.config['drive_sep_replace'] beets.config["drive_sep_replace"] = '0' self.i.artist = u'C:DOS' self.i.move() - self.assertRegex(self.i.path.decode(), r"C0DOS") + self.assertIn('C0DOS', self.i.path.decode()) beets.config["drive_sep_replace"] = old def test_read_only_file_copied_writable(self): From 9ee341374af8e1f3a4011d98aa0a20aaaa34886b Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 25 Jul 2020 15:56:00 -0700 Subject: [PATCH 611/613] fix subheading in contributing.rst --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b1bd22169..dc8617712 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -247,7 +247,7 @@ guidelines to follow: with the ``logging`` module, feed it through this function. Editor Settings -^^^^^^^^^^^^^^^ +--------------- Personally, I work on beets with `vim `__. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python From 08b51ee5bc009133140307d42fc80782a3905be1 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sun, 26 Jul 2020 20:53:30 +0200 Subject: [PATCH 612/613] add changelog entry --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0f41c38ec..64e6ab85c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -235,6 +235,9 @@ Fixes: * :doc:`/plugins/ipfs`: Fix Python 3 compatibility. Thanks to :user:`musoke`. :bug:`2554` +* Fix a bug that caused metadata starting with something resembling a drive + letter to be incorrectly split into an extra directory after the colon. + :bug:`3685` For plugin developers: From 30173b7331fb887f3b036865e4cad41ff4516a4c Mon Sep 17 00:00:00 2001 From: chipsnblip Date: Sun, 26 Jul 2020 13:50:05 -0700 Subject: [PATCH 613/613] update link to current homepage update link from non-existent page (https://2.python-requests.org/en/latest/) to the plugin's current homepage: https://requests.readthedocs.io/en/master/ --- docs/plugins/fetchart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 92ed5f32a..168ca0fa0 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -11,7 +11,7 @@ To use the ``fetchart`` plugin, first enable it in your configuration (see The plugin uses `requests`_ to fetch album art from the Web. -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ Fetching Album Art During Import --------------------------------