From ca8c557840389f85c2f8a4803d12853caf1a658c Mon Sep 17 00:00:00 2001 From: Nathan Musoke Date: Tue, 9 May 2017 14:19:01 +1200 Subject: [PATCH 001/293] bugfix: Python3ify the IPFS plugin Paths were being constructed in a Python 3-incompatible way by concating bytes and strings. Do this more carefully by encoding and decoding of binary and strings. --- beetsplug/ipfs.py | 12 ++++++------ docs/changelog.rst | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/beetsplug/ipfs.py b/beetsplug/ipfs.py index 9a9d6aa50..88e305213 100644 --- a/beetsplug/ipfs.py +++ b/beetsplug/ipfs.py @@ -190,7 +190,7 @@ class IPFSPlugin(BeetsPlugin): else: lib_name = _hash lib_root = os.path.dirname(lib.path) - remote_libs = lib_root + "/remotes" + remote_libs = os.path.join(lib_root, b"remotes") if not os.path.exists(remote_libs): try: os.makedirs(remote_libs) @@ -198,7 +198,7 @@ class IPFSPlugin(BeetsPlugin): msg = "Could not create {0}. Error: {1}".format(remote_libs, e) self._log.error(msg) return False - path = remote_libs + "/" + lib_name + ".db" + path = os.path.join(remote_libs, lib_name.encode() + b".db") if not os.path.exists(path): cmd = "ipfs get {0} -o".format(_hash).split() cmd.append(path) @@ -209,7 +209,7 @@ class IPFSPlugin(BeetsPlugin): return False # add all albums from remotes into a combined library - jpath = remote_libs + "/joined.db" + jpath = os.path.join(remote_libs, b"joined.db") jlib = library.Library(jpath) nlib = library.Library(path) for album in nlib.albums(): @@ -237,7 +237,7 @@ class IPFSPlugin(BeetsPlugin): return for album in albums: - ui.print_(format(album, fmt), " : ", album.ipfs) + ui.print_(format(album, fmt), " : ", album.ipfs.decode()) def query(self, lib, args): rlib = self.get_remote_lib(lib) @@ -246,8 +246,8 @@ class IPFSPlugin(BeetsPlugin): def get_remote_lib(self, lib): lib_root = os.path.dirname(lib.path) - remote_libs = lib_root + "/remotes" - path = remote_libs + "/joined.db" + remote_libs = os.path.join(lib_root, b"remotes") + path = os.path.join(remote_libs, b"joined.db") if not os.path.isfile(path): raise IOError return library.Library(path) diff --git a/docs/changelog.rst b/docs/changelog.rst index bf0d7d254..b80df36c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -102,6 +102,7 @@ Fixes: error message. Thanks to :user:`Mary011196`. :bug:`1676` :bug:`2508` * :doc:`/plugins/web`: Avoid a crash when sending binary data, such as Chromaprint fingerprints, in music attributes. :bug:`2542` :bug:`2532` +* :doc:`/plugins/ipfs`: Fix Python 3 compatibility. Two plugins had backends removed due to bitrot: From c5319274ca4e8322c250d0b7a9ae9ac036ec4432 Mon Sep 17 00:00:00 2001 From: Nathan Musoke Date: Tue, 9 May 2017 15:38:50 +1200 Subject: [PATCH 002/293] IPFS plugin: Add note to check hashes carefully In the future, just checking that a hash begins with "Qm" and has length 46 will likely not be sufficient. --- beetsplug/ipfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/ipfs.py b/beetsplug/ipfs.py index 88e305213..0e9b0c1f1 100644 --- a/beetsplug/ipfs.py +++ b/beetsplug/ipfs.py @@ -147,6 +147,8 @@ class IPFSPlugin(BeetsPlugin): def ipfs_get(self, lib, query): query = query[0] # Check if query is a hash + # TODO: generalize to other hashes; probably use a multihash + # implementation if query.startswith("Qm") and len(query) == 46: self.ipfs_get_from_hash(lib, query) else: From 25e55a88a3acf2c6324aac28395b7d5782fa5219 Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:23:18 -0400 Subject: [PATCH 003/293] Clarify getitem docstring. Log when falling back. I figure, let someone change the docstring again if the function gets a special case besides artist / album artist. --- beets/library.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 5d90ae43b..d5fb5fa75 100644 --- a/beets/library.py +++ b/beets/library.py @@ -410,7 +410,8 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): raise KeyError(key) def __getitem__(self, key): - """Get the value for a key. Certain unset values are remapped. + """Get the value for a key. `artist` and `albumartist` + are fallback values for each other when unmapped. """ value = self._get(key) @@ -418,8 +419,10 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: + log.debug('No artist, using album artist {0}'.format(value)) return self._get('albumartist') elif key == 'albumartist' and not value: + log.debug('No albumartist, using artist {0}'.format(value)) return self._get('artist') else: return value From 59dec3f4da17d107fc15a7108e02592cf09df8ba Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:29:32 -0400 Subject: [PATCH 004/293] Log the new value --- beets/library.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/beets/library.py b/beets/library.py index d5fb5fa75..257197149 100644 --- a/beets/library.py +++ b/beets/library.py @@ -419,11 +419,13 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - log.debug('No artist, using album artist {0}'.format(value)) - return self._get('albumartist') + new_value = self._get('albumartist') + log.debug('No artist, using album artist {0}'.format(new_value)) + return new_value elif key == 'albumartist' and not value: - log.debug('No albumartist, using artist {0}'.format(value)) - return self._get('artist') + new_value = self._get('artist') + log.debug('No albumartist, using artist {0}'.format(new_value)) + return new_value else: return value From 918d833e35c6230e74465773ab75399811af8d5a Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:30:25 -0400 Subject: [PATCH 005/293] fix(?) tabs --- beets/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index 257197149..d9afac677 100644 --- a/beets/library.py +++ b/beets/library.py @@ -419,11 +419,11 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - new_value = self._get('albumartist') + new_value = self._get('albumartist') log.debug('No artist, using album artist {0}'.format(new_value)) return new_value elif key == 'albumartist' and not value: - new_value = self._get('artist') + new_value = self._get('artist') log.debug('No albumartist, using artist {0}'.format(new_value)) return new_value else: From 73ede2642db21b5a3f3e929cc2d6901f580605d6 Mon Sep 17 00:00:00 2001 From: RollingStar Date: Wed, 30 Oct 2019 20:33:21 -0400 Subject: [PATCH 006/293] Better docstring; fix tabs again --- beets/library.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/library.py b/beets/library.py index d9afac677..3ad875991 100644 --- a/beets/library.py +++ b/beets/library.py @@ -411,7 +411,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def __getitem__(self, key): """Get the value for a key. `artist` and `albumartist` - are fallback values for each other when unmapped. + are fallback values for each other when not set. """ value = self._get(key) @@ -419,11 +419,11 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - new_value = self._get('albumartist') + new_value = self._get('albumartist') log.debug('No artist, using album artist {0}'.format(new_value)) return new_value elif key == 'albumartist' and not value: - new_value = self._get('artist') + new_value = self._get('artist') log.debug('No albumartist, using artist {0}'.format(new_value)) return new_value else: From 0191794285422eb4173421c04065f704f14a2b34 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 10 Nov 2019 21:59:58 +0100 Subject: [PATCH 007/293] subsonicplaylist plugin --- beetsplug/subsonicplaylist.py | 126 ++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + docs/plugins/index.rst | 1 + docs/plugins/subsonicplaylist.rst | 24 ++++++ 4 files changed, 152 insertions(+) create mode 100644 beetsplug/subsonicplaylist.py create mode 100644 docs/plugins/subsonicplaylist.rst diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py new file mode 100644 index 000000000..5b7ed95bc --- /dev/null +++ b/beetsplug/subsonicplaylist.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2019, Joris Jensen +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from __future__ import absolute_import, division, print_function + +import os +from hashlib import md5 +import xml.etree.ElementTree as ET +import random +import string +import requests +from beets.util import normpath, bytestring_path, mkdirall, syspath, \ + path_as_posix, sanitize_path + +from beets.ui import Subcommand + +from beets.plugins import BeetsPlugin + +__author__ = 'https://github.com/MrNuggelz' + + +class SubsonicPlaylistPlugin(BeetsPlugin): + + def __init__(self): + super(SubsonicPlaylistPlugin, self).__init__() + self.config.add( + { + 'relative_to': None, + 'playlist_dir': '.', + 'forward_slash': False, + 'playlist_ids': [], + 'playlist_names': [], + 'username': '', + 'password': '' + } + ) + self.config['password'].redact = True + + def create_playlist(self, xml, lib): + relative_to = self.config['relative_to'].get() + if relative_to: + relative_to = normpath(relative_to) + + playlist = ET.fromstring(xml)[0] + if playlist.attrib.get('code', '200') != '200': + alt_error = 'error getting playlist, but no error message found' + self._log.warn(playlist.attrib.get('message', alt_error)) + return + name = '{}.m3u'.format() + tracks = [(t.attrib['artist'], t.attrib['album'], t.attrib['title']) + for t in playlist] + track_paths = [] + for t in tracks: + query = 'artist:"{}" album:"{}" title:"{}"'.format(*t) + items = lib.items(query) + if len(items) > 0: + item_path = items[0].path + if relative_to: + item_path = os.path.relpath(items[0].path, relative_to) + track_paths.append(item_path) + else: + self._log.warn(u"{} | track not found ({})", name, query) + # write playlist + playlist_dir = self.config['playlist_dir'].as_filename() + playlist_dir = bytestring_path(playlist_dir) + m3u_path = normpath(os.path.join(playlist_dir, bytestring_path(name))) + mkdirall(m3u_path) + with open(syspath(m3u_path), 'wb') as f: + for path in track_paths: + if self.config['forward_slash'].get(): + path = path_as_posix(path) + f.write(path + b'\n') + + def get_playlist_by_id(self, playlist_id, lib): + xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text + self.create_playlist(xml, lib) + + def commands(self): + def build_playlist(lib, opts, args): + + if len(self.config['playlist_ids'].as_str_seq()) > 0: + for playlist_id in self.config['playlist_ids'].as_str_seq(): + self.get_playlist_by_id(playlist_id, lib) + if len(self.config['playlist_names'].as_str_seq()) > 0: + playlists = ET.fromstring(self.send('getPlaylists').text)[0] + if playlists.attrib.get('code', '200') != '200': + alt_error = 'error getting playlists,' \ + ' but no erro message found' + self._log.warn(playlists.attrib.get('message', alt_error)) + return + for name in self.config['playlist_names'].as_str_seq(): + for playlist in playlists: + if name == playlist.attrib['name']: + self.get_playlist_by_id(playlist.attrib['id'], lib) + + subsonicplaylist_cmds = Subcommand( + 'subsonicplaylist', help=u'import a subsonic playlist' + ) + subsonicplaylist_cmds.func = build_playlist + return [subsonicplaylist_cmds] + + def generate_token(self): + salt = ''.join(random.choices(string.ascii_lowercase + string.digits)) + return md5( + (self.config['password'].get() + salt).encode()).hexdigest(), salt + + def send(self, endpoint, params=''): + url = '{}/rest/{}?u={}&t={}&s={}&v=1.12.0&c=beets'.format( + self.config['base_url'].get(), + endpoint, + self.config['username'], + *self.generate_token()) + resp = requests.get(url + params) + return resp diff --git a/docs/changelog.rst b/docs/changelog.rst index 169da71ba..9a8d1ae5b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. * :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and `discogs_artistid` :bug: `3413` diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 066d6ec22..9cf2cc5a9 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -115,6 +115,7 @@ following to your configuration:: smartplaylist sonosupdate spotify + subsonicplaylist subsonicupdate the thumbnails diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst new file mode 100644 index 000000000..d6d64f60b --- /dev/null +++ b/docs/plugins/subsonicplaylist.rst @@ -0,0 +1,24 @@ +Subsonic Playlist Plugin +======================== + +The ``subsonicplaylist`` plugin allows to import playlist from a subsonic server + +Command Line Usage +------------------ + +To use the ``subsonicplaylist`` plugin, enable it in your configuration (see +:ref:`using-plugins`). Then use it by invoking the ``subsonicplaylist`` command. +The command will search the playlist on the subsonic server and create a playlist +using the beets library. Options to be defined in your config with their default value:: + + subsonicplaylist: + base_url: "https://your.subsonic.server" + 'relative_to': None, + 'playlist_dir': '.', + 'forward_slash': False, + 'playlist_ids': [], + 'playlist_names': [], + 'username': '', + 'password': '' + +Parameters `base_url`, `username` and `password` must be defined! \ No newline at end of file From ec696f5e58aa83240b578642fad9714455a3828e Mon Sep 17 00:00:00 2001 From: Joris Date: Mon, 11 Nov 2019 10:48:29 +0100 Subject: [PATCH 008/293] Removed unused sanitize_path import --- beetsplug/subsonicplaylist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 5b7ed95bc..9d3ee7cf6 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -22,7 +22,7 @@ import random import string import requests from beets.util import normpath, bytestring_path, mkdirall, syspath, \ - path_as_posix, sanitize_path + path_as_posix from beets.ui import Subcommand From bb305b17e14ee36bc46594418913d43c6128e02b Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 16 Nov 2019 14:36:48 +0100 Subject: [PATCH 009/293] set tags instead of creating m3u --- beetsplug/subsonicplaylist.py | 74 +++++++++++++++-------------------- 1 file changed, 31 insertions(+), 43 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 9d3ee7cf6..1dd73f939 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -15,14 +15,11 @@ from __future__ import absolute_import, division, print_function -import os from hashlib import md5 import xml.etree.ElementTree as ET import random import string import requests -from beets.util import normpath, bytestring_path, mkdirall, syspath, \ - path_as_posix from beets.ui import Subcommand @@ -37,9 +34,6 @@ class SubsonicPlaylistPlugin(BeetsPlugin): super(SubsonicPlaylistPlugin, self).__init__() self.config.add( { - 'relative_to': None, - 'playlist_dir': '.', - 'forward_slash': False, 'playlist_ids': [], 'playlist_names': [], 'username': '', @@ -48,62 +42,56 @@ class SubsonicPlaylistPlugin(BeetsPlugin): ) self.config['password'].redact = True - def create_playlist(self, xml, lib): - relative_to = self.config['relative_to'].get() - if relative_to: - relative_to = normpath(relative_to) + def update_tags(self, playlist_dict, lib): + for query, playlist_tag in playlist_dict.items(): + query = 'artist:"{}" album:"{}" title:"{}"'.format(*query) + items = lib.items(query) + if len(items) <= 0: + self._log.warn(u"{} | track not found ({})", playlist_tag, + query) + continue + for item in items: + item.update({'subsonic_playlist': playlist_tag}) + with lib.transaction(): + item.try_sync(write=True, move=False) + def get_playlist(self, playlist_id): + xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text playlist = ET.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' self._log.warn(playlist.attrib.get('message', alt_error)) return - name = '{}.m3u'.format() + + name = playlist.attrib.get('name', 'undefined') tracks = [(t.attrib['artist'], t.attrib['album'], t.attrib['title']) for t in playlist] - track_paths = [] - for t in tracks: - query = 'artist:"{}" album:"{}" title:"{}"'.format(*t) - items = lib.items(query) - if len(items) > 0: - item_path = items[0].path - if relative_to: - item_path = os.path.relpath(items[0].path, relative_to) - track_paths.append(item_path) - else: - self._log.warn(u"{} | track not found ({})", name, query) - # write playlist - playlist_dir = self.config['playlist_dir'].as_filename() - playlist_dir = bytestring_path(playlist_dir) - m3u_path = normpath(os.path.join(playlist_dir, bytestring_path(name))) - mkdirall(m3u_path) - with open(syspath(m3u_path), 'wb') as f: - for path in track_paths: - if self.config['forward_slash'].get(): - path = path_as_posix(path) - f.write(path + b'\n') - - def get_playlist_by_id(self, playlist_id, lib): - xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text - self.create_playlist(xml, lib) + return name, tracks def commands(self): def build_playlist(lib, opts, args): - - if len(self.config['playlist_ids'].as_str_seq()) > 0: - for playlist_id in self.config['playlist_ids'].as_str_seq(): - self.get_playlist_by_id(playlist_id, lib) + ids = self.config['playlist_ids'].as_str_seq() if len(self.config['playlist_names'].as_str_seq()) > 0: - playlists = ET.fromstring(self.send('getPlaylists').text)[0] + playlists = ET.fromstring(self.send('getPlaylists').text)[ + 0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ ' but no erro message found' - self._log.warn(playlists.attrib.get('message', alt_error)) + self._log.warn( + playlists.attrib.get('message', alt_error)) return for name in self.config['playlist_names'].as_str_seq(): for playlist in playlists: if name == playlist.attrib['name']: - self.get_playlist_by_id(playlist.attrib['id'], lib) + ids.append(playlist.attrib['id']) + playlist_dict = dict() + for playlist_id in ids: + name, tracks = self.get_playlist(playlist_id) + for track in tracks: + if track not in playlist_dict: + playlist_dict[track] = ';' + playlist_dict[track] += name + ';' + self.update_tags(playlist_dict, lib) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' From e9dee5dca83a02c9985d3fff61f18d9a83aea8c6 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 16 Nov 2019 20:30:53 +0100 Subject: [PATCH 010/293] added flag to delete old tags --- beetsplug/subsonicplaylist.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 1dd73f939..ef100c6da 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -34,6 +34,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): super(SubsonicPlaylistPlugin, self).__init__() self.config.add( { + 'delete': False, 'playlist_ids': [], 'playlist_names': [], 'username': '', @@ -70,6 +71,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def commands(self): def build_playlist(lib, opts, args): + self._parse_opts(opts) ids = self.config['playlist_ids'].as_str_seq() if len(self.config['playlist_names'].as_str_seq()) > 0: playlists = ET.fromstring(self.send('getPlaylists').text)[ @@ -91,14 +93,35 @@ class SubsonicPlaylistPlugin(BeetsPlugin): if track not in playlist_dict: playlist_dict[track] = ';' playlist_dict[track] += name + ';' + # delete old tags + if self.config['delete']: + for item in lib.items('subsonic_playlist:";"'): + item.update({'subsonic_playlist': ''}) + with lib.transaction(): + item.try_sync(write=True, move=False) + self.update_tags(playlist_dict, lib) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' ) + subsonicplaylist_cmds.parser.add_option( + u'-d', + u'--delete', + action='store_true', + help=u'delete tag from items not in any playlist anymore', + ) subsonicplaylist_cmds.func = build_playlist return [subsonicplaylist_cmds] + def _parse_opts(self, opts): + + if opts.delete: + self.config['delete'].set(True) + + self.opts = opts + return True + def generate_token(self): salt = ''.join(random.choices(string.ascii_lowercase + string.digits)) return md5( From 00330da623be15eb7fd8f4ad19be65d7222dd945 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 16 Nov 2019 21:10:41 +0100 Subject: [PATCH 011/293] updated documentation --- docs/plugins/subsonicplaylist.rst | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index d6d64f60b..22fe16816 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -1,21 +1,30 @@ Subsonic Playlist Plugin ======================== -The ``subsonicplaylist`` plugin allows to import playlist from a subsonic server +The ``subsonicplaylist`` plugin allows to import playlists from a subsonic server. +This is done by retrieving the track infos from the subsonic server, searching +them in the beets library and adding the playlist names to the +`subsonic_playlist` tag of the found items. The content of the tag has the format: + + subsonic_playlist: ";first playlist;second playlist" + +To get all items in a playlist use the query `;playlist name;`. Command Line Usage ------------------ To use the ``subsonicplaylist`` plugin, enable it in your configuration (see :ref:`using-plugins`). Then use it by invoking the ``subsonicplaylist`` command. -The command will search the playlist on the subsonic server and create a playlist -using the beets library. Options to be defined in your config with their default value:: +By default only the tags of the items found for playlists will be updated. +This means that, if one imported a playlist, then delete one song from it and +imported the playlist again, the deleted song will still have the playlist set +in its `subsonic_playlist` tag. To solve this problem one can use the `-d/--delete` +flag. This resets all `subsonic_playlist` tag before importing playlists. +Options to be defined in your config with their default value:: subsonicplaylist: - base_url: "https://your.subsonic.server" - 'relative_to': None, - 'playlist_dir': '.', - 'forward_slash': False, + 'base_url': "https://your.subsonic.server" + 'delete': False, 'playlist_ids': [], 'playlist_names': [], 'username': '', From d43cf35ad233739bf397dfdc484cfedb779fe90a Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Thu, 5 Dec 2019 20:06:46 -0700 Subject: [PATCH 012/293] Strip and lowercase Genius lyrics artist comparison --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 16699d9d3..1345018f0 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -395,7 +395,7 @@ class Genius(Backend): song_info = None for hit in json["response"]["hits"]: - if hit["result"]["primary_artist"]["name"] == artist: + if hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() == artist.lower(): song_info = hit break From c8e8e587f8ffd8319e941a3e201dfd6a516d368e Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Thu, 5 Dec 2019 20:06:49 -0700 Subject: [PATCH 013/293] Add debug logger for Genius lyrics no-match --- beetsplug/lyrics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 1345018f0..b10f8dd02 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -402,6 +402,8 @@ class Genius(Backend): if song_info: song_api_path = song_info["result"]["api_path"] return self.lyrics_from_song_api_path(song_api_path) + else: + self._log.debug(u'Genius did not return a matching artist entry') class LyricsWiki(SymbolsReplaced): From a1af5d280e56d51f2970997339582eae0305758b Mon Sep 17 00:00:00 2001 From: Tyler Faulk Date: Tue, 10 Dec 2019 22:54:31 -0500 Subject: [PATCH 014/293] added merge_environment_settings call in fetchart plugin to handle connections with proxy servers --- beetsplug/fetchart.py | 6 +++++- docs/changelog.rst | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index a815d4d9b..ca93d685e 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -164,13 +164,17 @@ def _logged_get(log, *args, **kwargs): message = 'getting URL' req = requests.Request('GET', *args, **req_kwargs) + with requests.Session() as s: s.headers = {'User-Agent': 'beets'} prepped = s.prepare_request(req) + settings = s.merge_environment_settings( + prepped.url, {}, None, None, None + ) + send_kwargs.update(settings) log.debug('{}: {}', message, prepped.url) return s.send(prepped, **send_kwargs) - class RequestMixin(object): """Adds a Requests wrapper to the class that uses the logger, which must be named `self._log`. diff --git a/docs/changelog.rst b/docs/changelog.rst index b5b6c3901..e24bb976e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -99,6 +99,9 @@ New features: Fixes: +* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take + environment variables such as proxy servers into account when making requests + :bug:`3450` * :doc:`/plugins/inline`: In function-style field definitions that refer to flexible attributes, values could stick around from one function invocation to the next. This meant that, when displaying a list of objects, later From 6d69d01016adaa780d660ee3c16199ad2afb96f5 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Mon, 13 Jan 2020 15:42:55 +0100 Subject: [PATCH 015/293] added database changed event to subsonicplaylist --- beetsplug/subsonicplaylist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index ef100c6da..fbbcd8ac3 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -20,6 +20,7 @@ import xml.etree.ElementTree as ET import random import string import requests +from beets import plugins from beets.ui import Subcommand @@ -101,6 +102,8 @@ class SubsonicPlaylistPlugin(BeetsPlugin): item.try_sync(write=True, move=False) self.update_tags(playlist_dict, lib) + ## notify plugins + plugins.send('database_change', lib=lib, model=self) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' From 53820c0a9843ce613feeb2d9896739f0d3cbf369 Mon Sep 17 00:00:00 2001 From: ybnd Date: Tue, 28 Jan 2020 09:45:20 +0100 Subject: [PATCH 016/293] Handle bs1770gain v0.6.0 XML output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove `0%\x08\x08` from output (backspace code doesn't resolve; progress percentages get spliced in) * Handle changed attributes/fields: * `sample-peak` attribute `factor` is called `amplitude` instead * Album summary is not included in a `summary` tag now, but in two separate `integrated` and `sample-peak` tags * Handle `lu` attribute * Get bs1770gain version * If v0.6.0 or later, add `--unit=ebu` flag to convert `db` attributes to LUFS * May be useful later on ### Output examples Track: ``` ``` Album: ``` ``` --- beetsplug/replaygain.py | 32 ++++++++++++++++++++++++++++++-- test/test_replaygain.py | 3 +-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 1076ac714..3e284ce62 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -22,6 +22,7 @@ import math import sys import warnings import enum +import re import xml.parsers.expat from six.moves import zip @@ -135,6 +136,11 @@ class Bs1770gainBackend(Backend): -18: "replaygain", } + version = re.search( + 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', + call(['bs1770gain', '--version']).stdout.decode('utf-8') + ).group(1) + def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) config.add({ @@ -252,6 +258,8 @@ class Bs1770gainBackend(Backend): cmd = [self.command] cmd += ["--" + method] cmd += ['--xml', '-p'] + if self.version >= '0.6.0': + cmd += ['--unit=ebu'] # set units to LU # Workaround for Windows: the underlying tool fails on paths # with the \\?\ prefix, so we don't use it here. This @@ -286,6 +294,7 @@ class Bs1770gainBackend(Backend): album_gain = {} # mutable variable so it can be set from handlers parser = xml.parsers.expat.ParserCreate(encoding='utf-8') state = {'file': None, 'gain': None, 'peak': None} + album_state = {'gain': None, 'peak': None} def start_element_handler(name, attrs): if name == u'track': @@ -294,9 +303,13 @@ class Bs1770gainBackend(Backend): raise ReplayGainError( u'duplicate filename in bs1770gain output') elif name == u'integrated': - state['gain'] = float(attrs[u'lu']) + if 'lu' in attrs: + state['gain'] = float(attrs[u'lu']) elif name == u'sample-peak': - state['peak'] = float(attrs[u'factor']) + if 'factor' in attrs: + state['peak'] = float(attrs[u'factor']) + elif 'amplitude' in attrs: + state['peak'] = float(attrs[u'amplitude']) def end_element_handler(name): if name == u'track': @@ -312,10 +325,25 @@ class Bs1770gainBackend(Backend): 'the output of bs1770gain') album_gain["album"] = Gain(state['gain'], state['peak']) state['gain'] = state['peak'] = None + elif len(per_file_gain) == len(path_list): + if state['gain'] is not None: + album_state['gain'] = state['gain'] + if state['peak'] is not None: + album_state['peak'] = state['peak'] + if album_state['gain'] is not None \ + and album_state['peak'] is not None: + album_gain["album"] = Gain( + album_state['gain'], album_state['peak']) + state['gain'] = state['peak'] = None + parser.StartElementHandler = start_element_handler parser.EndElementHandler = end_element_handler try: + if type(text) == bytes: + text = text.decode('utf-8') + while '\x08' in text: + text = re.sub('[^\x08]\x08', '', text) parser.Parse(text, True) except xml.parsers.expat.ExpatError: raise ReplayGainError( diff --git a/test/test_replaygain.py b/test/test_replaygain.py index fe0515bee..437b1426a 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain', ['--replaygain']): +if has_program('bs1770gain'): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -58,7 +58,6 @@ def reset_replaygain(item): class ReplayGainCliTestBase(TestHelper): - def setUp(self): self.setup_beets() self.config['replaygain']['backend'] = self.backend From c78afb1a97bb41788197ec1f7f3965e2e6a4f61b Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 16:37:56 +0100 Subject: [PATCH 017/293] Don't call bs1770gain outside of try statement --- beetsplug/replaygain.py | 12 +++++++----- test/test_replaygain.py | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3e284ce62..5b715191e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -136,11 +136,6 @@ class Bs1770gainBackend(Backend): -18: "replaygain", } - version = re.search( - 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', - call(['bs1770gain', '--version']).stdout.decode('utf-8') - ).group(1) - def __init__(self, config, log): super(Bs1770gainBackend, self).__init__(config, log) config.add({ @@ -155,6 +150,13 @@ class Bs1770gainBackend(Backend): try: call([cmd, "--help"]) self.command = cmd + try: + self.version = re.search( + '([0-9]+.[0-9]+.[0-9]+), ', + call([cmd, '--version']).stdout.decode('utf-8') + ).group(1) + except AttributeError: + self.version = '0.0.0' except OSError: raise FatalReplayGainError( u'Is bs1770gain installed?' diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 437b1426a..fe0515bee 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain'): +if has_program('bs1770gain', ['--replaygain']): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -58,6 +58,7 @@ def reset_replaygain(item): class ReplayGainCliTestBase(TestHelper): + def setUp(self): self.setup_beets() self.config['replaygain']['backend'] = self.backend From c1cb78c908b39d506aa6ea9384c1eb98938217cd Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 17:59:57 +0100 Subject: [PATCH 018/293] Small fixes in `replaygain.Bs1770gainBackend` and test_replaygain.py * Fix unspecified `gain_adjustment` when method defined in config * Fix difference between dB and LUFS values in case of mismatched `target_level`/`method`: ``` db_to_lufs( target_level ) - lufs_to_dB( -23 ) ``` * Ignore single assertion in case of bs1770gain (cherry picked from commit 2395bf224032c44f1ea5d28e0c63af96a92b96df) --- beetsplug/replaygain.py | 7 +++++-- test/test_replaygain.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 5b715191e..b86ef4b1e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -249,12 +249,15 @@ class Bs1770gainBackend(Backend): if self.__method != "": # backward compatibility to `method` option method = self.__method + gain_adjustment = target_level \ + - [k for k, v in self.methods.items() if v == method][0] elif target_level in self.methods: method = self.methods[target_level] gain_adjustment = 0 else: - method = self.methods[-23] - gain_adjustment = target_level - lufs_to_db(-23) + lufs_target = -23 + method = self.methods[lufs_target] + gain_adjustment = target_level - lufs_target # Construct shell command. cmd = [self.command] diff --git a/test/test_replaygain.py b/test/test_replaygain.py index fe0515bee..3f317aeb3 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -151,7 +151,9 @@ class ReplayGainCliTestBase(TestHelper): self.assertEqual(max(gains), min(gains)) self.assertNotEqual(max(gains), 0.0) - self.assertNotEqual(max(peaks), 0.0) + if not self.backend == "bs1770gain": + # Actually produces peaks == 0.0 ~ self.add_album_fixture + self.assertNotEqual(max(peaks), 0.0) def test_cli_writes_only_r128_tags(self): if self.backend == "command": From c3817a4c06ea2e029a9d6a07a5f0e1facd6779b1 Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 18:13:59 +0100 Subject: [PATCH 019/293] Implement review comments * safer version comparison * regex bytes directly * handle b'\x08 ...' case * test_replaygain.py: injected command output should match the type of the actual output --- beetsplug/replaygain.py | 37 ++++++++++++++++++++++++------------- test/test_replaygain.py | 4 ++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index b86ef4b1e..661fcc2e3 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -25,6 +25,7 @@ import enum import re import xml.parsers.expat from six.moves import zip +from packaging import version from beets import ui from beets.plugins import BeetsPlugin @@ -148,19 +149,18 @@ class Bs1770gainBackend(Backend): cmd = 'bs1770gain' try: - call([cmd, "--help"]) + version_out = call([cmd, '--version']) self.command = cmd - try: - self.version = re.search( - '([0-9]+.[0-9]+.[0-9]+), ', - call([cmd, '--version']).stdout.decode('utf-8') - ).group(1) - except AttributeError: - self.version = '0.0.0' + self.version = re.search( + '([0-9]+.[0-9]+.[0-9]+), ', + version_out.stdout.decode('utf-8') + ).group(1) except OSError: raise FatalReplayGainError( u'Is bs1770gain installed?' ) + except AttributeError: + self.version = '0.0.0' if not self.command: raise FatalReplayGainError( u'no replaygain command found: install bs1770gain' @@ -263,7 +263,7 @@ class Bs1770gainBackend(Backend): cmd = [self.command] cmd += ["--" + method] cmd += ['--xml', '-p'] - if self.version >= '0.6.0': + if version.parse(self.version) >= version.parse('0.6.0'): cmd += ['--unit=ebu'] # set units to LU # Workaround for Windows: the underlying tool fails on paths @@ -345,10 +345,21 @@ class Bs1770gainBackend(Backend): parser.EndElementHandler = end_element_handler try: - if type(text) == bytes: - text = text.decode('utf-8') - while '\x08' in text: - text = re.sub('[^\x08]\x08', '', text) + # Sometimes, the XML out put of `bs1770gain` gets spliced with + # some progress percentages: b'9%\x08\x0810%\x08\x08' + # that are supposed to be canceled out by appending + # a b'\x08' backspace characters for every character, + # + # For some reason, these backspace characters don't get + # resolved, resulting in mangled XML. + + # While there are backspace characters in the output + while b'\x08' in text: + text = re.sub(b'[^\x08]\x08|^\x08', b'', text) + # Replace every occurence of a non-backspace character + # followed by a backspace character or a backspace character + # at the beginning of the string by an empty byte string b'' + parser.Parse(text, True) except xml.parsers.expat.ExpatError: raise ReplayGainError( diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 3f317aeb3..3ac19f8f9 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']): else: GAIN_PROG_AVAILABLE = False -if has_program('bs1770gain', ['--replaygain']): +if has_program('bs1770gain'): LOUDNESS_PROG_AVAILABLE = True else: LOUDNESS_PROG_AVAILABLE = False @@ -252,7 +252,7 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): @patch('beetsplug.replaygain.call') def test_malformed_output(self, call_patch): # Return malformed XML (the ampersand should be &) - call_patch.return_value = CommandOutput(stdout=""" + call_patch.return_value = CommandOutput(stdout=b""" From 506be0259789c552f346d618d1774847e57ce0cf Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 30 Jan 2020 20:11:09 +0100 Subject: [PATCH 020/293] Remove `packaging` dependency --- beetsplug/replaygain.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 661fcc2e3..950969547 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -25,7 +25,6 @@ import enum import re import xml.parsers.expat from six.moves import zip -from packaging import version from beets import ui from beets.plugins import BeetsPlugin @@ -68,6 +67,11 @@ def call(args, **kwargs): raise ReplayGainError(u"argument encoding failed") +def after_version(version_a, version_b): + return tuple(int(s) for s in version_a.split('.')) \ + >= tuple(int(s) for s in version_b.split('.')) + + def db_to_lufs(db): """Convert db to LUFS. @@ -263,7 +267,7 @@ class Bs1770gainBackend(Backend): cmd = [self.command] cmd += ["--" + method] cmd += ['--xml', '-p'] - if version.parse(self.version) >= version.parse('0.6.0'): + if after_version(self.version, '0.6.0'): cmd += ['--unit=ebu'] # set units to LU # Workaround for Windows: the underlying tool fails on paths From bef473c8e85b375a2dc5a40feb82bc971286e550 Mon Sep 17 00:00:00 2001 From: ybnd Date: Fri, 31 Jan 2020 07:42:50 +0100 Subject: [PATCH 021/293] Remove spliced progress regex and add --suppress-progress flag --- beetsplug/replaygain.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 950969547..3c652c7bd 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -268,7 +268,8 @@ class Bs1770gainBackend(Backend): cmd += ["--" + method] cmd += ['--xml', '-p'] if after_version(self.version, '0.6.0'): - cmd += ['--unit=ebu'] # set units to LU + cmd += ['--unit=ebu'] # set units to LU + cmd += ['--suppress-progress'] # don't print % to XML output # Workaround for Windows: the underlying tool fails on paths # with the \\?\ prefix, so we don't use it here. This @@ -349,21 +350,6 @@ class Bs1770gainBackend(Backend): parser.EndElementHandler = end_element_handler try: - # Sometimes, the XML out put of `bs1770gain` gets spliced with - # some progress percentages: b'9%\x08\x0810%\x08\x08' - # that are supposed to be canceled out by appending - # a b'\x08' backspace characters for every character, - # - # For some reason, these backspace characters don't get - # resolved, resulting in mangled XML. - - # While there are backspace characters in the output - while b'\x08' in text: - text = re.sub(b'[^\x08]\x08|^\x08', b'', text) - # Replace every occurence of a non-backspace character - # followed by a backspace character or a backspace character - # at the beginning of the string by an empty byte string b'' - parser.Parse(text, True) except xml.parsers.expat.ExpatError: raise ReplayGainError( From 9f43408f1b25910f7ebe98e0a30aab6ff0a33611 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sun, 2 Feb 2020 15:57:43 -0700 Subject: [PATCH 022/293] Changelog and cleanup --- beetsplug/lyrics.py | 8 ++++++-- docs/changelog.rst | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index b10f8dd02..20e39548c 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -395,15 +395,19 @@ class Genius(Backend): song_info = None for hit in json["response"]["hits"]: - if hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() == artist.lower(): + # Genius uses zero-width characters to denote lowercase artist names + hit_artist = hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() + + if hit_artist == artist.lower(): song_info = hit break if song_info: + self._log.debug(u'fetched: {0}', song_info["result"]["url"]) song_api_path = song_info["result"]["api_path"] return self.lyrics_from_song_api_path(song_api_path) else: - self._log.debug(u'Genius did not return a matching artist entry') + self._log.debug(u'genius: no matching artist') class LyricsWiki(SymbolsReplaced): diff --git a/docs/changelog.rst b/docs/changelog.rst index c6b805b30..545bf7a84 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -134,6 +134,8 @@ Fixes: * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. Thanks to :user:`aereaux`. :bug:`3437` +* :doc:`/plugins/lyrics`: Fix a corner-case with Genius lowercase artist names + :bug:`3446` For plugin developers: From 95e0f54d7cea6c338efa47d16bc96c148970219f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 2 Feb 2020 21:04:55 -0500 Subject: [PATCH 023/293] Fix too-long lines (#3448) --- beetsplug/lyrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 20e39548c..0e797d5a3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -395,8 +395,10 @@ class Genius(Backend): song_info = None for hit in json["response"]["hits"]: - # Genius uses zero-width characters to denote lowercase artist names - hit_artist = hit["result"]["primary_artist"]["name"].strip(u'\u200b').lower() + # Genius uses zero-width characters to denote lowercase + # artist names. + hit_artist = hit["result"]["primary_artist"]["name"]. \ + strip(u'\u200b').lower() if hit_artist == artist.lower(): song_info = hit From 9465e933ba4d24dd409fe4de894da71808113179 Mon Sep 17 00:00:00 2001 From: ybnd Date: Wed, 5 Feb 2020 08:36:44 +0100 Subject: [PATCH 024/293] Add to changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 95fde6888..ffbcf3b97 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -151,6 +151,8 @@ Fixes: * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. Thanks to :user:`aereaux`. :bug:`3437` +* :doc:`/plugins/replaygain`: Support ``bs1770gain`` v0.6.0 and up + :bug:`3480` For plugin developers: From 7005691410549c65c680c66bd852a38ad725c981 Mon Sep 17 00:00:00 2001 From: ybnd Date: Wed, 5 Feb 2020 08:52:50 +0100 Subject: [PATCH 025/293] Add comment to clarify unexpected AttributeError handling --- beetsplug/replaygain.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3c652c7bd..f5a4a2c55 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -164,6 +164,9 @@ class Bs1770gainBackend(Backend): u'Is bs1770gain installed?' ) except AttributeError: + # Raised by ReplayGainLdnsCliMalformedTest.test_malformed_output + # in test_replaygain.py; bs1770gain backend runs even though + # the bs1770gain command is not found test.helper.has_program self.version = '0.0.0' if not self.command: raise FatalReplayGainError( From 63ea17365a5dc75a9decbba3899bb4975df3989c Mon Sep 17 00:00:00 2001 From: ybnd Date: Wed, 5 Feb 2020 09:03:45 +0100 Subject: [PATCH 026/293] Modify patched stdout in test_malformed_output --- beetsplug/replaygain.py | 7 +------ test/test_replaygain.py | 4 +++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index f5a4a2c55..646e3acce 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -156,18 +156,13 @@ class Bs1770gainBackend(Backend): version_out = call([cmd, '--version']) self.command = cmd self.version = re.search( - '([0-9]+.[0-9]+.[0-9]+), ', + 'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ', version_out.stdout.decode('utf-8') ).group(1) except OSError: raise FatalReplayGainError( u'Is bs1770gain installed?' ) - except AttributeError: - # Raised by ReplayGainLdnsCliMalformedTest.test_malformed_output - # in test_replaygain.py; bs1770gain backend runs even though - # the bs1770gain command is not found test.helper.has_program - self.version = '0.0.0' if not self.command: raise FatalReplayGainError( u'no replaygain command found: install bs1770gain' diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 3ac19f8f9..969f5c230 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -230,7 +230,9 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase): # Patch call to return nothing, bypassing the bs1770gain installation # check. - call_patch.return_value = CommandOutput(stdout=b"", stderr=b"") + call_patch.return_value = CommandOutput( + stdout=b'bs1770gain 0.0.0, ', stderr=b'' + ) try: self.load_plugins('replaygain') except Exception: From 964a6c2e63e56a9d04f1c34cc8b21511e0a11ba0 Mon Sep 17 00:00:00 2001 From: Tyler Faulk Date: Thu, 6 Feb 2020 12:10:38 -0500 Subject: [PATCH 027/293] restored whitespace to please style checker --- beetsplug/fetchart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ca93d685e..f78e6116a 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -175,6 +175,7 @@ def _logged_get(log, *args, **kwargs): log.debug('{}: {}', message, prepped.url) return s.send(prepped, **send_kwargs) + class RequestMixin(object): """Adds a Requests wrapper to the class that uses the logger, which must be named `self._log`. From 91be732bf467f9bc33a2455a6a2c5aff12db810d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 6 Feb 2020 22:18:15 -0500 Subject: [PATCH 028/293] Fix whitespace (#3453) --- beetsplug/fetchart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 408b947ca..ad96ece23 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -169,7 +169,7 @@ def _logged_get(log, *args, **kwargs): s.headers = {'User-Agent': 'beets'} prepped = s.prepare_request(req) settings = s.merge_environment_settings( - prepped.url, {}, None, None, None + prepped.url, {}, None, None, None ) send_kwargs.update(settings) log.debug('{}: {}', message, prepped.url) From d43d54e21cde97f57f19486925ab56b419254cc8 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 6 Feb 2020 22:22:54 -0500 Subject: [PATCH 029/293] Try to work around a Werkzeug change? --- beetsplug/web/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index f53fb3a95..21ff5d94e 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -169,7 +169,7 @@ class IdListConverter(BaseConverter): return ids def to_url(self, value): - return ','.join(value) + return ','.join(str(v) for v in value) class QueryConverter(PathConverter): From 9426ad73455657c616a116761e65debf898f8384 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 17 Feb 2020 09:40:30 +0100 Subject: [PATCH 030/293] Added beets-bpmanalyser to the list of 'other plugins' --- docs/plugins/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 7dc152bd1..955cc1405 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -299,6 +299,8 @@ Here are a few of the plugins written by the beets community: * `beets-mosaic`_ generates a montage of a mosiac from cover art. +* `beets-bpmanalyser`_ analyses songs and calculates tempo(bpm). + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -321,3 +323,4 @@ Here are a few of the plugins written by the beets community: .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic +.. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser From ddfb715e322c38cdf22019dacb96d81951e96ea7 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 17 Feb 2020 06:49:26 -0800 Subject: [PATCH 031/293] Style fixes for #3491 --- docs/plugins/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 955cc1405..125bdd934 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -299,7 +299,7 @@ Here are a few of the plugins written by the beets community: * `beets-mosaic`_ generates a montage of a mosiac from cover art. -* `beets-bpmanalyser`_ analyses songs and calculates tempo(bpm). +* `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check From 535af4bdb2ef7e9470798830e87a9ba4bad9217e Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Mon, 17 Feb 2020 07:40:38 -0800 Subject: [PATCH 032/293] parentwork: Only call store when the metadata has changed. Otherwise, this rewrites all your files every time. --- beetsplug/parentwork.py | 11 ++++++----- docs/changelog.rst | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index eaa8abb30..d40254696 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -89,10 +89,11 @@ class ParentWorkPlugin(BeetsPlugin): write = ui.should_write() for item in lib.items(ui.decargs(args)): - self.find_work(item, force_parent) - item.store() - if write: - item.try_write() + changed = self.find_work(item, force_parent) + if changed: + item.store() + if write: + item.try_write() command = ui.Subcommand( 'parentwork', help=u'fetche parent works, composers and dates') @@ -198,7 +199,7 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) if work_date: item['work_date'] = work_date - ui.show_model_changes( + return ui.show_model_changes( item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date']) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c91b422c..97cc7ca12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -158,6 +158,8 @@ Fixes: :bug:`3446` * :doc:`/plugins/replaygain`: Support ``bs1770gain`` v0.6.0 and up :bug:`3480` +* :doc:`/plugins/parentwork`: Don't save tracks when nothing has changed. + :bug:`3492` For plugin developers: From 86946ad4b7d03cd78d308456eec193b068078628 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sun, 15 Dec 2019 16:37:21 -0500 Subject: [PATCH 033/293] Allow the quality to be set for embedded/fetched cover art --- beets/art.py | 14 +++++++------- beets/util/artresizer.py | 11 ++++++----- beetsplug/convert.py | 2 +- beetsplug/embedart.py | 12 +++++++----- beetsplug/fetchart.py | 2 +- beetsplug/thumbnails.py | 2 +- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/beets/art.py b/beets/art.py index e7a087a05..e48be4a57 100644 --- a/beets/art.py +++ b/beets/art.py @@ -50,7 +50,7 @@ def get_art(log, item): return mf.art -def embed_item(log, item, imagepath, maxwidth=None, itempath=None, +def embed_item(log, item, imagepath, maxwidth=None, quality=75, itempath=None, compare_threshold=0, ifempty=False, as_album=False, id3v23=None): """Embed an image into the item's media file. @@ -64,7 +64,7 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, log.info(u'media file already contained art') return if maxwidth and not as_album: - imagepath = resize_image(log, imagepath, maxwidth) + imagepath = resize_image(log, imagepath, maxwidth, quality) # Get the `Image` object from the file. try: @@ -84,7 +84,7 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23) -def embed_album(log, album, maxwidth=None, quiet=False, +def embed_album(log, album, maxwidth=None, quality=75, quiet=False, compare_threshold=0, ifempty=False): """Embed album art into all of the album's items. """ @@ -97,20 +97,20 @@ def embed_album(log, album, maxwidth=None, quiet=False, displayable_path(imagepath), album) return if maxwidth: - imagepath = resize_image(log, imagepath, maxwidth) + imagepath = resize_image(log, imagepath, maxwidth, quality) log.info(u'Embedding album art into {0}', album) for item in album.items(): - embed_item(log, item, imagepath, maxwidth, None, + embed_item(log, item, imagepath, maxwidth, quality, None, compare_threshold, ifempty, as_album=True) -def resize_image(log, imagepath, maxwidth): +def resize_image(log, imagepath, maxwidth, quality): """Returns path to an image resized to maxwidth. """ log.debug(u'Resizing album art to {0} pixels wide', maxwidth) - imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath)) + imagepath = ArtResizer.shared.resize(maxwidth, quality, syspath(imagepath)) return imagepath diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 99e28c0cc..3a6f58752 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -59,7 +59,7 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize(maxwidth, path_in, path_out=None): +def pil_resize(maxwidth, quality, path_in, path_out=None): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -72,7 +72,7 @@ def pil_resize(maxwidth, path_in, path_out=None): im = Image.open(util.syspath(path_in)) size = maxwidth, maxwidth im.thumbnail(size, Image.ANTIALIAS) - im.save(util.py3_path(path_out)) + im.save(util.py3_path(path_out), quality=quality) return path_out except IOError: log.error(u"PIL cannot create thumbnail for '{0}'", @@ -80,7 +80,7 @@ def pil_resize(maxwidth, path_in, path_out=None): return path_in -def im_resize(maxwidth, path_in, path_out=None): +def im_resize(maxwidth, quality, path_in, path_out=None): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -96,6 +96,7 @@ def im_resize(maxwidth, path_in, path_out=None): cmd = ArtResizer.shared.im_convert_cmd + \ [util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), + '-quality', '{0}x'.format(quality), util.syspath(path_out, prefix=False)] try: @@ -190,14 +191,14 @@ class ArtResizer(six.with_metaclass(Shareable, object)): self.im_convert_cmd = ['magick'] self.im_identify_cmd = ['magick', 'identify'] - def resize(self, maxwidth, path_in, path_out=None): + def resize(self, maxwidth, quality, path_in, path_out=None): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a temporary file. For WEBPROXY, returns `path_in` unmodified. """ if self.local: func = BACKEND_FUNCS[self.method[0]] - return func(maxwidth, path_in, path_out) + return func(maxwidth, quality, path_in, path_out) else: return path_in diff --git a/beetsplug/convert.py b/beetsplug/convert.py index e7ac4f3ac..ab86800f6 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -422,7 +422,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(album.artpath), util.displayable_path(dest)) if not pretend: - ArtResizer.shared.resize(maxwidth, album.artpath, dest) + ArtResizer.shared.resize(maxwidth, 75, album.artpath, dest) else: if pretend: msg = 'ln' if hardlink else ('ln -s' if link else 'cp') diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 71681f024..e581a2ddf 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -59,7 +59,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): 'auto': True, 'compare_threshold': 0, 'ifempty': False, - 'remove_art_file': False + 'remove_art_file': False, + 'quality': 95, }) if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: @@ -86,6 +87,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): u"-y", u"--yes", action="store_true", help=u"skip confirmation" ) maxwidth = self.config['maxwidth'].get(int) + quality = self.config['quality'].get(int) compare_threshold = self.config['compare_threshold'].get(int) ifempty = self.config['ifempty'].get(bool) @@ -104,8 +106,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): return for item in items: - art.embed_item(self._log, item, imagepath, maxwidth, None, - compare_threshold, ifempty) + art.embed_item(self._log, item, imagepath, maxwidth, + quality, None, compare_threshold, ifempty) else: albums = lib.albums(decargs(args)) @@ -114,8 +116,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): return for album in albums: - art.embed_album(self._log, album, maxwidth, False, - compare_threshold, ifempty) + art.embed_album(self._log, album, maxwidth, quality, + False, compare_threshold, ifempty) self.remove_artfile(album) embed_cmd.func = embed_func diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad96ece23..99c592991 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -135,7 +135,7 @@ class Candidate(object): def resize(self, plugin): if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path) + self.path = ArtResizer.shared.resize(plugin.maxwidth, 75, self.path) def _logged_get(log, *args, **kwargs): diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index fe36fbd13..de949ea32 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -152,7 +152,7 @@ class ThumbnailsPlugin(BeetsPlugin): self._log.debug(u"{1}x{1} thumbnail for {0} exists and is " u"recent enough", album, size) return False - resized = ArtResizer.shared.resize(size, album.artpath, + resized = ArtResizer.shared.resize(size, 75, album.artpath, util.syspath(target)) self.add_tags(album, util.syspath(resized)) shutil.move(resized, target) From a30c90e6152ed13293267e5246119398e676fcad Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Mon, 17 Feb 2020 22:22:47 -0500 Subject: [PATCH 034/293] Fix one of the failing tests --- test/test_thumbnails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index dc03f06f7..0cf22d4a1 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -154,7 +154,7 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): any_order=True) resize = mock_artresizer.shared.resize - resize.assert_called_once_with(12345, path_to_art, md5_file) + resize.assert_called_once_with(12345, 75, path_to_art, md5_file) plugin.add_tags.assert_called_once_with(album, resize.return_value) mock_shutils.move.assert_called_once_with(resize.return_value, md5_file) From 036202e1c5a0cae19397affa6eeace1cfed5ea6c Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Tue, 18 Feb 2020 14:50:57 -0500 Subject: [PATCH 035/293] Default quality to 0 which means don't specify From the ImageMagick docs: "The default is to use the estimated quality of your input image if it can be determined, otherwise 92." In order to get the original behaviour we need to conditional add the quality parameter to the `magick` call. The quality range can be anything from 1 to 100, which gives us the convenience of using 0 to specify no specific quality level. --- beets/art.py | 17 +++++++++-------- beets/util/artresizer.py | 22 +++++++++++++--------- beetsplug/convert.py | 2 +- beetsplug/embedart.py | 10 ++++++---- test/test_thumbnails.py | 2 +- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/beets/art.py b/beets/art.py index e48be4a57..b8b172594 100644 --- a/beets/art.py +++ b/beets/art.py @@ -50,9 +50,9 @@ def get_art(log, item): return mf.art -def embed_item(log, item, imagepath, maxwidth=None, quality=75, itempath=None, - compare_threshold=0, ifempty=False, as_album=False, - id3v23=None): +def embed_item(log, item, imagepath, maxwidth=None, itempath=None, + compare_threshold=0, ifempty=False, as_album=False, id3v23=None, + quality=0): """Embed an image into the item's media file. """ # Conditions and filters. @@ -84,8 +84,8 @@ def embed_item(log, item, imagepath, maxwidth=None, quality=75, itempath=None, item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23) -def embed_album(log, album, maxwidth=None, quality=75, quiet=False, - compare_threshold=0, ifempty=False): +def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0, + ifempty=False, quality=0): """Embed album art into all of the album's items. """ imagepath = album.artpath @@ -102,15 +102,16 @@ def embed_album(log, album, maxwidth=None, quality=75, quiet=False, log.info(u'Embedding album art into {0}', album) for item in album.items(): - embed_item(log, item, imagepath, maxwidth, quality, None, - compare_threshold, ifempty, as_album=True) + embed_item(log, item, imagepath, maxwidth, None, compare_threshold, + ifempty, as_album=True, quality=quality) def resize_image(log, imagepath, maxwidth, quality): """Returns path to an image resized to maxwidth. """ log.debug(u'Resizing album art to {0} pixels wide', maxwidth) - imagepath = ArtResizer.shared.resize(maxwidth, quality, syspath(imagepath)) + imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath), + quality=quality) return imagepath diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 3a6f58752..09d138322 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -59,7 +59,7 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize(maxwidth, quality, path_in, path_out=None): +def pil_resize(maxwidth, path_in, path_out=None, quality=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -80,7 +80,7 @@ def pil_resize(maxwidth, quality, path_in, path_out=None): return path_in -def im_resize(maxwidth, quality, path_in, path_out=None): +def im_resize(maxwidth, path_in, path_out=None, quality=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -93,11 +93,15 @@ def im_resize(maxwidth, quality, path_in, path_out=None): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. - cmd = ArtResizer.shared.im_convert_cmd + \ - [util.syspath(path_in, prefix=False), - '-resize', '{0}x>'.format(maxwidth), - '-quality', '{0}x'.format(quality), - util.syspath(path_out, prefix=False)] + cmd = ArtResizer.shared.im_convert_cmd + [ + util.syspath(path_in, prefix=False), + '-resize', '{0}x>'.format(maxwidth), + ] + + if quality > 0: + cmd += ['-quality', '{0}'.format(quality)] + + cmd.append(util.syspath(path_out, prefix=False)) try: util.command_output(cmd) @@ -191,14 +195,14 @@ class ArtResizer(six.with_metaclass(Shareable, object)): self.im_convert_cmd = ['magick'] self.im_identify_cmd = ['magick', 'identify'] - def resize(self, maxwidth, quality, path_in, path_out=None): + def resize(self, maxwidth, path_in, path_out=None, quality=0): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a temporary file. For WEBPROXY, returns `path_in` unmodified. """ if self.local: func = BACKEND_FUNCS[self.method[0]] - return func(maxwidth, quality, path_in, path_out) + return func(maxwidth, path_in, path_out, quality=quality) else: return path_in diff --git a/beetsplug/convert.py b/beetsplug/convert.py index ab86800f6..e7ac4f3ac 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -422,7 +422,7 @@ class ConvertPlugin(BeetsPlugin): util.displayable_path(album.artpath), util.displayable_path(dest)) if not pretend: - ArtResizer.shared.resize(maxwidth, 75, album.artpath, dest) + ArtResizer.shared.resize(maxwidth, album.artpath, dest) else: if pretend: msg = 'ln' if hardlink else ('ln -s' if link else 'cp') diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index e581a2ddf..61a4d798f 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -60,7 +60,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): 'compare_threshold': 0, 'ifempty': False, 'remove_art_file': False, - 'quality': 95, + 'quality': 0, }) if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: @@ -107,7 +107,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): for item in items: art.embed_item(self._log, item, imagepath, maxwidth, - quality, None, compare_threshold, ifempty) + None, compare_threshold, ifempty, + quality=quality) else: albums = lib.albums(decargs(args)) @@ -116,8 +117,9 @@ class EmbedCoverArtPlugin(BeetsPlugin): return for album in albums: - art.embed_album(self._log, album, maxwidth, quality, - False, compare_threshold, ifempty) + art.embed_album(self._log, album, maxwidth, + False, compare_threshold, ifempty, + quality=quality) self.remove_artfile(album) embed_cmd.func = embed_func diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index 0cf22d4a1..dc03f06f7 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -154,7 +154,7 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): any_order=True) resize = mock_artresizer.shared.resize - resize.assert_called_once_with(12345, 75, path_to_art, md5_file) + resize.assert_called_once_with(12345, path_to_art, md5_file) plugin.add_tags.assert_called_once_with(album, resize.return_value) mock_shutils.move.assert_called_once_with(resize.return_value, md5_file) From 96b0e8a33ef0e9f38a4d6264cb4355434e7b17e5 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Tue, 18 Feb 2020 16:44:45 -0500 Subject: [PATCH 036/293] No longer need to pass a default quality here --- beetsplug/fetchart.py | 2 +- beetsplug/thumbnails.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 99c592991..ad96ece23 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -135,7 +135,7 @@ class Candidate(object): def resize(self, plugin): if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, 75, self.path) + self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path) def _logged_get(log, *args, **kwargs): diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index de949ea32..fe36fbd13 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -152,7 +152,7 @@ class ThumbnailsPlugin(BeetsPlugin): self._log.debug(u"{1}x{1} thumbnail for {0} exists and is " u"recent enough", album, size) return False - resized = ArtResizer.shared.resize(size, 75, album.artpath, + resized = ArtResizer.shared.resize(size, album.artpath, util.syspath(target)) self.add_tags(album, util.syspath(resized)) shutil.move(resized, target) From c9e6f910308a4fa46b74755cb0ba2e373e117df2 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Thu, 20 Feb 2020 18:55:56 +0100 Subject: [PATCH 037/293] Added beets-goingrunning to the list of 'other plugins' --- docs/plugins/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 125bdd934..5d6f4795e 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -301,6 +301,8 @@ Here are a few of the plugins written by the beets community: * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). +* `beets-goingrunning`_ copies songs to external device based on their tempo (BPM) and length to go with your running session. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -324,3 +326,4 @@ Here are a few of the plugins written by the beets community: .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser +.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ \ No newline at end of file From 37f3e226b0fd8b044a8fff73a848a40107595e62 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Thu, 20 Feb 2020 18:58:11 +0100 Subject: [PATCH 038/293] beets-goingrunning - shortened description --- docs/plugins/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 5d6f4795e..383466a68 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -301,7 +301,7 @@ Here are a few of the plugins written by the beets community: * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). -* `beets-goingrunning`_ copies songs to external device based on their tempo (BPM) and length to go with your running session. +* `beets-goingrunning`_ copies songs to external device to go with your running session. .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check From bc53695f24d06bcc33df2d0a4e6e6a739cab27fb Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sat, 22 Feb 2020 01:56:57 +0100 Subject: [PATCH 039/293] added missing test requirement --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 544721937..c50e65bf5 100755 --- a/setup.py +++ b/setup.py @@ -118,7 +118,8 @@ setup( 'responses', 'pyxdg', 'python-mpd2', - 'discogs-client' + 'discogs-client', + 'requests_oauthlib' ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] From 15ad525e0d904dab48d2575f5c2438b3f7d1276f Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:02:10 -0500 Subject: [PATCH 040/293] Mention quality level in docstrings --- beets/art.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/art.py b/beets/art.py index b8b172594..0cb9cdf1b 100644 --- a/beets/art.py +++ b/beets/art.py @@ -107,9 +107,11 @@ def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0, def resize_image(log, imagepath, maxwidth, quality): - """Returns path to an image resized to maxwidth. + """Returns path to an image resized to maxwidth and encoded with the + specified quality level. """ - log.debug(u'Resizing album art to {0} pixels wide', maxwidth) + log.debug(u'Resizing album art to {0} pixels wide and encoding at quality + level {1}', maxwidth, quality) imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath), quality=quality) return imagepath From d82573f8f3231863cb892d222250bd603a0fe382 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:02:20 -0500 Subject: [PATCH 041/293] Document quality setting --- docs/plugins/embedart.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index cc2fe6fc8..bd87abed6 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -58,6 +58,11 @@ file. The available options are: the aspect ratio is preserved. See also :ref:`image-resizing` for further caveats about image resizing. Default: 0 (disabled). +- **quality**: The quality level to use when encoding the image file when + downscaling to ``maxwidth``. The default behaviour depends on the method used + to scale the images. ImageMagick tries to estimate the input image quality and + uses 92 if it cannot be determined. Pillow defaults to 75. + Default: 0 (disabled) - **remove_art_file**: Automatically remove the album art file for the album after it has been embedded. This option is best used alongside the :doc:`FetchArt ` plugin to download art with the purpose of From 4e7fd47a3574e567482bc2fb7e9a3bb51215fc12 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:05:02 -0500 Subject: [PATCH 042/293] Add quality to art resizer docstring --- beets/util/artresizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 09d138322..cf457e797 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -198,7 +198,8 @@ class ArtResizer(six.with_metaclass(Shareable, object)): def resize(self, maxwidth, path_in, path_out=None, quality=0): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a - temporary file. For WEBPROXY, returns `path_in` unmodified. + temporary file and encodes with the specified quality level. + For WEBPROXY, returns `path_in` unmodified. """ if self.local: func = BACKEND_FUNCS[self.method[0]] From a729bd872959232dad22591bfa497b40f0974872 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:23:02 -0500 Subject: [PATCH 043/293] Send quality parameter to images.weserv.nl --- beets/util/artresizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index cf457e797..aee06a5b6 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -40,14 +40,19 @@ else: log = logging.getLogger('beets') -def resize_url(url, maxwidth): +def resize_url(url, maxwidth, quality=0): """Return a proxied image URL that resizes the original image to maxwidth (preserving aspect ratio). """ - return '{0}?{1}'.format(PROXY_URL, urlencode({ + params = { 'url': url.replace('http://', ''), 'w': maxwidth, - })) + } + + if quality > 0: + params['q'] = quality + + return '{0}?{1}'.format(PROXY_URL, urlencode(params)) def temp_file_for(path): @@ -215,7 +220,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)): if self.local: return url else: - return resize_url(url, maxwidth) + return resize_url(url, maxwidth, quality) @property def local(self): From 6a84949020d0b05be86c0da56a13c67222c6f3ee Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 12:43:22 -0500 Subject: [PATCH 044/293] Woops, this needs to be explicitly multiline --- beets/art.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/art.py b/beets/art.py index 0cb9cdf1b..20b0e96d2 100644 --- a/beets/art.py +++ b/beets/art.py @@ -110,7 +110,7 @@ def resize_image(log, imagepath, maxwidth, quality): """Returns path to an image resized to maxwidth and encoded with the specified quality level. """ - log.debug(u'Resizing album art to {0} pixels wide and encoding at quality + log.debug(u'Resizing album art to {0} pixels wide and encoding at quality \ level {1}', maxwidth, quality) imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath), quality=quality) From 0af1bf5fbbe9be452cd18cb6ad34d456739623cc Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 13:00:21 -0500 Subject: [PATCH 045/293] Pass a default quality in here --- beets/util/artresizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index aee06a5b6..8f14c8baf 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -212,7 +212,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)): else: return path_in - def proxy_url(self, maxwidth, url): + def proxy_url(self, maxwidth, url, quality=0): """Modifies an image URL according the method, returning a new URL. For WEBPROXY, a URL on the proxy server is returned. Otherwise, the URL is returned unmodified. From fe8ba17ced235df664d66b644c4bf5f05a816288 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Sat, 22 Feb 2020 13:36:35 -0500 Subject: [PATCH 046/293] Add quality setting to fetchart plugin --- beetsplug/fetchart.py | 12 ++++++++---- docs/plugins/fetchart.rst | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad96ece23..2fe8b0b2c 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -135,7 +135,8 @@ class Candidate(object): def resize(self, plugin): if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path) + self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path, + quality=plugin.quality) def _logged_get(log, *args, **kwargs): @@ -777,6 +778,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'auto': True, 'minwidth': 0, 'maxwidth': 0, + 'quality': 0, 'enforce_ratio': False, 'cautious': False, 'cover_names': ['cover', 'front', 'art', 'album', 'folder'], @@ -793,6 +795,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.minwidth = self.config['minwidth'].get(int) self.maxwidth = self.config['maxwidth'].get(int) + self.quality = self.config['quality'].get(int) # allow both pixel and percentage-based margin specifications self.enforce_ratio = self.config['enforce_ratio'].get( @@ -922,9 +925,10 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): def art_for_album(self, album, paths, local_only=False): """Given an Album object, returns a path to downloaded art for the album (or None if no art is found). If `maxwidth`, then images are - resized to this maximum pixel size. If `local_only`, then only local - image files from the filesystem are returned; no network requests - are made. + resized to this maximum pixel size. If `quality` then resized images + are saved at the specified quality level. If `local_only`, then only + local image files from the filesystem are returned; no network + requests are made. """ out = None diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 68212a582..23362aee0 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -42,6 +42,9 @@ file. The available options are: - **maxwidth**: A maximum image width to downscale fetched images if they are too big. The resize operation reduces image width to at most ``maxwidth`` pixels. The height is recomputed so that the aspect ratio is preserved. +- **quality**: The quality level to use when encoding the image file when + downscaling to ``maxwidth``. + Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. It is also possible to specify a certain deviation to the exact ratio to From d2e32a6b2056ec4237dc327c38194a50c4b27893 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 23 Feb 2020 23:29:15 +0100 Subject: [PATCH 047/293] Raising error on missing configuration file inclusion --- beets/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/__init__.py b/beets/__init__.py index 20075073c..20aed95e3 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -35,6 +35,8 @@ class IncludeLazyConfig(confuse.LazyConfig): filename = view.as_filename() if os.path.isfile(filename): self.set_file(filename) + else: + raise FileNotFoundError("Warning! Configuration file({0}) does not exist!".format(filename)) except confuse.NotFoundError: pass From dfa45f62a59b7bc431af2084fe35d84c2c2d08bd Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 00:07:24 +0100 Subject: [PATCH 048/293] fixed flake8 long line warning --- beets/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/__init__.py b/beets/__init__.py index 20aed95e3..c9863ee66 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -36,7 +36,8 @@ class IncludeLazyConfig(confuse.LazyConfig): if os.path.isfile(filename): self.set_file(filename) else: - raise FileNotFoundError("Warning! Configuration file({0}) does not exist!".format(filename)) + raise FileNotFoundError( + "Warning! Configuration file({0}) does not exist!".format(filename)) except confuse.NotFoundError: pass From c90f7aacfcb65eef3386c29d3c146f9f0a120ee1 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 00:22:34 +0100 Subject: [PATCH 049/293] fixed flake8 long line warning (maybe) --- beets/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index c9863ee66..ed3af5588 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -36,8 +36,8 @@ class IncludeLazyConfig(confuse.LazyConfig): if os.path.isfile(filename): self.set_file(filename) else: - raise FileNotFoundError( - "Warning! Configuration file({0}) does not exist!".format(filename)) + raise FileNotFoundError("Warning! Configuration file({0}) " + "does not exist!".format(filename)) except confuse.NotFoundError: pass From 3db55c7bf4f25c59102b4884165d2956ae80cfba Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 10:20:54 +0100 Subject: [PATCH 050/293] Simple warning on missing (included) configuration file. --- beets/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index ed3af5588..0b1048bdb 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -33,13 +33,12 @@ class IncludeLazyConfig(confuse.LazyConfig): try: for view in self['include']: filename = view.as_filename() - if os.path.isfile(filename): - self.set_file(filename) - else: - raise FileNotFoundError("Warning! Configuration file({0}) " - "does not exist!".format(filename)) + self.set_file(filename) except confuse.NotFoundError: pass + except confuse.ConfigReadError as err: + print("Warning! Missing configuration file! {}".format(err.reason)) + pass config = IncludeLazyConfig('beets', __name__) From 253d4c76d00f6fcf0e7cd6b30934aef0cf44212a Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 10:43:03 +0100 Subject: [PATCH 051/293] removed redundant import and redundant filename variable --- beets/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index 0b1048bdb..bed59d05f 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -15,8 +15,6 @@ from __future__ import division, absolute_import, print_function -import os - import confuse __version__ = u'1.5.0' @@ -32,8 +30,7 @@ class IncludeLazyConfig(confuse.LazyConfig): try: for view in self['include']: - filename = view.as_filename() - self.set_file(filename) + self.set_file(view.as_filename()) except confuse.NotFoundError: pass except confuse.ConfigReadError as err: From 131227eff44b0fa596abfcd7780b6ea81cc50829 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 24 Feb 2020 22:38:30 +0100 Subject: [PATCH 052/293] writing warning message to stderr --- beets/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/__init__.py b/beets/__init__.py index bed59d05f..e9cc10e08 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -16,6 +16,7 @@ from __future__ import division, absolute_import, print_function import confuse +from sys import stderr __version__ = u'1.5.0' __author__ = u'Adrian Sampson ' @@ -34,7 +35,8 @@ class IncludeLazyConfig(confuse.LazyConfig): except confuse.NotFoundError: pass except confuse.ConfigReadError as err: - print("Warning! Missing configuration file! {}".format(err.reason)) + stderr.write("Configuration 'import' failed: {}" + .format(err.reason)) pass From d06665413c5b871c9375c28f5544eb80e6cc0d1c Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Tue, 25 Feb 2020 15:23:52 +0100 Subject: [PATCH 053/293] minor fixes and changelog entry --- beets/__init__.py | 3 +-- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index e9cc10e08..e3e5fdf83 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -35,9 +35,8 @@ class IncludeLazyConfig(confuse.LazyConfig): except confuse.NotFoundError: pass except confuse.ConfigReadError as err: - stderr.write("Configuration 'import' failed: {}" + stderr.write("configuration `import` failed: {}" .format(err.reason)) - pass config = IncludeLazyConfig('beets', __name__) diff --git a/docs/changelog.rst b/docs/changelog.rst index 97cc7ca12..9a4ad0988 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -160,6 +160,8 @@ Fixes: :bug:`3480` * :doc:`/plugins/parentwork`: Don't save tracks when nothing has changed. :bug:`3492` +* Added a warning when configuration files defined in the `include` directive + of the configuration file fail to be imported. For plugin developers: From 20d28948a3cb51c0a4993d913bc716dc1a476e4a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 25 Feb 2020 08:56:06 -0800 Subject: [PATCH 054/293] Changelog bug link for #3498 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a4ad0988..81ba590f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -162,6 +162,7 @@ Fixes: :bug:`3492` * Added a warning when configuration files defined in the `include` directive of the configuration file fail to be imported. + :bug:`3498` For plugin developers: From a9e11fcfebbb80f3a0fe90664856a3398c7b5b6a Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Wed, 26 Feb 2020 19:23:59 -0500 Subject: [PATCH 055/293] Add some info about valid quality levels --- docs/plugins/embedart.rst | 9 ++++++--- docs/plugins/fetchart.rst | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index bd87abed6..ece5e0350 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -59,9 +59,12 @@ file. The available options are: caveats about image resizing. Default: 0 (disabled). - **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. The default behaviour depends on the method used - to scale the images. ImageMagick tries to estimate the input image quality and - uses 92 if it cannot be determined. Pillow defaults to 75. + downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. + Higher numbers result in better image quality while lower numbers will result + in smaller files. 65-75 is a good starting point. The default behaviour + depends on the method used to scale the images. ImageMagick tries to estimate + the input image quality and uses 92 if it cannot be determined. Pillow + defaults to 75. Default: 0 (disabled) - **remove_art_file**: Automatically remove the album art file for the album after it has been embedded. This option is best used alongside the diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 23362aee0..8c5d0eb7b 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -43,7 +43,9 @@ file. The available options are: too big. The resize operation reduces image width to at most ``maxwidth`` pixels. The height is recomputed so that the aspect ratio is preserved. - **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. + downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. + Higher numbers result in better image quality while lower numbers will result + in smaller files. 65-75 is a good starting point. Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. From b3fec61f5409b05fa2c31d19f5160e1526332fd4 Mon Sep 17 00:00:00 2001 From: Daniel Barber Date: Wed, 26 Feb 2020 19:31:42 -0500 Subject: [PATCH 056/293] Add details to changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 97cc7ca12..dc09a404f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,9 @@ Changelog New features: +* :doc:`plugins/fetchart`: and :doc:`plugins/embedart`: Added a new ``quality`` + option that controls the quality of the image output when the image is + resized. * :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ Thanks to :user:`BrainDamage`. * :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to From 00c6d1439ee29b3b4592d6f65cfa57b1d1f96575 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 26 Feb 2020 19:15:44 -0800 Subject: [PATCH 057/293] Docs tweaks for #3493 --- docs/changelog.rst | 2 +- docs/plugins/embedart.rst | 13 ++++++------- docs/plugins/fetchart.rst | 10 ++++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b16593f1d..7ef19871b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog New features: -* :doc:`plugins/fetchart`: and :doc:`plugins/embedart`: Added a new ``quality`` +* :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is resized. * :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ diff --git a/docs/plugins/embedart.rst b/docs/plugins/embedart.rst index ece5e0350..defd3fa4b 100644 --- a/docs/plugins/embedart.rst +++ b/docs/plugins/embedart.rst @@ -58,13 +58,12 @@ file. The available options are: the aspect ratio is preserved. See also :ref:`image-resizing` for further caveats about image resizing. Default: 0 (disabled). -- **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. - Higher numbers result in better image quality while lower numbers will result - in smaller files. 65-75 is a good starting point. The default behaviour - depends on the method used to scale the images. ImageMagick tries to estimate - the input image quality and uses 92 if it cannot be determined. Pillow - defaults to 75. +- **quality**: The JPEG quality level to use when compressing images (when + ``maxwidth`` is set). This should be either a number from 1 to 100 or 0 to + use the default quality. 65–75 is usually a good starting point. The default + behavior depends on the imaging tool used for scaling: ImageMagick tries to + estimate the input image quality and uses 92 if it cannot be determined, and + PIL defaults to 75. Default: 0 (disabled) - **remove_art_file**: Automatically remove the album art file for the album after it has been embedded. This option is best used alongside the diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 8c5d0eb7b..4441d4e30 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -42,10 +42,12 @@ file. The available options are: - **maxwidth**: A maximum image width to downscale fetched images if they are too big. The resize operation reduces image width to at most ``maxwidth`` pixels. The height is recomputed so that the aspect ratio is preserved. -- **quality**: The quality level to use when encoding the image file when - downscaling to ``maxwidth``. Can be a number from 1-100 or 0 to disable. - Higher numbers result in better image quality while lower numbers will result - in smaller files. 65-75 is a good starting point. +- **quality**: The JPEG quality level to use when compressing images (when + ``maxwidth`` is set). This should be either a number from 1 to 100 or 0 to + use the default quality. 65–75 is usually a good starting point. The default + behavior depends on the imaging tool used for scaling: ImageMagick tries to + estimate the input image quality and uses 92 if it cannot be determined, and + PIL defaults to 75. Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. From 594da1597a392a4917dab407198211bd23e8c2b2 Mon Sep 17 00:00:00 2001 From: smichel17 Date: Sun, 1 Mar 2020 11:45:14 -0500 Subject: [PATCH 058/293] Document running chroma plugin in verbose mode For #941 --- docs/plugins/chroma.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index 1b86073b8..58a51da2a 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -96,7 +96,9 @@ Usage Once you have all the dependencies sorted out, enable the ``chroma`` plugin in your configuration (see :ref:`using-plugins`) to benefit from fingerprinting -the next time you run ``beet import``. +the next time you run ``beet import``. The first time you do this, you may wish +to run in verbose mode (``beet -v import``) in order to verify that the +``chroma`` plugin is operational, since it does not show an indicator otherwise. You can also use the ``beet fingerprint`` command to generate fingerprints for items already in your library. (Provide a query to fingerprint a subset of your From fcc1951e7a104f4bc363070264a58278fa2b882d Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 00:16:16 +0100 Subject: [PATCH 059/293] corrected typo whilst reading --- beets/dbcore/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 3195b52c9..b13f2638a 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -189,7 +189,7 @@ class LazyConvertDict(object): class Model(object): """An abstract object representing an object in the database. Model - objects act like dictionaries (i.e., the allow subscript access like + objects act like dictionaries (i.e., they allow subscript access like ``obj['field']``). The same field set is available via attribute access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are available: From 545c65d903e38d37fd2c1734ec69eac609bea035 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 1 Mar 2020 19:35:23 -0500 Subject: [PATCH 060/293] Massage wording for #3504 --- docs/plugins/chroma.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index 58a51da2a..a6b60e6d8 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -96,9 +96,9 @@ Usage Once you have all the dependencies sorted out, enable the ``chroma`` plugin in your configuration (see :ref:`using-plugins`) to benefit from fingerprinting -the next time you run ``beet import``. The first time you do this, you may wish -to run in verbose mode (``beet -v import``) in order to verify that the -``chroma`` plugin is operational, since it does not show an indicator otherwise. +the next time you run ``beet import``. (The plugin doesn't produce any obvious +output by default. If you want to confirm that it's enabled, you can try +running in verbose mode once with ``beet -v import``.) You can also use the ``beet fingerprint`` command to generate fingerprints for items already in your library. (Provide a query to fingerprint a subset of your From d2d2b646c1bf26629663035b000b1a85ce949c56 Mon Sep 17 00:00:00 2001 From: jmwatte Date: Tue, 17 Nov 2015 12:37:15 +0100 Subject: [PATCH 061/293] Add plugin for Fish shell tab completion --- beetsplug/fish.py | 274 ++++++++++++++++++++++++++++++++++++++++++ docs/plugins/fish.rst | 43 +++++++ 2 files changed, 317 insertions(+) create mode 100644 beetsplug/fish.py create mode 100644 docs/plugins/fish.rst diff --git a/beetsplug/fish.py b/beetsplug/fish.py new file mode 100644 index 000000000..3a7682c2a --- /dev/null +++ b/beetsplug/fish.py @@ -0,0 +1,274 @@ +# This file is part of beets. +# Copyright 2015, winters jean-marie. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""If you use the fish-shell http://fishshell.com/ ... this will do +autocomplete for you. It does the main commands and options for beet +and the plugins. +It gives you all the album and itemfields (like genre, album) but not all the +values for these. It suggest genre: or album: but not genre: Pop..Jazz...Rock +You can get that by specifying ex. --extravalues genre. +""" + +from __future__ import (division, absolute_import, print_function, + unicode_literals) + +from beets.plugins import BeetsPlugin +from beets import library, ui +from beets.ui import commands +from operator import attrgetter +import os +BL_NEED2 = """complete -c beet -n '__fish_beet_needs_command' {} {}\n""" +BL_USE3 = """complete -c beet -n '__fish_beet_using_command {}' {} {}\n""" +BL_SUBS = """complete -c beet -n '__fish_at_level {} ""' {} {}\n""" +BL_EXTRA3 = """complete -c beet -n '__fish_beet_use_extra {}' {} {}\n""" + +HEAD = ''' +function __fish_beet_needs_command + set cmd (commandline -opc) + if test (count $cmd) -eq 1 + return 0 + end + return 1 +end + +function __fish_beet_using_command + set cmd (commandline -opc) + set needle (count $cmd) + if test $needle -gt 1 + if begin test $argv[1] = $cmd[2]; + and not contains -- $cmd[$needle] $FIELDS; end + return 0 + end + end + return 1 +end + +function __fish_beet_use_extra + set cmd (commandline -opc) + set needle (count $cmd) + if test $argv[2] = $cmd[$needle] + return 0 + end + return 1 +end +''' + + +class FishPlugin(BeetsPlugin): + + def commands(self): + cmd = ui.Subcommand('fish', help='make fish autocomplete beet') + cmd.func = self.run + cmd.parser.add_option('-f', '--noFields', action='store_true', + default=False, + help='no item/album fields for autocomplete') + cmd.parser.add_option( + '-e', + '--extravalues', + action='append', + type='choice', + choices=library.Item.all_keys() + + library.Album.all_keys(), + help='pick field, get field-values for autocomplete') + return [cmd] + + def run(self, lib, opts, args): + # we gather the commands from beet and from the plugins. + # we take the album and item fields. + # it wanted, we take the values from these fields. + # we make a giant string of tehm formatted in a way that + # allows fish to do autocompletion for beet. + homeDir = os.path.expanduser("~") + completePath = os.path.join(homeDir, '.config/fish/completions') + try: + os.makedirs(completePath) + except OSError: + if not os.path.isdir(completePath): + raise + pathAndFile = os.path.join(completePath, 'beet.fish') + nobasicfields = opts.noFields # do not complete for item/album fields + extravalues = opts.extravalues # ex complete all artist values + beetcmds = sorted( + (commands.default_commands + + commands.plugins.commands()), + key=attrgetter('name')) + fields = sorted(set( + library.Album.all_keys() + library.Item.all_keys())) + # collect cmds and their aliases and their help message + cmd_names_help = [] + for cmd in beetcmds: + names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names.append(cmd.name) + for name in names: + cmd_names_help.append((name, cmd.help)) + # here we go assembling the string + totstring = HEAD + "\n" + totstring += get_cmds_list([name[0] for name in cmd_names_help]) + totstring += '' if nobasicfields else get_standard_fields(fields) + totstring += get_extravalues(lib, extravalues) if extravalues else '' + totstring += "\n" + "# ====== {} =====".format( + "setup basic beet completion") + "\n" * 2 + totstring += get_basic_beet_options() + totstring += "\n" + "# ====== {} =====".format( + "setup field completion for subcommands") + "\n" + totstring += get_subcommands( + cmd_names_help, nobasicfields, extravalues) + # setup completion for all the command-options + totstring += get_all_commands(beetcmds) + + with open(pathAndFile, 'w') as fish_file: + fish_file.write(totstring) + + +def get_cmds_list(cmds_names): + # make list of all commands in beet&plugins + substr = '' + substr += ( + "set CMDS " + " ".join(cmds_names) + ("\n" * 2) + ) + return substr + + +def get_standard_fields(fields): + # make list of item/album fields & append with ':' + fields = (field + ":" for field in fields) + substr = '' + substr += ( + "set FIELDS " + " ".join(fields) + ("\n" * 2) + ) + return substr + + +def get_extravalues(lib, extravalues): + # make list of all values from a item/album field + # so type artist: and get completion for stones, beatles .. + word = '' + setOfValues = get_set_of_values_for_field(lib, extravalues) + for fld in extravalues: + extraname = fld.upper() + 'S' + word += ( + "set " + extraname + " " + " ".join(sorted(setOfValues[fld])) + + ("\n" * 2) + ) + return word + + +def get_set_of_values_for_field(lib, fields): + # get the unique values from a item/album field + dictOfFields = {} + for each in fields: + dictOfFields[each] = set() + for item in lib.items(): + for field in fields: + dictOfFields[field].add(wrap(item[field])) + return dictOfFields + + +def get_basic_beet_options(): + word = ( + BL_NEED2.format("-l format-item", + "-f -d 'print with custom format'") + + BL_NEED2.format("-l format-album", + "-f -d 'print with custom format'") + + BL_NEED2.format("-s l -l library", + "-f -r -d 'library database file to use'") + + BL_NEED2.format("-s d -l directory", + "-f -r -d 'destination music directory'") + + BL_NEED2.format("-s v -l verbose", + "-f -d 'print debugging information'") + + + BL_NEED2.format("-s c -l config", + "-f -r -d 'path to configuration file'") + + BL_NEED2.format("-s h -l help", + "-f -d 'print this help message and exit'")) + return word + + +def get_subcommands(cmd_name_and_help, nobasicfields, extravalues): + # formatting for fish to complete our fields/values + word = "" + for cmdname, cmdhelp in cmd_name_and_help: + word += "\n" + "# ------ {} -------".format( + "fieldsetups for " + cmdname) + "\n" + word += ( + BL_NEED2.format( + ("-a " + cmdname), + ("-f " + "-d " + wrap(clean_whitespace(cmdhelp))))) + + if nobasicfields is False: + word += ( + BL_USE3.format( + cmdname, + ("-a " + wrap("$FIELDS")), + ("-f " + "-d " + wrap("fieldname")))) + + if extravalues: + for f in extravalues: + setvar = wrap("$" + f.upper() + "S") + word += " ".join(BL_EXTRA3.format( + (cmdname + " " + f + ":"), + ('-f ' + '-A ' + '-a ' + setvar), + ('-d ' + wrap(f))).split()) + "\n" + return word + + +def get_all_commands(beetcmds): + # formatting for fish to complete command-options + word = "" + for cmd in beetcmds: + names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names.append(cmd.name) + for name in names: + word += "\n" + word += ("\n" * 2) + "# ====== {} =====".format( + "completions for " + name) + "\n" + + for option in cmd.parser._get_all_options()[1:]: + cmd_LO = (" -l " + option._long_opts[0].replace('--', '') + )if option._long_opts else '' + cmd_SO = (" -s " + option._short_opts[0].replace('-', '') + ) if option._short_opts else '' + cmd_needARG = ' -r ' if option.nargs in [1] else '' + cmd_helpstr = (" -d " + wrap(' '.join(option.help.split())) + ) if option.help else '' + cmd_arglist = (' -a ' + wrap(" ".join(option.choices)) + ) if option.choices else '' + + word += " ".join(BL_USE3.format( + name, + (cmd_needARG + cmd_SO + cmd_LO + " -f " + cmd_arglist), + cmd_helpstr).split()) + "\n" + + word = (word + " ".join(BL_USE3.format( + name, + ("-s " + "h " + "-l " + "help" + " -f "), + ('-d ' + wrap("print help") + "\n") + ).split())) + return word + + +def clean_whitespace(word): + # remove to much whitespace,tabs in string + return " ".join(word.split()) + + +def wrap(word): + # need " or ' around strings but watch out if they're in the string + sptoken = '\"' + if ('"') in word and ("'") in word: + word.replace('"', sptoken) + return '"' + word + '"' + + tok = '"' if "'" in word else "'" + return tok + word + tok diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst new file mode 100644 index 000000000..a6e41a46d --- /dev/null +++ b/docs/plugins/fish.rst @@ -0,0 +1,43 @@ +Fish plugins +============ + +The ``fish`` plugin adds a ``beet fish`` command that will create a fish +autocompletion file ``beet.fish`` in ``~/.config/fish/completions`` +This makes `fish`_ - a different shell - autocomplete commands for beet. + +.. _fish: http://fishshell.com/ + +Configuring +=========== + +This will only make sense if you have the `fish`_ shell installed. +Enable the ``fish`` plugin (see :ref:`using-plugins`). +If you install or disable plugins, run ``beet fish`` again. It takes the values +from the plugins you have enabled. + +Using +===== + +Type ``beet fish``. Hit ``enter`` and will see the file ``beet.fish`` appear +in ``.config/fish/completions`` in your home folder. + +For a not-fish user: After you type ``beet`` in your fish-prompt and ``TAB`` +you will get the autosuggestions for all your plugins/commands and +typing ``-`` will get you all the options available to you. +If you type ``beet ls`` and you ``TAB`` you will get a list of all the album/item +fields that beet offers. Start typing ``genr`` ``TAB`` and fish completes +``genre:`` ... ready to type on... + +Options +======= + +The default is that you get autocompletion for all the album/item fields. +You can disable that with ``beet fish -f`` In that case you only get all +the plugins/commands/options. Everything else you type in yourself. +If you want completion for a specific album/item field, you can get that like +this ``beet fish -e genre`` or ``beet fish -e genre -e albumartist`` . +Then when you type at your fish-prompt ``beet list genre:`` and you ``TAB`` +you will get a list of all your genres to choose from. +REMEMBER : we get all the values of these fields and put them in the completion +file. It is not meant to be a replacement of your database. In other words : +speed and size matters. From b53a91662361f0434232e51a54c0fcef83351eac Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 11:53:33 +0100 Subject: [PATCH 062/293] added normalize method to the Integer class --- beets/dbcore/types.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index 521a5a1ee..5aa2b9812 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -131,6 +131,14 @@ class Integer(Type): query = query.NumericQuery model_type = int + def normalize(self, value): + try: + return self.model_type(round(float(value))) + except ValueError: + return self.null + except TypeError: + return self.null + class PaddedInt(Integer): """An integer field that is formatted with a given number of digits, From 5fc4d7c35e6d508b9084e1db518f2e24cbe3af28 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 13:00:15 +0100 Subject: [PATCH 063/293] - added `field_two` as type STRING in ModelFixture1 - renamed test test_format_fixed_field to test_format_fixed_field_string - test_format_fixed_field_string not tests `field_two` with string values - added new test_format_fixed_field_integer to test field `field_one` as INTEGER - added new test_format_fixed_field_integer_normalized to test rounding float values --- test/test_dbcore.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 9bf78de67..0d40896da 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -53,6 +53,7 @@ class ModelFixture1(dbcore.Model): _fields = { 'id': dbcore.types.PRIMARY_ID, 'field_one': dbcore.types.INTEGER, + 'field_two': dbcore.types.STRING, } _types = { 'some_float_field': dbcore.types.FLOAT, @@ -355,7 +356,7 @@ class ModelTest(unittest.TestCase): def test_items(self): model = ModelFixture1(self.db) model.id = 5 - self.assertEqual({('id', 5), ('field_one', 0)}, + self.assertEqual({('id', 5), ('field_one', 0), ('field_two', '')}, set(model.items())) def test_delete_internal_field(self): @@ -370,10 +371,28 @@ class ModelTest(unittest.TestCase): class FormatTest(unittest.TestCase): - def test_format_fixed_field(self): + def test_format_fixed_field_integer(self): model = ModelFixture1() - model.field_one = u'caf\xe9' + model.field_one = 155 value = model.formatted().get('field_one') + self.assertEqual(value, u'155') + + def test_format_fixed_field_integer_normalized(self): + """The normalize method of the Integer class rounds floats + """ + model = ModelFixture1() + model.field_one = 142.432 + value = model.formatted().get('field_one') + self.assertEqual(value, u'142') + + model.field_one = 142.863 + value = model.formatted().get('field_one') + self.assertEqual(value, u'143') + + def test_format_fixed_field_string(self): + model = ModelFixture1() + model.field_two = u'caf\xe9' + value = model.formatted().get('field_two') self.assertEqual(value, u'caf\xe9') def test_format_flex_field(self): From 832c7326af8660e900988807ee2a1890cf24e67b Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 2 Mar 2020 13:03:18 +0100 Subject: [PATCH 064/293] corrected test to account for `year` and `disctotal` field now being treated as `types.INTEGER` --- test/test_autotag.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 28b7fd209..5ab3c4b49 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -21,6 +21,8 @@ import re import copy import unittest +from beets.dbcore.types import Integer, PaddedInt + from test import _common from beets import autotag from beets.autotag import match @@ -91,7 +93,10 @@ class PluralityTest(_common.TestCase): for i in range(5)] likelies, _ = match.current_metadata(items) for f in fields: - self.assertEqual(likelies[f], '%s_1' % f) + if type(items[0]._fields[f]) in (Integer, PaddedInt): + self.assertEqual(likelies[f], 0) + else: + self.assertEqual(likelies[f], '%s_1' % f) def _make_item(title, track, artist=u'some artist'): From 82c3867fc086e6729218c2884f494cfefbd11f56 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 13:52:35 +0100 Subject: [PATCH 065/293] Rewrite Fish completion plugin docs & code comments --- beetsplug/fish.py | 58 ++++++++++++++++--------------- docs/plugins/fish.rst | 79 ++++++++++++++++++++++++------------------- 2 files changed, 74 insertions(+), 63 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index 3a7682c2a..b81b9c387 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -1,5 +1,6 @@ # This file is part of beets. # Copyright 2015, winters jean-marie. +# Copyright 2020, Justin Mayer # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -12,12 +13,13 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""If you use the fish-shell http://fishshell.com/ ... this will do -autocomplete for you. It does the main commands and options for beet -and the plugins. -It gives you all the album and itemfields (like genre, album) but not all the -values for these. It suggest genre: or album: but not genre: Pop..Jazz...Rock -You can get that by specifying ex. --extravalues genre. +"""This plugin generates tab completions for Beets commands for the Fish shell +, including completions for Beets commands, plugin +commands, and option flags. Also generated are completions for all the album +and track fields, suggesting for example `genre:` or `album:` when querying the +Beets database. Completions for the *values* of those fields are not generated by +default but can be included via the `-e` or `--extravalues` flag. For example: +`beet fish -e genre -e albumartist` """ from __future__ import (division, absolute_import, print_function, @@ -68,11 +70,11 @@ end class FishPlugin(BeetsPlugin): def commands(self): - cmd = ui.Subcommand('fish', help='make fish autocomplete beet') + cmd = ui.Subcommand('fish', help='generate Fish shell tab completions') cmd.func = self.run cmd.parser.add_option('-f', '--noFields', action='store_true', default=False, - help='no item/album fields for autocomplete') + help='omit album/track field completions') cmd.parser.add_option( '-e', '--extravalues', @@ -80,15 +82,15 @@ class FishPlugin(BeetsPlugin): type='choice', choices=library.Item.all_keys() + library.Album.all_keys(), - help='pick field, get field-values for autocomplete') + help='include specified field *values* in completions') return [cmd] def run(self, lib, opts, args): - # we gather the commands from beet and from the plugins. - # we take the album and item fields. - # it wanted, we take the values from these fields. - # we make a giant string of tehm formatted in a way that - # allows fish to do autocompletion for beet. + # Gather the commands from Beets core and its plugins. + # Collect the album and track fields. + # If specified, also collect the values for these fields. + # Make a giant string of all the above, formatted in a way that + # allows Fish to do tab completion for the `beet` command. homeDir = os.path.expanduser("~") completePath = os.path.join(homeDir, '.config/fish/completions') try: @@ -97,22 +99,22 @@ class FishPlugin(BeetsPlugin): if not os.path.isdir(completePath): raise pathAndFile = os.path.join(completePath, 'beet.fish') - nobasicfields = opts.noFields # do not complete for item/album fields - extravalues = opts.extravalues # ex complete all artist values + nobasicfields = opts.noFields # Do not complete for album/track fields + extravalues = opts.extravalues # e.g., Also complete artists names beetcmds = sorted( (commands.default_commands + commands.plugins.commands()), key=attrgetter('name')) fields = sorted(set( library.Album.all_keys() + library.Item.all_keys())) - # collect cmds and their aliases and their help message + # Collect commands, their aliases, and their help text cmd_names_help = [] for cmd in beetcmds: names = ["\?" if alias == "?" else alias for alias in cmd.aliases] names.append(cmd.name) for name in names: cmd_names_help.append((name, cmd.help)) - # here we go assembling the string + # Concatenate the string totstring = HEAD + "\n" totstring += get_cmds_list([name[0] for name in cmd_names_help]) totstring += '' if nobasicfields else get_standard_fields(fields) @@ -124,7 +126,7 @@ class FishPlugin(BeetsPlugin): "setup field completion for subcommands") + "\n" totstring += get_subcommands( cmd_names_help, nobasicfields, extravalues) - # setup completion for all the command-options + # Set up completion for all the command options totstring += get_all_commands(beetcmds) with open(pathAndFile, 'w') as fish_file: @@ -132,7 +134,7 @@ class FishPlugin(BeetsPlugin): def get_cmds_list(cmds_names): - # make list of all commands in beet&plugins + # Make a list of all Beets core & plugin commands substr = '' substr += ( "set CMDS " + " ".join(cmds_names) + ("\n" * 2) @@ -141,7 +143,7 @@ def get_cmds_list(cmds_names): def get_standard_fields(fields): - # make list of item/album fields & append with ':' + # Make a list of album/track fields and append with ':' fields = (field + ":" for field in fields) substr = '' substr += ( @@ -151,8 +153,8 @@ def get_standard_fields(fields): def get_extravalues(lib, extravalues): - # make list of all values from a item/album field - # so type artist: and get completion for stones, beatles .. + # Make a list of all values from an album/track field. + # 'beet ls albumartist: ' yields completions for ABBA, Beatles, etc. word = '' setOfValues = get_set_of_values_for_field(lib, extravalues) for fld in extravalues: @@ -165,7 +167,7 @@ def get_extravalues(lib, extravalues): def get_set_of_values_for_field(lib, fields): - # get the unique values from a item/album field + # Get unique values from a specified album/track field dictOfFields = {} for each in fields: dictOfFields[each] = set() @@ -196,7 +198,7 @@ def get_basic_beet_options(): def get_subcommands(cmd_name_and_help, nobasicfields, extravalues): - # formatting for fish to complete our fields/values + # Formatting for Fish to complete our fields/values word = "" for cmdname, cmdhelp in cmd_name_and_help: word += "\n" + "# ------ {} -------".format( @@ -224,7 +226,7 @@ def get_subcommands(cmd_name_and_help, nobasicfields, extravalues): def get_all_commands(beetcmds): - # formatting for fish to complete command-options + # Formatting for Fish to complete command options word = "" for cmd in beetcmds: names = ["\?" if alias == "?" else alias for alias in cmd.aliases] @@ -259,12 +261,12 @@ def get_all_commands(beetcmds): def clean_whitespace(word): - # remove to much whitespace,tabs in string + # Remove excess whitespace and tabs in a string return " ".join(word.split()) def wrap(word): - # need " or ' around strings but watch out if they're in the string + # Need " or ' around strings but watch out if they're in the string sptoken = '\"' if ('"') in word and ("'") in word: word.replace('"', sptoken) diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst index a6e41a46d..b2cb096ee 100644 --- a/docs/plugins/fish.rst +++ b/docs/plugins/fish.rst @@ -1,43 +1,52 @@ -Fish plugins -============ - -The ``fish`` plugin adds a ``beet fish`` command that will create a fish -autocompletion file ``beet.fish`` in ``~/.config/fish/completions`` -This makes `fish`_ - a different shell - autocomplete commands for beet. - -.. _fish: http://fishshell.com/ - -Configuring +Fish Plugin =========== -This will only make sense if you have the `fish`_ shell installed. -Enable the ``fish`` plugin (see :ref:`using-plugins`). -If you install or disable plugins, run ``beet fish`` again. It takes the values -from the plugins you have enabled. +The ``fish`` plugin adds a ``beet fish`` command that creates a `Fish shell`_ +tab-completion file named ``beet.fish`` in ``~/.config/fish/completions``. +This enables tab-completion of ``beet`` commands for the `Fish shell`_. -Using -===== +.. _Fish shell: https://fishshell.com/ -Type ``beet fish``. Hit ``enter`` and will see the file ``beet.fish`` appear -in ``.config/fish/completions`` in your home folder. +Configuration +------------- -For a not-fish user: After you type ``beet`` in your fish-prompt and ``TAB`` -you will get the autosuggestions for all your plugins/commands and -typing ``-`` will get you all the options available to you. -If you type ``beet ls`` and you ``TAB`` you will get a list of all the album/item -fields that beet offers. Start typing ``genr`` ``TAB`` and fish completes -``genre:`` ... ready to type on... +Enable the ``fish`` plugin (see :ref:`using-plugins`) on a system running the +`Fish shell`_. + +Usage +----- + +Type ``beet fish`` to generate the ``beet.fish`` completions file at: +``~/.config/fish/completions/``. If you later install or disable plugins, run +``beet fish`` again to update the completions based on the enabled plugins. + +For users not accustomed to tab completion… After you type ``beet`` followed by +a space in your shell prompt and then the ``TAB`` key, you should see a list of +the beets commands (and their abbreviated versions) that can be invoked in your +current environment. Similarly, typing ``beet -`` will show you all the +option flags available to you, which also applies to subcommands such as +``beet import -``. If you type ``beet ls`` followed by a space and then the +and the ``TAB`` key, you will see a list of all the album/track fields that can +be used in beets queries. For example, typing ``beet ls ge`` will complete +to ``genre:`` and leave you ready to type the rest of your query. Options -======= +------- -The default is that you get autocompletion for all the album/item fields. -You can disable that with ``beet fish -f`` In that case you only get all -the plugins/commands/options. Everything else you type in yourself. -If you want completion for a specific album/item field, you can get that like -this ``beet fish -e genre`` or ``beet fish -e genre -e albumartist`` . -Then when you type at your fish-prompt ``beet list genre:`` and you ``TAB`` -you will get a list of all your genres to choose from. -REMEMBER : we get all the values of these fields and put them in the completion -file. It is not meant to be a replacement of your database. In other words : -speed and size matters. +In addition to beets commands, plugin commands, and option flags, the generated +completions also include by default all the album/track fields. If you only want +the former and do not want the album/track fields included in the generated +completions, use ``beet fish -f`` to only generate completions for beets/plugin +commands and option flags. + +If you want generated completions to also contain album/track field *values* for +the items in your library, you can use the ``-e`` or ``--extravalues`` option. +For example: ``beet fish -e genre`` or ``beet fish -e genre -e albumartist`` +In the latter case, subsequently typing ``beet list genre: `` will display +a list of all the genres in your library and ``beet list albumartist: `` +will show a list of the album artists in your library. Keep in mind that all of +these values will be put into the generated completions file, so use this option +with care when specified fields contain a large number of values. Libraries with, +for example, very large numbers of genres/artists may result in higher memory +utilization, completion latency, et cetera. This option is not meant to replace +database queries altogether. From 05db0d18eb7f01f319bb30deb6d348d0ffe5150d Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 15:38:56 +0100 Subject: [PATCH 066/293] Don't escape question marks in Fish completions Fish shell previously interpreted question marks as glob characters, but that behavior has been deprecated and will soon be removed. Plus, the completion for `help` and its alias `?` does not currently seem to behave as expected anyway and is thus, at present, of limited utility. --- beetsplug/fish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index b81b9c387..fd9753733 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -110,7 +110,7 @@ class FishPlugin(BeetsPlugin): # Collect commands, their aliases, and their help text cmd_names_help = [] for cmd in beetcmds: - names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names = [alias for alias in cmd.aliases] names.append(cmd.name) for name in names: cmd_names_help.append((name, cmd.help)) @@ -229,7 +229,7 @@ def get_all_commands(beetcmds): # Formatting for Fish to complete command options word = "" for cmd in beetcmds: - names = ["\?" if alias == "?" else alias for alias in cmd.aliases] + names = [alias for alias in cmd.aliases] names.append(cmd.name) for name in names: word += "\n" From f465c90e78d6bb102aec307057e37ad85e55422a Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 15:46:04 +0100 Subject: [PATCH 067/293] Enforce PEP-8 compliance on Fish completion plugin --- beetsplug/fish.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index fd9753733..b842ac70f 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # This file is part of beets. # Copyright 2015, winters jean-marie. # Copyright 2020, Justin Mayer @@ -17,13 +18,12 @@ , including completions for Beets commands, plugin commands, and option flags. Also generated are completions for all the album and track fields, suggesting for example `genre:` or `album:` when querying the -Beets database. Completions for the *values* of those fields are not generated by -default but can be included via the `-e` or `--extravalues` flag. For example: +Beets database. Completions for the *values* of those fields are not generated +by default but can be added via the `-e` / `--extravalues` flag. For example: `beet fish -e genre -e albumartist` """ -from __future__ import (division, absolute_import, print_function, - unicode_literals) +from __future__ import division, absolute_import, print_function from beets.plugins import BeetsPlugin from beets import library, ui @@ -91,14 +91,14 @@ class FishPlugin(BeetsPlugin): # If specified, also collect the values for these fields. # Make a giant string of all the above, formatted in a way that # allows Fish to do tab completion for the `beet` command. - homeDir = os.path.expanduser("~") - completePath = os.path.join(homeDir, '.config/fish/completions') + home_dir = os.path.expanduser("~") + completion_dir = os.path.join(home_dir, '.config/fish/completions') try: - os.makedirs(completePath) + os.makedirs(completion_dir) except OSError: - if not os.path.isdir(completePath): + if not os.path.isdir(completion_dir): raise - pathAndFile = os.path.join(completePath, 'beet.fish') + completion_file_path = os.path.join(completion_dir, 'beet.fish') nobasicfields = opts.noFields # Do not complete for album/track fields extravalues = opts.extravalues # e.g., Also complete artists names beetcmds = sorted( @@ -129,7 +129,7 @@ class FishPlugin(BeetsPlugin): # Set up completion for all the command options totstring += get_all_commands(beetcmds) - with open(pathAndFile, 'w') as fish_file: + with open(completion_file_path, 'w') as fish_file: fish_file.write(totstring) @@ -156,11 +156,11 @@ def get_extravalues(lib, extravalues): # Make a list of all values from an album/track field. # 'beet ls albumartist: ' yields completions for ABBA, Beatles, etc. word = '' - setOfValues = get_set_of_values_for_field(lib, extravalues) + values_set = get_set_of_values_for_field(lib, extravalues) for fld in extravalues: extraname = fld.upper() + 'S' word += ( - "set " + extraname + " " + " ".join(sorted(setOfValues[fld])) + "set " + extraname + " " + " ".join(sorted(values_set[fld])) + ("\n" * 2) ) return word @@ -168,13 +168,13 @@ def get_extravalues(lib, extravalues): def get_set_of_values_for_field(lib, fields): # Get unique values from a specified album/track field - dictOfFields = {} + fields_dict = {} for each in fields: - dictOfFields[each] = set() + fields_dict[each] = set() for item in lib.items(): for field in fields: - dictOfFields[field].add(wrap(item[field])) - return dictOfFields + fields_dict[field].add(wrap(item[field])) + return fields_dict def get_basic_beet_options(): @@ -237,11 +237,11 @@ def get_all_commands(beetcmds): "completions for " + name) + "\n" for option in cmd.parser._get_all_options()[1:]: - cmd_LO = (" -l " + option._long_opts[0].replace('--', '') - )if option._long_opts else '' - cmd_SO = (" -s " + option._short_opts[0].replace('-', '') - ) if option._short_opts else '' - cmd_needARG = ' -r ' if option.nargs in [1] else '' + cmd_l = (" -l " + option._long_opts[0].replace('--', '') + )if option._long_opts else '' + cmd_s = (" -s " + option._short_opts[0].replace('-', '') + ) if option._short_opts else '' + cmd_need_arg = ' -r ' if option.nargs in [1] else '' cmd_helpstr = (" -d " + wrap(' '.join(option.help.split())) ) if option.help else '' cmd_arglist = (' -a ' + wrap(" ".join(option.choices)) @@ -249,7 +249,7 @@ def get_all_commands(beetcmds): word += " ".join(BL_USE3.format( name, - (cmd_needARG + cmd_SO + cmd_LO + " -f " + cmd_arglist), + (cmd_need_arg + cmd_s + cmd_l + " -f " + cmd_arglist), cmd_helpstr).split()) + "\n" word = (word + " ".join(BL_USE3.format( From 14a654bbdbbeacf2c5792d112a0ef4eb03c06a10 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Mon, 2 Mar 2020 09:56:40 +0100 Subject: [PATCH 068/293] Add Fish completion to changelog & plugin index --- docs/changelog.rst | 2 ++ docs/plugins/index.rst | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7ef19871b..e33cf8c12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets * :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is resized. @@ -217,6 +218,7 @@ For packagers: the test may no longer be necessary. * This version drops support for Python 3.4. +.. _Fish shell: https://fishshell.com/ .. _MediaFile: https://github.com/beetbox/mediafile .. _Confuse: https://github.com/beetbox/confuse .. _works: https://musicbrainz.org/doc/Work diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 383466a68..6c643ce61 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -78,6 +78,7 @@ following to your configuration:: export fetchart filefilter + fish freedesktop fromfilename ftintitle @@ -184,6 +185,7 @@ Interoperability * :doc:`badfiles`: Check audio file integrity. * :doc:`embyupdate`: Automatically notifies `Emby`_ whenever the beets library changes. +* :doc:`fish`: Adds `Fish shell`_ tab autocompletion to ``beet`` commands. * :doc:`importfeeds`: Keep track of imported files via ``.m3u`` playlist file(s) or symlinks. * :doc:`ipfs`: Import libraries from friends and get albums from them via ipfs. * :doc:`kodiupdate`: Automatically notifies `Kodi`_ whenever the beets library @@ -203,6 +205,7 @@ Interoperability .. _Emby: https://emby.media +.. _Fish shell: https://fishshell.com/ .. _Plex: https://plex.tv .. _Kodi: https://kodi.tv .. _Sonos: https://sonos.com @@ -326,4 +329,4 @@ Here are a few of the plugins written by the beets community: .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser -.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ \ No newline at end of file +.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ From fbd1266bc5ed6c71dda030ef9313763c9039a376 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sat, 14 Mar 2020 21:30:22 +0100 Subject: [PATCH 069/293] simplified condition in test and added changelog entry --- docs/changelog.rst | 3 +++ test/test_autotag.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7ef19871b..fca3e686f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -166,6 +166,9 @@ Fixes: * Added a warning when configuration files defined in the `include` directive of the configuration file fail to be imported. :bug:`3498` +* Added the normalize method to the dbcore.types.INTEGER class which now + properly returns integer values. + :bug:`762` and :bug:`3507` For plugin developers: diff --git a/test/test_autotag.py b/test/test_autotag.py index 5ab3c4b49..f8f329024 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -93,7 +93,7 @@ class PluralityTest(_common.TestCase): for i in range(5)] likelies, _ = match.current_metadata(items) for f in fields: - if type(items[0]._fields[f]) in (Integer, PaddedInt): + if isinstance(likelies[f], int): self.assertEqual(likelies[f], 0) else: self.assertEqual(likelies[f], '%s_1' % f) From d5a52cbd26beee9f571fe6ea433b65bc75ee246d Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sat, 14 Mar 2020 21:51:46 +0100 Subject: [PATCH 070/293] removed unused imports --- test/test_autotag.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index f8f329024..a11bc8fac 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -21,8 +21,6 @@ import re import copy import unittest -from beets.dbcore.types import Integer, PaddedInt - from test import _common from beets import autotag from beets.autotag import match From 99a3343c0c5860f4708d5272e5e1d680dc787ad6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 14 Mar 2020 20:27:57 -0400 Subject: [PATCH 071/293] A little more detail in changelog for #3508 --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f0997271..d3379b36c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -168,8 +168,9 @@ Fixes: of the configuration file fail to be imported. :bug:`3498` * Added the normalize method to the dbcore.types.INTEGER class which now - properly returns integer values. - :bug:`762` and :bug:`3507` + properly returns integer values, which should avoid problems where fields + like ``bpm`` would sometimes store non-integer values. + :bug:`762` :bug:`3507` :bug:`3508` For plugin developers: From 8988d908c5f39d6192fed61777bb37da464a20c8 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 12:52:12 +0100 Subject: [PATCH 072/293] match method converted to instance method --- beets/dbcore/query.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 8fb64e206..239eaa965 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -156,10 +156,9 @@ class NoneQuery(FieldQuery): def col_clause(self): return self.field + " IS NULL", () - @classmethod - def match(cls, item): + def match(self, item): try: - return item[cls.field] is None + return item.get(self.field) is None except KeyError: return True From eb4d2ef5c9655da307a238b7644ec00f25d3e6f9 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 12:52:56 +0100 Subject: [PATCH 073/293] added missing abstract method --- beets/dbcore/query.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 239eaa965..1bac90c06 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -162,6 +162,10 @@ class NoneQuery(FieldQuery): except KeyError: return True + @classmethod + def value_match(cls, pattern, value): + return pattern == value + def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) From 93744ff00a10dc3ecba33545d41b8b0e57dbf3f5 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 14:40:40 +0100 Subject: [PATCH 074/293] removed unnecessary KeyError test --- beets/dbcore/query.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 1bac90c06..5835ddc32 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -157,10 +157,7 @@ class NoneQuery(FieldQuery): return self.field + " IS NULL", () def match(self, item): - try: - return item.get(self.field) is None - except KeyError: - return True + return item.get(self.field, default=None) is None @classmethod def value_match(cls, pattern, value): From d6538e5f0c0e7eae2b4e9fee8243a2724d3da066 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 14:57:02 +0100 Subject: [PATCH 075/293] removed value_match method - not reachable? --- beets/dbcore/query.py | 4 ---- test/test_query.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 5835ddc32..db4b861c4 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -159,10 +159,6 @@ class NoneQuery(FieldQuery): def match(self, item): return item.get(self.field, default=None) is None - @classmethod - def value_match(cls, pattern, value): - return pattern == value - def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) diff --git a/test/test_query.py b/test/test_query.py index c0ab2a171..5e1556a45 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -772,6 +772,25 @@ class NoneQueryTest(unittest.TestCase, TestHelper): matched = self.lib.items(NoneQuery(u'rg_track_gain')) self.assertInResult(item, matched) + def test_match_slow(self): + item = self.add_item() + matched = self.lib.items(NoneQuery(u'rg_track_peak', fast=False)) + self.assertInResult(item, matched) + + def test_match_slow_after_set_none(self): + item = self.add_item(rg_track_gain=0) + matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) + self.assertNotInResult(item, matched) + + item['rg_track_gain'] = None + item.store() + matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) + self.assertInResult(item, matched) + + def test_match_repr(self): + q = NoneQuery(u'rg_track_gain', fast=False) + self.assertEquals("NoneQuery('rg_track_gain', False)", str(q)) + class NotQueryMatchTest(_common.TestCase): """Test `query.NotQuery` matching against a single item, using the same From 238f2244c91baaa0f5a9cb16c3ec6ca4ebb8a853 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:23:52 +0100 Subject: [PATCH 076/293] value_match is now correctly implemented --- beets/dbcore/query.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index db4b861c4..03b647a0d 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -152,12 +152,16 @@ class NoneQuery(FieldQuery): def __init__(self, field, fast=True): super(NoneQuery, self).__init__(field, None, fast) + self.pattern = None def col_clause(self): return self.field + " IS NULL", () def match(self, item): - return item.get(self.field, default=None) is None + return self.value_match(self.pattern, item.get(self.field, default=None)) + + def value_match(self, pattern, value): + return value is pattern def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) From 3f2f125b090c20889bbeca28f72be4dd4ea85d3c Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:24:17 +0100 Subject: [PATCH 077/293] better repr testing --- test/test_query.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 5e1556a45..855dd8967 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -787,9 +787,11 @@ class NoneQueryTest(unittest.TestCase, TestHelper): matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) self.assertInResult(item, matched) - def test_match_repr(self): - q = NoneQuery(u'rg_track_gain', fast=False) - self.assertEquals("NoneQuery('rg_track_gain', False)", str(q)) + def test_query_repr(self): + fld = u'rg_track_gain' + self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld))) + self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld, fast=True))) + self.assertEquals("NoneQuery('{}', False)".format(str(fld)), str(NoneQuery(fld, fast=False))) class NotQueryMatchTest(_common.TestCase): From 532c6d7c825ae40a9091e32f5eb54863f591a72b Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:41:32 +0100 Subject: [PATCH 078/293] better repr testing #2 --- test/test_query.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 855dd8967..aa41497b7 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -788,10 +788,12 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) def test_query_repr(self): - fld = u'rg_track_gain' - self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld))) - self.assertEquals("NoneQuery('{}', True)".format(str(fld)), str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery('{}', False)".format(str(fld)), str(NoneQuery(fld, fast=False))) + self.assertEquals("NoneQuery('rg_track_gain', True)", + str(NoneQuery(u'rg_track_gain'))) + self.assertEquals("NoneQuery('rg_track_gain', True)", + str(NoneQuery(u'rg_track_gain', fast=True))) + self.assertEquals("NoneQuery('rg_track_gain', False)", + str(NoneQuery(u'rg_track_gain', fast=False))) class NotQueryMatchTest(_common.TestCase): From 8e68b5ff2e89dbd270b7493e8153cb3c90e2fd08 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 15:54:01 +0100 Subject: [PATCH 079/293] cleaning up --- beets/dbcore/query.py | 3 ++- test/test_query.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 03b647a0d..e4f5b8a29 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -158,7 +158,8 @@ class NoneQuery(FieldQuery): return self.field + " IS NULL", () def match(self, item): - return self.value_match(self.pattern, item.get(self.field, default=None)) + return self.value_match(self.pattern, + item.get(self.field, default=None)) def value_match(self, pattern, value): return value is pattern diff --git a/test/test_query.py b/test/test_query.py index aa41497b7..10206eb98 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -788,11 +788,11 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) def test_query_repr(self): - self.assertEquals("NoneQuery('rg_track_gain', True)", + self.assertEquals("NoneQuery(u'rg_track_gain', True)", str(NoneQuery(u'rg_track_gain'))) - self.assertEquals("NoneQuery('rg_track_gain', True)", + self.assertEquals("NoneQuery(u'rg_track_gain', True)", str(NoneQuery(u'rg_track_gain', fast=True))) - self.assertEquals("NoneQuery('rg_track_gain', False)", + self.assertEquals("NoneQuery(u'rg_track_gain', False)", str(NoneQuery(u'rg_track_gain', fast=False))) From 935768d98326c662e9d62add0fdb9aa05d9bb05f Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 19:19:09 +0100 Subject: [PATCH 080/293] fixing repr tests --- test/test_query.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 10206eb98..ad770e1b9 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -788,12 +788,21 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) def test_query_repr(self): - self.assertEquals("NoneQuery(u'rg_track_gain', True)", - str(NoneQuery(u'rg_track_gain'))) - self.assertEquals("NoneQuery(u'rg_track_gain', True)", - str(NoneQuery(u'rg_track_gain', fast=True))) - self.assertEquals("NoneQuery(u'rg_track_gain', False)", - str(NoneQuery(u'rg_track_gain', fast=False))) + fld = u'rg_track_gain' + if sys.version_info <= (2, 7): + self.assertEquals("NoneQuery('u{}', True)".format(fld), + str(NoneQuery(fld))) + self.assertEquals("NoneQuery('u{}', True)".format(fld), + str(NoneQuery(fld, fast=True))) + self.assertEquals("NoneQuery('u{}', False)".format(fld), + str(NoneQuery(fld, fast=False))) + else: + self.assertEquals("NoneQuery('{}', True)".format(fld), + str(NoneQuery(fld))) + self.assertEquals("NoneQuery('{}', True)".format(fld), + str(NoneQuery(fld, fast=True))) + self.assertEquals("NoneQuery('{}', False)".format(fld), + str(NoneQuery(fld, fast=False))) class NotQueryMatchTest(_common.TestCase): From ceb901fcca8958c7930a79dab1830533833e477c Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 19:30:24 +0100 Subject: [PATCH 081/293] struggling with old python --- test/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_query.py b/test/test_query.py index ad770e1b9..06f2deec2 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -789,7 +789,7 @@ class NoneQueryTest(unittest.TestCase, TestHelper): def test_query_repr(self): fld = u'rg_track_gain' - if sys.version_info <= (2, 7): + if sys.version_info <= (3, 0): self.assertEquals("NoneQuery('u{}', True)".format(fld), str(NoneQuery(fld))) self.assertEquals("NoneQuery('u{}', True)".format(fld), From ac1a3851faa04fb7bca41bc5cbbdf403c42c0cc9 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 19:36:38 +0100 Subject: [PATCH 082/293] typo fix --- test/test_query.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 06f2deec2..fc8e76735 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -790,11 +790,11 @@ class NoneQueryTest(unittest.TestCase, TestHelper): def test_query_repr(self): fld = u'rg_track_gain' if sys.version_info <= (3, 0): - self.assertEquals("NoneQuery('u{}', True)".format(fld), + self.assertEquals("NoneQuery(u'{}', True)".format(fld), str(NoneQuery(fld))) - self.assertEquals("NoneQuery('u{}', True)".format(fld), + self.assertEquals("NoneQuery(u'{}', True)".format(fld), str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery('u{}', False)".format(fld), + self.assertEquals("NoneQuery(u'{}', False)".format(fld), str(NoneQuery(fld, fast=False))) else: self.assertEquals("NoneQuery('{}', True)".format(fld), From 4c6993989caf9e01d269ae03715adb6a175b23fd Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Sun, 15 Mar 2020 20:57:47 +0100 Subject: [PATCH 083/293] changelog entry --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d3379b36c..457275763 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -171,6 +171,8 @@ Fixes: properly returns integer values, which should avoid problems where fields like ``bpm`` would sometimes store non-integer values. :bug:`762` :bug:`3507` :bug:`3508` +* Removed `@classmethod`` decorator from dbcore.query.NoneQuery.match method + failing with AttributeError when called. It is now an instance method.` For plugin developers: From 4ba7b8da313a535da8ad9972cc984557c04a5909 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 16 Mar 2020 10:12:09 +0100 Subject: [PATCH 084/293] sampsyo's docs/changelog.rst correction Co-Authored-By: Adrian Sampson --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 457275763..a51c85cad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -171,8 +171,9 @@ Fixes: properly returns integer values, which should avoid problems where fields like ``bpm`` would sometimes store non-integer values. :bug:`762` :bug:`3507` :bug:`3508` -* Removed `@classmethod`` decorator from dbcore.query.NoneQuery.match method - failing with AttributeError when called. It is now an instance method.` +* Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method + failing with AttributeError when called. It is now an instance method. + :bug:`3516` :bug:`3517` For plugin developers: From 611659d03c6e75b25aea994cf9daa0732df61679 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 16 Mar 2020 10:16:18 +0100 Subject: [PATCH 085/293] removed value_match and repr tests --- beets/dbcore/query.py | 7 +------ test/test_query.py | 17 ----------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index e4f5b8a29..db4b861c4 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -152,17 +152,12 @@ class NoneQuery(FieldQuery): def __init__(self, field, fast=True): super(NoneQuery, self).__init__(field, None, fast) - self.pattern = None def col_clause(self): return self.field + " IS NULL", () def match(self, item): - return self.value_match(self.pattern, - item.get(self.field, default=None)) - - def value_match(self, pattern, value): - return value is pattern + return item.get(self.field, default=None) is None def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) diff --git a/test/test_query.py b/test/test_query.py index fc8e76735..f88a12c92 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -787,23 +787,6 @@ class NoneQueryTest(unittest.TestCase, TestHelper): matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False)) self.assertInResult(item, matched) - def test_query_repr(self): - fld = u'rg_track_gain' - if sys.version_info <= (3, 0): - self.assertEquals("NoneQuery(u'{}', True)".format(fld), - str(NoneQuery(fld))) - self.assertEquals("NoneQuery(u'{}', True)".format(fld), - str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery(u'{}', False)".format(fld), - str(NoneQuery(fld, fast=False))) - else: - self.assertEquals("NoneQuery('{}', True)".format(fld), - str(NoneQuery(fld))) - self.assertEquals("NoneQuery('{}', True)".format(fld), - str(NoneQuery(fld, fast=True))) - self.assertEquals("NoneQuery('{}', False)".format(fld), - str(NoneQuery(fld, fast=False))) - class NotQueryMatchTest(_common.TestCase): """Test `query.NotQuery` matching against a single item, using the same From b34d1f71a93b3663f70f46a4d5ed08702b14c5ca Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 16 Mar 2020 09:08:57 -0400 Subject: [PATCH 086/293] Slightly terser `get` call for #3517 --- beets/dbcore/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index db4b861c4..4f19f4f8d 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -157,7 +157,7 @@ class NoneQuery(FieldQuery): return self.field + " IS NULL", () def match(self, item): - return item.get(self.field, default=None) is None + return item.get(self.field) is None def __repr__(self): return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self) From 19d6dfc8f301c4123989227084842d1c25807594 Mon Sep 17 00:00:00 2001 From: x1ppy <> Date: Thu, 23 Jan 2020 15:07:26 -0500 Subject: [PATCH 087/293] Support extra tags for MusicBrainz queries --- beets/autotag/hooks.py | 16 +++++++++++----- beets/autotag/match.py | 9 ++++++++- beets/autotag/mb.py | 22 ++++++++++++++++++++-- beets/config_default.yaml | 1 + beets/plugins.py | 7 ++++--- docs/changelog.rst | 2 ++ docs/reference/config.rst | 20 ++++++++++++++++++++ test/test_importer.py | 2 +- 8 files changed, 67 insertions(+), 12 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 030f371ba..d7c701db6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -614,17 +614,21 @@ def tracks_for_id(track_id): @plugins.notify_info_yielded(u'albuminfo_received') -def album_candidates(items, artist, album, va_likely): +def album_candidates(items, artist, album, va_likely, extra_tags): """Search for album matches. ``items`` is a list of Item objects that make up the album. ``artist`` and ``album`` are the respective names (strings), which may be derived from the item list or may be entered by the user. ``va_likely`` is a boolean indicating whether - the album is likely to be a "various artists" release. + the album is likely to be a "various artists" release. ``extra_tags`` + is an optional dictionary of additional tags used to further + constrain the search. """ + # Base candidates if we have album and artist to match. if artist and album: try: - for candidate in mb.match_album(artist, album, len(items)): + for candidate in mb.match_album(artist, album, len(items), + extra_tags): yield candidate except mb.MusicBrainzAPIError as exc: exc.log(log) @@ -632,13 +636,15 @@ def album_candidates(items, artist, album, va_likely): # Also add VA matches from MusicBrainz where appropriate. if va_likely and album: try: - for candidate in mb.match_album(None, album, len(items)): + for candidate in mb.match_album(None, album, len(items), + extra_tags): yield candidate except mb.MusicBrainzAPIError as exc: exc.log(log) # Candidates from plugins. - for candidate in plugins.candidates(items, artist, album, va_likely): + for candidate in plugins.candidates(items, artist, album, va_likely, + extra_tags): yield candidate diff --git a/beets/autotag/match.py b/beets/autotag/match.py index 71b62adb7..f57cac739 100644 --- a/beets/autotag/match.py +++ b/beets/autotag/match.py @@ -447,6 +447,12 @@ def tag_album(items, search_artist=None, search_album=None, search_artist, search_album = cur_artist, cur_album log.debug(u'Search terms: {0} - {1}', search_artist, search_album) + extra_tags = None + if config['musicbrainz']['extra_tags']: + tag_list = config['musicbrainz']['extra_tags'].get() + extra_tags = {k: v for (k, v) in likelies.items() if k in tag_list} + log.debug(u'Additional search terms: {0}', extra_tags) + # Is this album likely to be a "various artist" release? va_likely = ((not consensus['artist']) or (search_artist.lower() in VA_ARTISTS) or @@ -457,7 +463,8 @@ def tag_album(items, search_artist=None, search_album=None, for matched_candidate in hooks.album_candidates(items, search_artist, search_album, - va_likely): + va_likely, + extra_tags): _add_candidate(items, candidates, matched_candidate) log.debug(u'Evaluating {0} candidates.', len(candidates)) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 1a6e0b1f1..f86d3be71 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -38,6 +38,14 @@ else: SKIPPED_TRACKS = ['[data track]'] +FIELDS_TO_MB_KEYS = { + 'catalognum': 'catno', + 'country': 'country', + 'label': 'label', + 'media': 'format', + 'year': 'date', +} + musicbrainzngs.set_useragent('beets', beets.__version__, 'https://beets.io/') @@ -411,13 +419,13 @@ def album_info(release): return info -def match_album(artist, album, tracks=None): +def match_album(artist, album, tracks=None, extra_tags=None): """Searches for a single album ("release" in MusicBrainz parlance) and returns an iterator over AlbumInfo objects. May raise a MusicBrainzAPIError. The query consists of an artist name, an album name, and, - optionally, a number of tracks on the album. + optionally, a number of tracks on the album and any other extra tags. """ # Build search criteria. criteria = {'release': album.lower().strip()} @@ -429,6 +437,16 @@ def match_album(artist, album, tracks=None): if tracks is not None: criteria['tracks'] = six.text_type(tracks) + # Additional search cues from existing metadata. + if extra_tags: + for tag in extra_tags: + key = FIELDS_TO_MB_KEYS[tag] + value = six.text_type(extra_tags.get(tag, '')).lower().strip() + if key == 'catno': + value = value.replace(u' ', '') + if value: + criteria[key] = value + # Abort if we have no search terms. if not any(criteria.values()): return diff --git a/beets/config_default.yaml b/beets/config_default.yaml index cf9ae6bf9..0fd6eb592 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -103,6 +103,7 @@ musicbrainz: ratelimit: 1 ratelimit_interval: 1.0 searchlimit: 5 + extra_tags: [] match: strong_rec_thresh: 0.04 diff --git a/beets/plugins.py b/beets/plugins.py index 73d85cdd3..8606ebc69 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -172,7 +172,7 @@ class BeetsPlugin(object): """ return beets.autotag.hooks.Distance() - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): """Should return a sequence of AlbumInfo objects that match the album whose items are provided. """ @@ -379,11 +379,12 @@ def album_distance(items, album_info, mapping): return dist -def candidates(items, artist, album, va_likely): +def candidates(items, artist, album, va_likely, extra_tags=None): """Gets MusicBrainz candidates for an album from each plugin. """ for plugin in find_plugins(): - for candidate in plugin.candidates(items, artist, album, va_likely): + for candidate in plugin.candidates(items, artist, album, va_likely, + extra_tags): yield candidate diff --git a/docs/changelog.rst b/docs/changelog.rst index a51c85cad..3769164be 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog New features: +* A new :ref:`extra_tags` configuration option allows more tagged metadata + to be included in MusicBrainz queries. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets * :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 7dcd53801..46f14f2c5 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -701,6 +701,26 @@ MusicBrainz server. Default: ``5``. +.. _extra_tags: + +extra_tags +~~~~~~~~~~ + +By default, beets will use only the artist, album, and track count to query +MusicBrainz. Additional tags to be queried can be supplied with the +``extra_tags`` setting. For example:: + + musicbrainz: + extra_tags: [year, catalognum, country, media, label] + +This setting should improve the autotagger results if the metadata with the +given tags match the metadata returned by MusicBrainz. + +Note that the only tags supported by this setting are the ones listed in the +above example. + +Default: ``[]`` + .. _match-config: Autotagger Matching Options diff --git a/test/test_importer.py b/test/test_importer.py index 8f637a077..3418d4628 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -79,7 +79,7 @@ class AutotagStub(object): autotag.mb.album_for_id = self.mb_album_for_id autotag.mb.track_for_id = self.mb_track_for_id - def match_album(self, albumartist, album, tracks): + def match_album(self, albumartist, album, tracks, extra_tags): if self.matching == self.IDENT: yield self._make_album_match(albumartist, album, tracks) From f306591a99f0a153d3116f36aaffa0ed5e5d4788 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sat, 28 Mar 2020 17:36:02 +1000 Subject: [PATCH 088/293] add the extra_tags option to all required plugins --- beetsplug/beatport.py | 2 +- beetsplug/chroma.py | 2 +- beetsplug/cue.py | 2 +- beetsplug/discogs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/beatport.py b/beetsplug/beatport.py index 6a45ab93a..df0abb2fc 100644 --- a/beetsplug/beatport.py +++ b/beetsplug/beatport.py @@ -355,7 +355,7 @@ class BeatportPlugin(BeetsPlugin): config=self.config ) - def candidates(self, items, artist, release, va_likely): + def candidates(self, items, artist, release, va_likely, extra_tags=None): """Returns a list of AlbumInfo objects for beatport search results matching release and artist (if not various). """ diff --git a/beetsplug/chroma.py b/beetsplug/chroma.py index c4230b069..54ae90098 100644 --- a/beetsplug/chroma.py +++ b/beetsplug/chroma.py @@ -191,7 +191,7 @@ class AcoustidPlugin(plugins.BeetsPlugin): dist.add_expr('track_id', info.track_id not in recording_ids) return dist - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): albums = [] for relid in prefix(_all_releases(items), MAX_RELEASES): album = hooks.album_for_mbid(relid) diff --git a/beetsplug/cue.py b/beetsplug/cue.py index fd564b55c..92ca8784a 100644 --- a/beetsplug/cue.py +++ b/beetsplug/cue.py @@ -24,7 +24,7 @@ class CuePlugin(BeetsPlugin): # self.register_listener('import_task_start', self.look_for_cues) - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): import pdb pdb.set_trace() diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 0ba27d7dd..86ace9aa8 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -175,7 +175,7 @@ class DiscogsPlugin(BeetsPlugin): config=self.config ) - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): """Returns a list of AlbumInfo objects for discogs search results matching an album and artist (if not various). """ From 333d5d1dd3a1e578427328a3ce5a09ee342fcd4c Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 30 Mar 2020 20:08:40 +0100 Subject: [PATCH 089/293] fetchart: Add Last.fm artwork source --- beetsplug/fetchart.py | 70 ++++++++++++++++++++++++++++++++++++++- docs/changelog.rst | 3 ++ docs/plugins/fetchart.rst | 17 ++++++++-- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 2fe8b0b2c..28067c310 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -21,6 +21,7 @@ from contextlib import closing import os import re from tempfile import NamedTemporaryFile +from collections import OrderedDict import requests @@ -742,11 +743,72 @@ class FileSystem(LocalArtSource): match=Candidate.MATCH_FALLBACK) +class LastFM(RemoteArtSource): + NAME = u"Last.fm" + + # Sizes in priority order. + SIZES = OrderedDict([ + ('mega', (300, 300)), + ('extralarge', (300, 300)), + ('large', (174, 174)), + ('medium', (64, 64)), + ('small', (34, 34)), + ]) + + if util.SNI_SUPPORTED: + API_URL = 'https://ws.audioscrobbler.com/2.0' + else: + API_URL = 'http://ws.audioscrobbler.com/2.0' + + def __init__(self, *args, **kwargs): + super(LastFM, self).__init__(*args, **kwargs) + self.key = self._config['lastfm_key'].get(), + + def get(self, album, plugin, paths): + if not album.mb_albumid: + return + + try: + response = self.request(self.API_URL, params={ + 'method': 'album.getinfo', + 'api_key': self.key, + 'mbid': album.mb_albumid, + 'format': 'json', + }) + except requests.RequestException: + self._log.debug(u'lastfm: error receiving response') + return + + try: + data = response.json() + + if 'error' in data: + if data['error'] == 6: + self._log.debug('lastfm: no results for {}', + album.mb_albumid) + else: + self._log.error( + 'lastfm: failed to get album info: {} ({})', + data['message'], data['error']) + else: + images = {image['size']: image['#text'] + for image in data['album']['image']} + + # Provide candidates in order of size. + for size in self.SIZES.keys(): + if size in images: + yield self._candidate(url=images[size], + size=self.SIZES[size]) + except ValueError: + self._log.debug(u'lastfm: error loading response: {}' + .format(response.text)) + return + # Try each source in turn. SOURCES_ALL = [u'filesystem', u'coverart', u'itunes', u'amazon', u'albumart', - u'wikipedia', u'google', u'fanarttv'] + u'wikipedia', u'google', u'fanarttv', u'lastfm'] ART_SOURCES = { u'filesystem': FileSystem, @@ -757,6 +819,7 @@ ART_SOURCES = { u'wikipedia': Wikipedia, u'google': GoogleImages, u'fanarttv': FanartTV, + u'lastfm': LastFM, } SOURCE_NAMES = {v: k for k, v in ART_SOURCES.items()} @@ -787,11 +850,13 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'google_key': None, 'google_engine': u'001442825323518660753:hrh5ch1gjzm', 'fanarttv_key': None, + 'lastfm_key': None, 'store_source': False, 'high_resolution': False, }) self.config['google_key'].redact = True self.config['fanarttv_key'].redact = True + self.config['lastfm_key'].redact = True self.minwidth = self.config['minwidth'].get(int) self.maxwidth = self.config['maxwidth'].get(int) @@ -831,6 +896,9 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): if not self.config['google_key'].get() and \ u'google' in available_sources: available_sources.remove(u'google') + if not self.config['lastfm_key'].get() and \ + u'lastfm' in available_sources: + available_sources.remove(u'lastfm') available_sources = [(s, c) for s in available_sources for c in ART_SOURCES[s].VALID_MATCHING_CRITERIA] diff --git a/docs/changelog.rst b/docs/changelog.rst index 3769164be..33ad386ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -113,6 +113,8 @@ New features: titles. Thanks to :user:`cole-miller`. :bug:`3459` +* :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. + :bug:`3530` Fixes: @@ -233,6 +235,7 @@ For packagers: .. _works: https://musicbrainz.org/doc/Work .. _Deezer: https://www.deezer.com .. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli +.. _last.fm: https://last.fm 1.4.9 (May 30, 2019) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 4441d4e30..e8f7b6d92 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -58,9 +58,9 @@ file. The available options are: - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. Default: ``filesystem coverart itunes amazon albumart``, i.e., everything but - ``wikipedia``, ``google`` and ``fanarttv``. Enable those sources for more - matches at the cost of some speed. They are searched in the given order, - thus in the default config, no remote (Web) art source are queried if + ``wikipedia``, ``google``, ``fanarttv`` and ``lastfm``. Enable those sources + for more matches at the cost of some speed. They are searched in the given + order, thus in the default config, no remote (Web) art source are queried if local art is found in the filesystem. To use a local image as fallback, move it to the end of the list. For even more fine-grained control over the search order, see the section on :ref:`album-art-sources` below. @@ -71,6 +71,8 @@ file. The available options are: Default: The `beets custom search engine`_, which searches the entire web. - **fanarttv_key**: The personal API key for requesting art from fanart.tv. See below. +- **lastfm_key**: The personal API key for requesting art from Last.fm. See + below. - **store_source**: If enabled, fetchart stores the artwork's source in a flexible tag named ``art_source``. See below for the rationale behind this. Default: ``no``. @@ -221,6 +223,15 @@ personal key will give you earlier access to new art. .. _on their blog: https://fanart.tv/2015/01/personal-api-keys/ +Last.fm +''''''' + +To use the Last.fm backend, you need to `register for a Last.fm API key`_. Set +the ``lastfm_key`` configuration option to your API key, then add ``lastfm`` to +the list of sources in your configutation. + +.. _register for a Last.fm API key: https://www.last.fm/api/account/create + Storing the Artwork's Source ---------------------------- From 1bec3c4c9f6f29289c1b0f3709289cc2cdccc6ad Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 30 Mar 2020 22:15:13 +0200 Subject: [PATCH 090/293] added all my plugins + corrected 1 typo --- docs/plugins/index.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 6c643ce61..76ed19fa3 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -300,12 +300,20 @@ Here are a few of the plugins written by the beets community: * `beet-summarize`_ can compute lots of counts and statistics about your music library. -* `beets-mosaic`_ generates a montage of a mosiac from cover art. +* `beets-mosaic`_ generates a montage of a mosaic from cover art. + +* `beets-goingrunning`_ generates playlists to go with your running sessions. + +* `beets-xtractor`_ extracts low- and high-level musical information from your songs. + +* `beets-yearfixer`_ attempts to fix all missing ``original_year`` and ``year`` fields. + +* `beets-autofix`_ automates repetitive tasks to keep your library in order. + +* `beets-describe`_ gives you the full picture of a single attribute of your library items. * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). -* `beets-goingrunning`_ copies songs to external device to go with your running session. - .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -328,5 +336,9 @@ Here are a few of the plugins written by the beets community: .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl .. _beet-summarize: https://github.com/steven-murray/beet-summarize .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic +.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning +.. _beets-xtractor: https://github.com/adamjakab/BeetsPluginXtractor +.. _beets-yearfixer: https://github.com/adamjakab/BeetsPluginYearFixer +.. _beets-autofix: https://github.com/adamjakab/BeetsPluginAutofix +.. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser -.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning/ From 8908416f6f66d2596a08daefeba0144cb27a7b98 Mon Sep 17 00:00:00 2001 From: x1ppy <59340737+x1ppy@users.noreply.github.com> Date: Mon, 6 Apr 2020 17:00:17 -0400 Subject: [PATCH 091/293] Add beets-originquery plugin --- docs/plugins/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 76ed19fa3..1247d0ed0 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -314,6 +314,9 @@ Here are a few of the plugins written by the beets community: * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). +* `beets-originquery`_ augments MusicBrainz queries with locally-sourced data + to improve autotagger results. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -342,3 +345,4 @@ Here are a few of the plugins written by the beets community: .. _beets-autofix: https://github.com/adamjakab/BeetsPluginAutofix .. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser +.. _beets-originquery: https://github.com/x1ppy/beets-originquery From 3c089117940de5d66504400db8b37603d98b60bf Mon Sep 17 00:00:00 2001 From: x1ppy <59340737+x1ppy@users.noreply.github.com> Date: Mon, 6 Apr 2020 17:01:11 -0400 Subject: [PATCH 092/293] Add missing extra_tags parameter to MetadataSourcePlugin (#3540) --- beets/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 8606ebc69..695725cb8 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -715,7 +715,7 @@ class MetadataSourcePlugin(object): return id_ return None - def candidates(self, items, artist, album, va_likely): + def candidates(self, items, artist, album, va_likely, extra_tags=None): """Returns a list of AlbumInfo objects for Search API results matching an ``album`` and ``artist`` (if not various). From 8e28f0b694bc04fa0735c8d3f6feb8232647c8d7 Mon Sep 17 00:00:00 2001 From: lijacky Date: Sun, 12 Apr 2020 00:05:17 -0400 Subject: [PATCH 093/293] added null check for genius lyrics scrape --- beetsplug/lyrics.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0e797d5a3..bb225007d 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,7 +373,13 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. - lyrics = html.find("div", class_="lyrics").get_text() + try: + lyrics = html.find("div", class_="lyrics").get_text() + except AttributeError as exc: + # html is a NoneType cannot retrieve lyrics + self._log.debug(u'Genius lyrics for {0} not found: {1}', + page_url, exc) + return None return lyrics From 29d7b80847fd4f1d5c4f47d50d758160d6866c39 Mon Sep 17 00:00:00 2001 From: lijacky Date: Wed, 15 Apr 2020 22:23:27 -0400 Subject: [PATCH 094/293] Added unit test for null check --- test/test_lyrics.py | 61 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index f7ea538e2..e72b2d4f8 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -24,7 +24,7 @@ import sys import unittest from mock import patch -from test import _common +import _common from beets import logging from beets.library import Item @@ -39,6 +39,7 @@ from mock import MagicMock log = logging.getLogger('beets.test_lyrics') raw_backend = lyrics.Backend({}, log) google = lyrics.Google(MagicMock(), log) +genius = lyrics.Genius(MagicMock(), log) class LyricsPluginTest(unittest.TestCase): @@ -213,6 +214,29 @@ class MockFetchUrl(object): content = f.read() return content +class GeniusMockGet(object): + def __init__(self, pathval='fetched_path'): + self.pathval = pathval + self.fetched = None + + def __call__(self, url, headers=False): + from requests.models import Response + # for the first requests.get() return a path + if headers: + response = Response() + response.status_code = 200 + response._content = b'{"meta":{"status":200},"response":{"song":{"path":"/lyrics/sample"}}}' + return response + # for the second requests.get() return the genius page + else: + from mock import PropertyMock + self.fetched = url + fn = url_to_filename(url) + with open(fn, 'r') as f: + content = f.read() + response = Response() + type(response).text = PropertyMock(return_value=content) + return response def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" @@ -395,6 +419,41 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): google.is_page_candidate(url, url_title, s['title'], u'Sunn O)))') +class LyricsGeniusBaseTest(unittest.TestCase): + + def setUp(self): + """Set up configuration.""" + try: + __import__('bs4') + except ImportError: + self.skipTest('Beautiful Soup 4 not available') + if sys.version_info[:3] < (2, 7, 3): + self.skipTest("Python's built-in HTML parser is not good enough") + + +class LyricsGeniusScrapTest(LyricsGeniusBaseTest): + """Checks that Genius backend works as intended. + """ + import requests + def setUp(self): + """Set up configuration""" + LyricsGeniusBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @patch.object(requests, 'get', GeniusMockGet()) + def test_no_lyrics_div(self): + """Ensure that `lyrics_from_song_api_path` doesn't crash when the html + for a Genius page contain
...
+ """ + # https://github.com/beetbox/beets/issues/3535 + # expected return value None + try: + self.assertEqual(genius.lyrics_from_song_api_path('/no_lyric_page'), None) + except AttributeError: + # if AttributeError we aren't doing a null check + self.assertTrue(False) + + class SlugTests(unittest.TestCase): def test_slug(self): From 525202e52919e971781a867f20d49b1548161d43 Mon Sep 17 00:00:00 2001 From: lijacky Date: Wed, 15 Apr 2020 22:30:31 -0400 Subject: [PATCH 095/293] adding genius sample html --- test/rsrc/lyrics/geniuscom/sample.txt | 1119 +++++++++++++++++++++++++ 1 file changed, 1119 insertions(+) create mode 100644 test/rsrc/lyrics/geniuscom/sample.txt diff --git a/test/rsrc/lyrics/geniuscom/sample.txt b/test/rsrc/lyrics/geniuscom/sample.txt new file mode 100644 index 000000000..da8a4d0b6 --- /dev/null +++ b/test/rsrc/lyrics/geniuscom/sample.txt @@ -0,0 +1,1119 @@ + + + + + + + +SAMPLE – SONG Lyrics | Genius Lyrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + {{:: 'cloud_flare_always_on_short_message' | i18n }} +
Check @genius for updates. We'll have things fixed soon. + +
+
+ + +
+ GENIUS +
+ + + +
+ + + + + + + + + + + +
+ + +
+ + + +
+ + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+
+
+ Https%3a%2f%2fimages +
+
+ +
+
+ +

SONG

+

+ SAMPLE +

+ +

+ + +

+

+ +
+ Produced by + + + Carter Lang & The Antydote + +
+ + +

+

+ +
+ Album + ALBUM +
+ + +

+ +
+
+
+
+
+
+
+ + +
+
+
+ +

SONG Lyrics

+ +
+
+ + +

[Verse 1]
+Fearin' not growin' up
+Keepin' me up at night
+Am I doin' enough?
+Feel like I'm wastin' time
+
+[Pre-Chorus]
+Promise to get a little
+Better as I get older
+And you're so patient
+And sick of waitin'
+Promise to do better
+Shoulda coulda
+Prolly wanna let me go
+But you can't, oh
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+
+[Chorus]
+Please don't take it, don't take it personal
+Like I know you usually do
+Please don't take it, don't take it personal
+Like I know you usually do
+Please, please
+Don't take it personal
+Don't take it personal
+Darling, like I know you will, ooh
+
+[Verse 2]
+Forget to call your mama on the weekend
+You should put yourself in time out
+(Shame, shame on you)
+But lately you've been feelin' so good
+I forget my future, never pull out
+(Shame, shame on me)
+Baby the money'll make it easier for me
+To run and hide out somewhere
+(So far away)
+Hoppin' through poppy fields
+Dodgin' evil witches
+These houses keep droppin' everywhere
+
+[Pre-Chorus]
+Promise to get a little
+Better as I get older
+And you're so patient
+And sick of waitin'
+Promise to do better
+Shoulda coulda
+Prolly wanna let me go
+But you can't, oh
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+Right now it's really pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+
+[Chorus]
+Please don't take it, don't take it personal
+Like I know you usually do
+Please don't take it, take it personal
+
+[Outro]
+Like winters fall on us, heavy
+Take it off me, all it off
+Winter, I can't stand this
+Snow is falling all on me

+ + + + +
+
+ +
+ +
+
More on Genius
+ +
+ +
+
+
+ +
+
+ +
+
+
+ + + +
+ +

+ About “SONG” +

+ + +
+
+

The fifth track on SAMPLE’s debut album, ALBUM is the most reminiscent of her previous ep’s with a pop-disco sound.

+ +

She reflects about her insecurities on not maturing as quickly as her partner is. SONG season is usually the last big event before graduation for a high schooler, and SAMPLE is too busy living in the moment that she forgets she has her future to look forward to.

+ +

+ +

This is also around the age when friends are growing apart and have decided where and what they’re doing with their lives, and a lot are trying to hold on to the people they’ve known, even though they’re growing up differently. The person SAMPLE is talking to in the song seems to take everything, even change, to heart.

+
+
+
+ +
+
+
    + +
  • +
    +

    What have the artists said about this song

    +
    + +
    +

    SAMPLE tweeted:

    + +

    Iss called SONG cus I ain’t have no friends in high school so I went to Miami w my mama for SONG instead . Lol

    +
    + + +
    • + + + +
+
+ + +
+
+

"SONG" Track Info

+ + +
+ Written By + + + The Antydote, Carter Lang & SAMPLE + +
+ + + + + +
+ Released by + + + RCA Records & Top Dawg Entertainment + +
+ + + + +
+ Mixing + + + Mixed by Ali + +
+ + + + + +
+ Recorded At + TDE Red Room, Carson, California +
+ + + +
+ Release Date + June 9, 2017 +
+ + + + + + + + + + + + + + + + + + +
+ Interpolated By + + +
+ Wicked Games (StarFox Remix) by The Weeknd +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+
+
+ + Https%3a%2f%2fimages + +
+ + ALBUM + + + SAMPLE +
+
+ +
+ +
+
+ + 1.   + + + + # + + + + +
+
+ +
+
+ + 2.   + + + + # + + + + +
+
+ +
+
+ + 3.   + + + + # + + + + +
+
+ +
+
+ + 4.   + + + + # + + + + +
+
+ +
+
+ + 5.   + + + + SONG + + + + +
+
+ +
+
+ + 6.   + + + + # + + + + +
+
+ +
+
+ + 7.   + + + + # + + + + +
+
+ +
+
+ + 8.   + + + + # + + + + +
+
+ +
+
+ + 9.   + + + + # + + + + +
+
+ +
+
+ + 10.   + + + + # + + + + +
+
+ +
+
+ + 11.   + + + + # + + + + +
+
+ +
+
+ + 12.   + + + + # + + + + +
+
+ +
+
+ + 13.   + + + + # + + + + +
+
+ +
+
+ + 14.   + + + + # + + + + +
+
+ + + +
+ +
+
+ +
+
+
+ + + + + + + + + +
+
+ + + + +
+ © 2020 Genius Media Group Inc. +
+
+ +
+ Verified Artists + + All Artists: +
    + +
  • + A +
    • + +
  • + B +
    • + +
  • + C +
    • + +
  • + D +
    • + +
  • + E +
    • + +
  • + F +
    • + +
  • + G +
    • + +
  • + H +
    • + +
  • + I +
    • + +
  • + J +
    • + +
  • + K +
    • + +
  • + L +
    • + +
  • + M +
    • + +
  • + N +
    • + +
  • + O +
    • + +
  • + P +
    • + +
  • + Q +
    • + +
  • + R +
    • + +
  • + S +
    • + +
  • + T +
    • + +
  • + U +
    • + +
  • + V +
    • + +
  • + W +
    • + +
  • + X +
    • + +
  • + Y +
    • + +
  • + Z +
    • + +
  • + # +
    • + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + From b7ecf32f2876848611dcced77c0a2624198d35a9 Mon Sep 17 00:00:00 2001 From: lijacky Date: Thu, 16 Apr 2020 17:20:08 -0400 Subject: [PATCH 096/293] style changes --- test/test_lyrics.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index e72b2d4f8..6c23c627a 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -24,7 +24,7 @@ import sys import unittest from mock import patch -import _common +from test import _common from beets import logging from beets.library import Item @@ -214,7 +214,9 @@ class MockFetchUrl(object): content = f.read() return content + class GeniusMockGet(object): + def __init__(self, pathval='fetched_path'): self.pathval = pathval self.fetched = None @@ -225,7 +227,8 @@ class GeniusMockGet(object): if headers: response = Response() response.status_code = 200 - response._content = b'{"meta":{"status":200},"response":{"song":{"path":"/lyrics/sample"}}}' + response._content = b'{"meta":{"status":200},\ + "response":{"song":{"path":"/lyrics/sample"}}}' return response # for the second requests.get() return the genius page else: @@ -238,6 +241,7 @@ class GeniusMockGet(object): type(response).text = PropertyMock(return_value=content) return response + def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" if not text: @@ -432,9 +436,11 @@ class LyricsGeniusBaseTest(unittest.TestCase): class LyricsGeniusScrapTest(LyricsGeniusBaseTest): + """Checks that Genius backend works as intended. """ import requests + def setUp(self): """Set up configuration""" LyricsGeniusBaseTest.setUp(self) @@ -443,12 +449,13 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): """Ensure that `lyrics_from_song_api_path` doesn't crash when the html - for a Genius page contain
...
+ for a Genius page contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None try: - self.assertEqual(genius.lyrics_from_song_api_path('/no_lyric_page'), None) + self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + None) except AttributeError: # if AttributeError we aren't doing a null check self.assertTrue(False) From 9ec0d725e52a06863ac733e2f560dbc2b08fc014 Mon Sep 17 00:00:00 2001 From: lijacky Date: Fri, 17 Apr 2020 17:14:21 -0400 Subject: [PATCH 097/293] Changes given feedback on https://github.com/beetbox/beets/pull/3554 and trimmed sample html --- beetsplug/lyrics.py | 13 +- test/rsrc/lyrics/geniuscom/sample.txt | 1289 +++++-------------------- test/test_lyrics.py | 8 +- 3 files changed, 229 insertions(+), 1081 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index bb225007d..00b8820f4 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,13 +373,14 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. - try: - lyrics = html.find("div", class_="lyrics").get_text() - except AttributeError as exc: - # html is a NoneType cannot retrieve lyrics - self._log.debug(u'Genius lyrics for {0} not found: {1}', - page_url, exc) + lyrics_div = html.find("div", class_="lyrics") + + # nullcheck + if lyrics_div is None: + self._log.debug(u'Genius lyrics for {0} not found', + page_url) return None + lyrics = lyrics_div.get_text() return lyrics diff --git a/test/rsrc/lyrics/geniuscom/sample.txt b/test/rsrc/lyrics/geniuscom/sample.txt index da8a4d0b6..1648d070a 100644 --- a/test/rsrc/lyrics/geniuscom/sample.txt +++ b/test/rsrc/lyrics/geniuscom/sample.txt @@ -1,1119 +1,270 @@ - + + //]]> + -SAMPLE – SONG Lyrics | Genius Lyrics + SAMPLE – SONG Lyrics | g-example Lyrics - - + + - + - + + + - + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ga('create', "UA-10346621-1", 'auto', {'useAmpClientId': true}); + ga('set', 'dimension1', "false"); + ga('set', 'dimension2', "songs#show"); + ga('set', 'dimension3', "r-b"); + ga('set', 'dimension4', "true"); + ga('set', 'dimension5', 'false'); + ga('set', 'dimension6', "none"); + ga('send', 'pageview'); + + - - + - - +
+
+
- - - - -
- - {{:: 'cloud_flare_always_on_short_message' | i18n }} -
Check @genius for updates. We'll have things fixed soon. - -
-
- - -
- GENIUS -
- - - -
- - - - - - - - - - - -
- - -
- - - -
- - - - - - - - - - - - - - - - -
- -
- -
-
-
-
-
- Https%3a%2f%2fimages -
-
- -
-
- -

SONG

-

- SAMPLE -

- -

- - -

-

- -
- Produced by - - - Carter Lang & The Antydote - -
- - -

-

- -
- Album - ALBUM -
- - -

- -
-
-
-
-
-
-
- - -
-
-
- -

SONG Lyrics

- -
-
- - -

[Verse 1]
-Fearin' not growin' up
-Keepin' me up at night
-Am I doin' enough?
-Feel like I'm wastin' time
-
-[Pre-Chorus]
-Promise to get a little
-Better as I get older
-And you're so patient
-And sick of waitin'
-Promise to do better
-Shoulda coulda
-Prolly wanna let me go
-But you can't, oh
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-
-[Chorus]
-Please don't take it, don't take it personal
-Like I know you usually do
-Please don't take it, don't take it personal
-Like I know you usually do
-Please, please
-Don't take it personal
-Don't take it personal
-Darling, like I know you will, ooh
-
-[Verse 2]
-Forget to call your mama on the weekend
-You should put yourself in time out
-(Shame, shame on you)
-But lately you've been feelin' so good
-I forget my future, never pull out
-(Shame, shame on me)
-Baby the money'll make it easier for me
-To run and hide out somewhere
-(So far away)
-Hoppin' through poppy fields
-Dodgin' evil witches
-These houses keep droppin' everywhere
-
-[Pre-Chorus]
-Promise to get a little
-Better as I get older
-And you're so patient
-And sick of waitin'
-Promise to do better
-Shoulda coulda
-Prolly wanna let me go
-But you can't, oh
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-Right now it's really pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-
-[Chorus]
-Please don't take it, don't take it personal
-Like I know you usually do
-Please don't take it, take it personal
-
-[Outro]
-Like winters fall on us, heavy
-Take it off me, all it off
-Winter, I can't stand this
-Snow is falling all on me

- - - - -
-
- -
-
-
More on Genius
- +
+ g-example
+ + + +
-
-
-
- -
-
-
+ + - +
+
+
+
+
+ # +
+
-
+
+
+

SONG

+

+ + SAMPLE + +

+

+
+ Produced by + + Person 1 & Person 2 + +
+

+

+
+ Album + ALBUM +
+

+
+
+ +
+
+
+ +
+
+
+

SONG Lyrics

+
+
+ !!!! MISSING LYRICS HERE !!! +
+
+
+
+
More on g-example
+
+
+
+
+
-

- About “SONG” -

- - -
-
-

The fifth track on SAMPLE’s debut album, ALBUM is the most reminiscent of her previous ep’s with a pop-disco sound.

+
+ Released by -

She reflects about her insecurities on not maturing as quickly as her partner is. SONG season is usually the last big event before graduation for a high schooler, and SAMPLE is too busy living in the moment that she forgets she has her future to look forward to.

- -

- -

This is also around the age when friends are growing apart and have decided where and what they’re doing with their lives, and a lot are trying to hold on to the people they’ve known, even though they’re growing up differently. The person SAMPLE is talking to in the song seems to take everything, even change, to heart.

-
-
-
- -
-
-
    - -
  • -
    -

    What have the artists said about this song

    -
    - -
    -

    SAMPLE tweeted:

    - -

    Iss called SONG cus I ain’t have no friends in high school so I went to Miami w my mama for SONG instead . Lol

    -
    - - -
    • - - - -
-
- - -
-
-

"SONG" Track Info

- - -
- Written By - - - The Antydote, Carter Lang & SAMPLE - -
- - - - - -
- Released by - - - RCA Records & Top Dawg Entertainment - -
- - - - -
- Mixing - - - Mixed by Ali - -
- - - - - -
- Recorded At - TDE Red Room, Carson, California -
- - - -
- Release Date - June 9, 2017 -
- - - - - - - - - - - - - - - - - - -
- Interpolated By - - -
- Wicked Games (StarFox Remix) by The Weeknd + + Records & Top +
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - +
+ Mixing + + Mixed by Person + +
+ +
+ Recorded At + City, Place +
+ +
+ Release Date + Feb 30, 1290 +
+ +
+ Interpolated By + + +
+ # +
+ +
-
-
- - Https%3a%2f%2fimages - -
- - ALBUM - - - SAMPLE -
-
- -
- -
-
- - 1.   - - - - # - - - - -
-
- -
-
- - 2.   - - - - # - - - - -
-
- -
-
- - 3.   - - - - # - - - - -
-
- -
-
- - 4.   - - - - # - - - - -
-
- -
-
- - 5.   - - - - SONG - - - - -
-
- -
-
- - 6.   - - - - # - - - - -
-
- -
-
- - 7.   - - - - # - - - - -
-
- -
-
- - 8.   - - - - # - - - - -
-
- -
-
- - 9.   - - - - # - - - - -
-
- -
-
- - 10.   - - - - # - - - - -
-
- -
-
- - 11.   - - - - # - - - - -
-
- -
-
- - 12.   - - - - # - - - - -
-
- -
-
- - 13.   - - - - # - - - - -
-
- -
-
- - 14.   - - - - # - - - - -
-
- - - -
- + + # + +
+ + ALBUM + + SAMPLE +
-
- -
-
-
- - - - - - - + +
-
- -
- Privacy Policy - - - Licensing - Jobs - Developers - Terms of Use - Copyright Policy - Contact us - - Do Not Sell My Personal Information - - - Sign in + + + - All Artists: -
    - -
  • - A -
    • - -
  • - B -
    • - -
  • - C -
    • - -
  • - D -
    • - -
  • - E -
    • - -
  • - F -
    • - -
  • - G -
    • - -
  • - H -
    • - -
  • - I -
    • - -
  • - J -
    • - -
  • - K -
    • - -
  • - L -
    • - -
  • - M -
    • - -
  • - N -
    • - -
  • - O -
    • - -
  • - P -
    • - -
  • - Q -
    • - -
  • - R -
    • - -
  • - S -
    • - -
  • - T -
    • - -
  • - U -
    • - -
  • - V -
    • - -
  • - W -
    • - -
  • - X -
    • - -
  • - Y -
    • - -
  • - Z -
    • - -
  • - # -
    • - -
- -
- -
- - - - - - - - - - - - - - - - - + + + + + - diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 6c23c627a..83a229f62 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -453,12 +453,8 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): """ # https://github.com/beetbox/beets/issues/3535 # expected return value None - try: - self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), - None) - except AttributeError: - # if AttributeError we aren't doing a null check - self.assertTrue(False) + self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + None) class SlugTests(unittest.TestCase): From 175c392ef33928f7decffed038cce90cdf00a7b6 Mon Sep 17 00:00:00 2001 From: lijacky Date: Fri, 17 Apr 2020 17:25:36 -0400 Subject: [PATCH 098/293] add fix to changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 33ad386ad..fd0c225f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,6 +178,8 @@ Fixes: * Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method failing with AttributeError when called. It is now an instance method. :bug:`3516` :bug:`3517` +* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper + :bug:`3535` For plugin developers: From 269e309513b54ab362a54f0a82851ac29630823f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 17 Apr 2020 18:28:21 -0400 Subject: [PATCH 099/293] Slight simplification for #3554 --- beetsplug/lyrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 00b8820f4..cca0fa94e 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -374,15 +374,12 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. lyrics_div = html.find("div", class_="lyrics") - - # nullcheck if lyrics_div is None: self._log.debug(u'Genius lyrics for {0} not found', page_url) return None - lyrics = lyrics_div.get_text() - return lyrics + return lyrics_div.get_text() def fetch(self, artist, title): search_url = self.base_url + "/search" From c93ea0b36b89bb7e62d7fe6cbd068c80ca6f9556 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 17 Apr 2020 18:29:27 -0400 Subject: [PATCH 100/293] Link to fix in #3554 --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fd0c225f9..6bb08b143 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,8 +178,9 @@ Fixes: * Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method failing with AttributeError when called. It is now an instance method. :bug:`3516` :bug:`3517` -* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper - :bug:`3535` +* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper. + Thanks to :user:`thejli21`. + :bug:`3535` :bug:`3554` For plugin developers: From a5ef8c7b09db2dc95df269de158d8f34f33ad015 Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 19 Apr 2020 00:50:14 +0530 Subject: [PATCH 101/293] Add artist_sort as the last entry in the artists to search This will handle cases where the artist name includes some characters which can cause a search failure for some websites - mainly unicode. Fixes https://github.com/beetbox/beets/issues/3340. --- beetsplug/lyrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index cca0fa94e..19acbcadc 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -200,12 +200,13 @@ def search_pairs(item): alternatives.append(match.group(1)) return alternatives - title, artist = item.title, item.artist + title, artist, artist_sort = item.title, item.artist, item.artist_sort patterns = [ # Remove any featuring artists from the artists name r"(.*?) {0}".format(plugins.feat_tokens())] artists = generate_alternatives(artist, patterns) + artists.append(artist_sort) patterns = [ # Remove a parenthesized suffix from a title string. Common From 45c8e51d6cd92574d0dafffe73a98ad21a627aad Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 19 Apr 2020 19:08:42 +0530 Subject: [PATCH 102/293] Fix redundant requests for artist == artist_sort --- beetsplug/lyrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 19acbcadc..0aebe9606 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -206,7 +206,10 @@ def search_pairs(item): # Remove any featuring artists from the artists name r"(.*?) {0}".format(plugins.feat_tokens())] artists = generate_alternatives(artist, patterns) - artists.append(artist_sort) + # Use the artist_sort as fallback only if it differs from artist to avoid + # repeated remote requests with the same search terms + if artist != artist_sort: + artists.append(artist_sort) patterns = [ # Remove a parenthesized suffix from a title string. Common From 625e9253c03bc4f2c5cab6c54e014cfbe8bca49e Mon Sep 17 00:00:00 2001 From: Ashhar Hasan Date: Sun, 19 Apr 2020 19:05:55 +0530 Subject: [PATCH 103/293] Add tests for artist_sort as lyrics search fallback Adjust doc comment to highlight that artist_sort is used as a fallback --- beetsplug/lyrics.py | 3 +++ test/test_lyrics.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0aebe9606..20e1c8858 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -187,6 +187,9 @@ def search_pairs(item): In addition to the artist and title obtained from the `item` the method tries to strip extra information like paranthesized suffixes and featured artists from the strings and add them as candidates. + The artist sort name is added as a fallback candidate to help in + cases where artist name includes special characters or is in a + non-latin script. The method also tries to split multiple titles separated with `/`. """ def generate_alternatives(string, patterns): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 83a229f62..81d62fea9 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -95,6 +95,27 @@ class LyricsPluginTest(unittest.TestCase): self.assertEqual(('Alice and Bob', ['song']), list(lyrics.search_pairs(item))[0]) + def test_search_artist_sort(self): + item = Item(artist='CHVRCHΞS', title='song', artist_sort='CHVRCHES') + self.assertIn(('CHVRCHΞS', ['song']), + lyrics.search_pairs(item)) + self.assertIn(('CHVRCHES', ['song']), + lyrics.search_pairs(item)) + + # Make sure that the original artist name is still the first entry + self.assertEqual(('CHVRCHΞS', ['song']), + list(lyrics.search_pairs(item))[0]) + + item = Item(artist='横山克', title='song', artist_sort='Masaru Yokoyama') + self.assertIn(('横山克', ['song']), + lyrics.search_pairs(item)) + self.assertIn(('Masaru Yokoyama', ['song']), + lyrics.search_pairs(item)) + + # Make sure that the original artist name is still the first entry + self.assertEqual(('横山克', ['song']), + list(lyrics.search_pairs(item))[0]) + def test_search_pairs_multi_titles(self): item = Item(title='1 / 2', artist='A') self.assertIn(('A', ['1 / 2']), lyrics.search_pairs(item)) From 425bbc366ff505e44777dd308c06af47382caaa9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 19 Apr 2020 11:12:24 -0400 Subject: [PATCH 104/293] Changelog/thanks for #3558 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6bb08b143..69f855507 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -181,6 +181,10 @@ Fixes: * :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper. Thanks to :user:`thejli21`. :bug:`3535` :bug:`3554` +* :doc:`/plugins/lyrics`: Use the artist sort name to search for lyrics, which + can help find matches when the artist name has special characters. + Thanks to :user:`hashhar`. + :bug:`3340` :bug:`3558` For plugin developers: From 1c03f58e0d606184d68b219daa4aa846834a157a Mon Sep 17 00:00:00 2001 From: Kylie McClain Date: Sat, 18 Apr 2020 22:23:53 -0400 Subject: [PATCH 105/293] fetchart: improve description of -q flag --- beetsplug/fetchart.py | 2 +- docs/plugins/fetchart.rst | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 28067c310..763440238 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -979,7 +979,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): cmd.parser.add_option( u'-q', u'--quiet', dest='quiet', action='store_true', default=False, - help=u'shows only quiet art' + help=u'quiet mode: do not output albums that already have artwork' ) def func(lib, opts, args): diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index e8f7b6d92..92ed5f32a 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -126,8 +126,9 @@ art:: $ beet fetchart [-q] [query] -By default the command will display all results, the ``-q`` or ``--quiet`` -switch will only display results for album arts that are still missing. +By default the command will display all albums matching the ``query``. When the +``-q`` or ``--quiet`` switch is given, only albums for which artwork has been +fetched, or for which artwork could not be found will be printed. .. _image-resizing: From 5143e630d8e23ba69893988bbd7d635629809ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Koutensk=C3=BD?= Date: Wed, 22 Apr 2020 19:44:47 +0200 Subject: [PATCH 106/293] Fix #3533 - crash when calculating RG/R128 mix --- beetsplug/replaygain.py | 8 ++++---- docs/changelog.rst | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 646e3acce..916541df6 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1326,10 +1326,10 @@ class ReplayGainPlugin(BeetsPlugin): if (any([self.should_use_r128(item) for item in album.items()]) and not all(([self.should_use_r128(item) for item in album.items()]))): - raise ReplayGainError( - u"Mix of ReplayGain and EBU R128 detected" - u" for some tracks in album {0}".format(album) - ) + self._log.info( + u"Cannot calculate gain for album {0} (incompatible formats)", + album) + return tag_vals = self.tag_specific_values(album.items()) store_track_gain, store_album_gain, target_level, peak = tag_vals diff --git a/docs/changelog.rst b/docs/changelog.rst index 69f855507..e2f1521eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -185,6 +185,12 @@ Fixes: can help find matches when the artist name has special characters. Thanks to :user:`hashhar`. :bug:`3340` :bug:`3558` +* :doc:`/plugins/replaygain`: Trying to calculate volume gain for an album + consisting of some formats using ``ReplayGain`` and some using ``R128`` + will no longer crash; instead it is skipped and and a message is logged. + The log message has also been rewritten for to improve clarity. + Thanks to :user:`autrimpo`. + :bug:`3533` For plugin developers: From 47deb2f084ab293ead30d99a7b874e2e23317531 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 23 Apr 2020 08:03:51 -0400 Subject: [PATCH 107/293] Switch log level for #3563, fix #3533 --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 916541df6..6f15b454e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1326,7 +1326,7 @@ class ReplayGainPlugin(BeetsPlugin): if (any([self.should_use_r128(item) for item in album.items()]) and not all(([self.should_use_r128(item) for item in album.items()]))): - self._log.info( + self._log.error( u"Cannot calculate gain for album {0} (incompatible formats)", album) return From 981d4dc829b91c8bbc3d1821a93a6c68cedc9e86 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 19:50:59 +0200 Subject: [PATCH 108/293] First try adding new albuminfo and trackinfo class --- beets/autotag/hooks.py | 171 ++++++++++++----------------------------- 1 file changed, 51 insertions(+), 120 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index d7c701db6..2555ed69f 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -39,8 +39,51 @@ except AttributeError: # Classes used to represent candidate options. +class Map(dict): + """ + Example: + m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) + """ + def __init__(self, *args, **kwargs): + super(Map, self).__init__(*args, **kwargs) + for arg in args: + if isinstance(arg, dict): + for k, v in arg.iteritems(): + self[k] = v + + if kwargs: + for k, v in kwargs.iteritems(): + self[k] = v + + def __getattr__(self, attr): + return self.get(attr) + + def __setattr__(self, key, value): + self.__setitem__(key, value) + + def __setitem__(self, key, value): + super(Map, self).__setitem__(key, value) + self.__dict__.update({key: value}) + + def __delattr__(self, item): + self.__delitem__(item) + + def __delitem__(self, key): + super(Map, self).__delitem__(key) + del self.__dict__[key] + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self,state): + for key in state: + self.__setattr__(key,state[key]) + def __hash__(self): + return hash(tuple(sorted(self.items()))) -class AlbumInfo(object): + + +class AlbumInfo(Map): """Describes a canonical release that may be used to match a release in the library. Consists of these data members: @@ -49,77 +92,18 @@ class AlbumInfo(object): - ``artist``: name of the release's primary artist - ``artist_id`` - ``tracks``: list of TrackInfo objects making up the release - - ``asin``: Amazon ASIN - - ``albumtype``: string describing the kind of release - - ``va``: boolean: whether the release has "various artists" - - ``year``: release year - - ``month``: release month - - ``day``: release day - - ``label``: music label responsible for the release - - ``mediums``: the number of discs in this release - - ``artist_sort``: name of the release's artist for sorting - - ``releasegroup_id``: MBID for the album's release group - - ``catalognum``: the label's catalog number for the release - - ``script``: character set used for metadata - - ``language``: human language of the metadata - - ``country``: the release country - - ``albumstatus``: MusicBrainz release status (Official, etc.) - - ``media``: delivery mechanism (Vinyl, etc.) - - ``albumdisambig``: MusicBrainz release disambiguation comment - - ``releasegroupdisambig``: MusicBrainz release group - disambiguation comment. - - ``artist_credit``: Release-specific artist name - - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) - - ``data_url``: The data source release URL. ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, album, album_id, artist, artist_id, tracks, asin=None, - albumtype=None, va=False, year=None, month=None, day=None, - label=None, mediums=None, artist_sort=None, - releasegroup_id=None, catalognum=None, script=None, - language=None, country=None, style=None, genre=None, - albumstatus=None, media=None, albumdisambig=None, - releasegroupdisambig=None, artist_credit=None, - original_year=None, original_month=None, - original_day=None, data_source=None, data_url=None, - discogs_albumid=None, discogs_labelid=None, - discogs_artistid=None): + def __init__(self, album, album_id, artist, artist_id, tracks, **kwargs): self.album = album self.album_id = album_id self.artist = artist self.artist_id = artist_id self.tracks = tracks - self.asin = asin - self.albumtype = albumtype - self.va = va - self.year = year - self.month = month - self.day = day - self.label = label - self.mediums = mediums - self.artist_sort = artist_sort - self.releasegroup_id = releasegroup_id - self.catalognum = catalognum - self.script = script - self.language = language - self.country = country - self.style = style - self.genre = genre - self.albumstatus = albumstatus - self.media = media - self.albumdisambig = albumdisambig - self.releasegroupdisambig = releasegroupdisambig - self.artist_credit = artist_credit - self.original_year = original_year - self.original_month = original_month - self.original_day = original_day - self.data_source = data_source - self.data_url = data_url - self.discogs_albumid = discogs_albumid - self.discogs_labelid = discogs_labelid - self.discogs_artistid = discogs_artistid + for arg in kwargs: + self.__setattr__(arg,kwargs[arg]) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -143,75 +127,22 @@ class AlbumInfo(object): track.decode(codec) -class TrackInfo(object): +class TrackInfo(Map): """Describes a canonical track present on a release. Appears as part of an AlbumInfo's ``tracks`` list. Consists of these data members: - ``title``: name of the track - ``track_id``: MusicBrainz ID; UUID fragment only - - ``release_track_id``: MusicBrainz ID respective to a track on a - particular release; UUID fragment only - - ``artist``: individual track artist name - - ``artist_id`` - - ``length``: float: duration of the track in seconds - - ``index``: position on the entire release - - ``media``: delivery mechanism (Vinyl, etc.) - - ``medium``: the disc number this track appears on in the album - - ``medium_index``: the track's position on the disc - - ``medium_total``: the number of tracks on the item's disc - - ``artist_sort``: name of the track artist for sorting - - ``disctitle``: name of the individual medium (subtitle) - - ``artist_credit``: Recording-specific artist name - - ``data_source``: The original data source (MusicBrainz, Discogs, etc.) - - ``data_url``: The data source release URL. - - ``lyricist``: individual track lyricist name - - ``composer``: individual track composer name - - ``composer_sort``: individual track composer sort name - - ``arranger`: individual track arranger name - - ``track_alt``: alternative track number (tape, vinyl, etc.) - - ``work`: individual track work title - - ``mb_workid`: individual track work id - - ``work_disambig`: individual track work diambiguation Only ``title`` and ``track_id`` are required. The rest of the fields may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, title, track_id, release_track_id=None, artist=None, - artist_id=None, length=None, index=None, medium=None, - medium_index=None, medium_total=None, artist_sort=None, - disctitle=None, artist_credit=None, data_source=None, - data_url=None, media=None, lyricist=None, composer=None, - composer_sort=None, arranger=None, track_alt=None, - work=None, mb_workid=None, work_disambig=None, bpm=None, - initial_key=None, genre=None): + def __init__(self, title, track_id,**kwargs): self.title = title self.track_id = track_id - self.release_track_id = release_track_id - self.artist = artist - self.artist_id = artist_id - self.length = length - self.index = index - self.media = media - self.medium = medium - self.medium_index = medium_index - self.medium_total = medium_total - self.artist_sort = artist_sort - self.disctitle = disctitle - self.artist_credit = artist_credit - self.data_source = data_source - self.data_url = data_url - self.lyricist = lyricist - self.composer = composer - self.composer_sort = composer_sort - self.arranger = arranger - self.track_alt = track_alt - self.work = work - self.mb_workid = mb_workid - self.work_disambig = work_disambig - self.bpm = bpm - self.initial_key = initial_key - self.genre = genre + for arg in kwargs: + self.__setattr__(arg,kwargs[arg]) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): From da43ff9c188276548c30237e6d93c6c45106bebe Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 20:43:30 +0200 Subject: [PATCH 109/293] arrange __getattr__ to behave normally --- beets/autotag/hooks.py | 94 ++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 2555ed69f..cc8e6a68a 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -39,49 +39,53 @@ except AttributeError: # Classes used to represent candidate options. -class Map(dict): - """ - Example: - m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) - """ - def __init__(self, *args, **kwargs): - super(Map, self).__init__(*args, **kwargs) - for arg in args: - if isinstance(arg, dict): - for k, v in arg.iteritems(): - self[k] = v - - if kwargs: - for k, v in kwargs.iteritems(): - self[k] = v - - def __getattr__(self, attr): - return self.get(attr) - - def __setattr__(self, key, value): - self.__setitem__(key, value) - - def __setitem__(self, key, value): - super(Map, self).__setitem__(key, value) - self.__dict__.update({key: value}) - - def __delattr__(self, item): - self.__delitem__(item) - - def __delitem__(self, key): - super(Map, self).__delitem__(key) - del self.__dict__[key] - - def __getstate__(self): - return self.__dict__ - - def __setstate__(self,state): - for key in state: - self.__setattr__(key,state[key]) - def __hash__(self): - return hash(tuple(sorted(self.items()))) +class Map(dict): + """ + Example: + m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, + sports=['Soccer']) + """ + def __init__(self, *args, **kwargs): + super(Map, self).__init__(*args, **kwargs) + for arg in args: + if isinstance(arg, dict): + for k, v in arg.iteritems(): + self[k] = v + + if kwargs: + for k, v in kwargs.iteritems(): + self[k] = v + + def __getattr__(self, attr): + if attr in self: + return self.get(attr) + else: + raise AttributeError + + def __setattr__(self, key, value): + self.__setitem__(key, value) + + def __setitem__(self, key, value): + super(Map, self).__setitem__(key, value) + self.__dict__.update({key: value}) + + def __delattr__(self, item): + self.__delitem__(item) + + def __delitem__(self, key): + super(Map, self).__delitem__(key) + del self.__dict__[key] + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, state): + for key in state: + self.__setattr__(key, state[key]) + + def __hash__(self): + return hash(tuple(sorted(self.items()))) - class AlbumInfo(Map): """Describes a canonical release that may be used to match a release @@ -103,7 +107,7 @@ class AlbumInfo(Map): self.artist_id = artist_id self.tracks = tracks for arg in kwargs: - self.__setattr__(arg,kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -138,11 +142,11 @@ class TrackInfo(Map): may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, title, track_id,**kwargs): + def __init__(self, title, track_id, **kwargs): self.title = title self.track_id = track_id for arg in kwargs: - self.__setattr__(arg,kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): From fea6ffc0386eed693fdeef27556635eef4d44bbd Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 22:24:24 +0200 Subject: [PATCH 110/293] arrange decode, set all attributes to be flexible --- beets/autotag/hooks.py | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index cc8e6a68a..dd9726b84 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -100,14 +100,9 @@ class AlbumInfo(Map): ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, album, album_id, artist, artist_id, tracks, **kwargs): - self.album = album - self.album_id = album_id - self.artist = artist - self.artist_id = artist_id - self.tracks = tracks + def __init__(self, **kwargs): for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -116,19 +111,11 @@ class AlbumInfo(Map): """Ensure that all string attributes on this object, and the constituent `TrackInfo` objects, are decoded to Unicode. """ - for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', - 'catalognum', 'script', 'language', 'country', 'style', - 'genre', 'albumstatus', 'albumdisambig', - 'releasegroupdisambig', 'artist_credit', - 'media', 'discogs_albumid', 'discogs_labelid', - 'discogs_artistid']: + for fld in self: value = getattr(self, fld) - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) - - if self.tracks: - for track in self.tracks: - track.decode(codec) + if type(value) == str: + if isinstance(value, bytes): + setattr(self, fld, value.decode(codec, 'ignore')) class TrackInfo(Map): @@ -142,22 +129,20 @@ class TrackInfo(Map): may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, title, track_id, **kwargs): - self.title = title - self.track_id = track_id + def __init__(self, **kwargs): for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.__setattr__(arg, kwargs[arg]) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): """Ensure that all string attributes on this object are decoded to Unicode. """ - for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', - 'artist_credit', 'media']: + for fld in self: value = getattr(self, fld) - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) + if type(value) == str: + if isinstance(value, bytes): + setattr(self, fld, value.decode(codec, 'ignore')) # Candidate distance scoring. From 53ce6f8b3d7d62c4ca54966042530f6c6e7d7c45 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 22:32:03 +0200 Subject: [PATCH 111/293] all attributes are flexible, so no positional arguments when initiating the class --- beets/autotag/mb.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index f86d3be71..ea8ef24da 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -193,8 +193,8 @@ def track_info(recording, index=None, medium=None, medium_index=None, the number of tracks on the medium. Each number is a 1-based index. """ info = beets.autotag.hooks.TrackInfo( - recording['title'], - recording['id'], + title=recording['title'], + track_id=recording['id'], index=index, medium=medium, medium_index=medium_index, @@ -341,11 +341,11 @@ def album_info(release): track_infos.append(ti) info = beets.autotag.hooks.AlbumInfo( - release['title'], - release['id'], - artist_name, - release['artist-credit'][0]['artist']['id'], - track_infos, + album=release['title'], + album_id=release['id'], + artist=artist_name, + artist_id=release['artist-credit'][0]['artist']['id'], + tracks=track_infos, mediums=len(release['medium-list']), artist_sort=artist_sort_name, artist_credit=artist_credit_name, From 1b2c8398b11ade93621583e06c0aea24b83c39b5 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 23:13:08 +0200 Subject: [PATCH 112/293] cleaning up beets/autotag/__init__.py --- beets/autotag/__init__.py | 154 ++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 65 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index f9e38413e..2dcf93c52 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -41,31 +41,10 @@ log = logging.getLogger('beets') def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ - item.artist = track_info.artist - item.artist_sort = track_info.artist_sort - item.artist_credit = track_info.artist_credit - item.title = track_info.title - item.mb_trackid = track_info.track_id - item.mb_releasetrackid = track_info.release_track_id - if track_info.artist_id: - item.mb_artistid = track_info.artist_id - if track_info.data_source: - item.data_source = track_info.data_source - - if track_info.lyricist is not None: - item.lyricist = track_info.lyricist - if track_info.composer is not None: - item.composer = track_info.composer - if track_info.composer_sort is not None: - item.composer_sort = track_info.composer_sort - if track_info.arranger is not None: - item.arranger = track_info.arranger - if track_info.work is not None: - item.work = track_info.work - if track_info.mb_workid is not None: - item.mb_workid = track_info.mb_workid - if track_info.work_disambig is not None: - item.work_disambig = track_info.work_disambig + print('zer' in track_info) + for attr in track_info: + print(attr in track_info) + item.__setattr__(attr, getattr(track_info, attr)) # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? @@ -78,25 +57,50 @@ def apply_metadata(album_info, mapping): for item, track_info in mapping.items(): # Artist or artist credit. if config['artist_credit']: - item.artist = (track_info.artist_credit or - track_info.artist or - album_info.artist_credit or - album_info.artist) - item.albumartist = (album_info.artist_credit or - album_info.artist) + + if 'artist_credit' in track_info: + item.artist = track_info.artist_credit + elif 'artist' in track_info: + item.artist = track_info.artist + elif 'artist_credit' in album_info: + item.artist = album_info.artist_credit + elif 'artist' in album_info: + item.artist = album_info.artist + + if 'artist_credit' in album_info: + item.albumartist = album_info.artist_credit + elif 'artist' in album_info: + item.albumartist = album_info.artist + else: - item.artist = (track_info.artist or album_info.artist) - item.albumartist = album_info.artist + if 'artist' in track_info: + item.artist = track_info.artist + elif 'artist' in album_info: + item.artist = album_info.artist + + if 'artist' in album_info: + item.albumartist = album_info.artist # Album. - item.album = album_info.album + if 'album' in album_info: + item.album = album_info.album # Artist sort and credit names. - item.artist_sort = track_info.artist_sort or album_info.artist_sort - item.artist_credit = (track_info.artist_credit or - album_info.artist_credit) - item.albumartist_sort = album_info.artist_sort - item.albumartist_credit = album_info.artist_credit + if 'artist_sort' in track_info: + item.artist_sort = track_info.artist_sort + elif 'artist_sort' in album_info: + item.artist_sort = album_info.artist_sort + + if 'artist_credit' in track_info: + item.artist_credit = track_info.artist_credit + elif 'artist_credit' in album_info: + item.artist_credit = album_info.artist_credit + + if 'albumartist_sort' in album_info: + item.albumartist_sort = album_info.artist_sort + + if 'albumartist_credit' in album_info: + item.albumartist_credit = album_info.artist_credit # Release date. for prefix in '', 'original_': @@ -106,7 +110,10 @@ def apply_metadata(album_info, mapping): for suffix in 'year', 'month', 'day': key = prefix + suffix - value = getattr(album_info, key) or 0 + if key in album_info: + value = getattr(album_info, key) + else: + value = 0 # If we don't even have a year, apply nothing. if suffix == 'year' and not value: @@ -122,40 +129,55 @@ def apply_metadata(album_info, mapping): item[suffix] = value # Title. - item.title = track_info.title + if 'title' in track_info: + item.title = track_info.title if config['per_disc_numbering']: # We want to let the track number be zero, but if the medium index # is not provided we need to fall back to the overall index. - if track_info.medium_index is not None: + if 'medium_index' in track_info: item.track = track_info.medium_index - else: + elif 'index' in track_info: item.track = track_info.index - item.tracktotal = track_info.medium_total or len(album_info.tracks) + if 'medium_total' in track_info: + item.tracktotal = track_info.medium_total + elif 'tracks' in album_info: + item.tracktotal = len(album_info.tracks) else: - item.track = track_info.index - item.tracktotal = len(album_info.tracks) + if 'index' in track_info: + item.track = track_info.index + if 'tracks' in album_info: + item.tracktotal = len(album_info.tracks) # Disc and disc count. - item.disc = track_info.medium - item.disctotal = album_info.mediums + if 'medium' in track_info: + item.disc = track_info.medium + if 'mediums' in album_info: + item.disctotal = album_info.mediums # MusicBrainz IDs. - item.mb_trackid = track_info.track_id - item.mb_releasetrackid = track_info.release_track_id - item.mb_albumid = album_info.album_id - if track_info.artist_id: + if 'track_id' in track_info: + item.mb_trackid = track_info.track_id + if 'release_track_id' in track_info: + item.mb_releasetrackid = track_info.release_track_id + if 'album_id' in album_info: + item.mb_albumid = album_info.album_id + if 'artist_id' in track_info: item.mb_artistid = track_info.artist_id - else: + elif 'artist_id' in album_info: item.mb_artistid = album_info.artist_id - item.mb_albumartistid = album_info.artist_id - item.mb_releasegroupid = album_info.releasegroup_id + if 'artist_id' in album_info: + item.mb_albumartistid = album_info.artist_id + if 'releasegroup_id' in album_info: + item.mb_releasegroupid = album_info.releasegroup_id # Compilation flag. - item.comp = album_info.va + if 'va' in album_info: + item.comp = album_info.va # Track alt. - item.track_alt = track_info.track_alt + if 'track_alt' in track_info: + item.track_alt = track_info.track_alt # Miscellaneous/nullable metadata. misc_fields = { @@ -197,14 +219,16 @@ def apply_metadata(album_info, mapping): # field is explicitly allowed to be overwritten for field in misc_fields['album']: clobber = field in config['overwrite_null']['album'].as_str_seq() - value = getattr(album_info, field) - if value is None and not clobber: - continue - item[field] = value + if field in album_info: + value = getattr(album_info, field) + if value is None and not clobber: + continue + item[field] = value for field in misc_fields['track']: clobber = field in config['overwrite_null']['track'].as_str_seq() - value = getattr(track_info, field) - if value is None and not clobber: - continue - item[field] = value + if field in track_info: + value = getattr(track_info, field) + if value is None and not clobber: + continue + item[field] = value From 62566ee61ddde9c6de8d6e527820fa32acaf579f Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 25 Apr 2020 23:13:38 +0200 Subject: [PATCH 113/293] remove prints for testing --- beets/autotag/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 2dcf93c52..530dcaf6e 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -41,9 +41,7 @@ log = logging.getLogger('beets') def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ - print('zer' in track_info) for attr in track_info: - print(attr in track_info) item.__setattr__(attr, getattr(track_info, attr)) # At the moment, the other metadata is left intact (including album From f507f0463906adfaae0aec6f0c0acbe3b1cb2814 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:21:13 +0200 Subject: [PATCH 114/293] reintroduce default arguments, adapt all occurences of TrackInfo and AlbumInfo to the absence of positional arguments --- beets/autotag/hooks.py | 84 +++++++++++++++++++++++++++++++++++++++++- beetsplug/cue.py | 3 +- beetsplug/discogs.py | 7 ++-- test/test_autotag.py | 47 +++++++++++------------ test/test_ui.py | 6 ++- 5 files changed, 116 insertions(+), 31 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index dd9726b84..fcb4ba161 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -100,7 +100,51 @@ class AlbumInfo(Map): ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, **kwargs): + def __init__(self, album=None, album_id=None, artist=None, artist_id=None, + tracks, asin=None, albumtype=None, va=False, year=None, + month=None, day=None, label=None, mediums=None, + artist_sort=None, releasegroup_id=None, catalognum=None, + script=None, language=None, country=None, style=None, + genre=None, albumstatus=None, media=None, albumdisambig=None, + releasegroupdisambig=None, artist_credit=None, + original_year=None, original_month=None, + original_day=None, data_source=None, data_url=None, + discogs_albumid=None, discogs_labelid=None, + discogs_artistid=None, **kwargs): + self.album = album + self.album_id = album_id + self.artist = artist + self.artist_id = artist_id + self.tracks = tracks + self.asin = asin + self.albumtype = albumtype + self.va = va + self.year = year + self.month = month + self.day = day + self.label = label + self.mediums = mediums + self.artist_sort = artist_sort + self.releasegroup_id = releasegroup_id + self.catalognum = catalognum + self.script = script + self.language = language + self.country = country + self.style = style + self.genre = genre + self.albumstatus = albumstatus + self.media = media + self.albumdisambig = albumdisambig + self.releasegroupdisambig = releasegroupdisambig + self.artist_credit = artist_credit + self.original_year = original_year + self.original_month = original_month + self.original_day = original_day + self.data_source = data_source + self.data_url = data_url + self.discogs_albumid = discogs_albumid + self.discogs_labelid = discogs_labelid + self.discogs_artistid = discogs_artistid for arg in kwargs: self.__setattr__(arg, kwargs[arg]) @@ -129,7 +173,43 @@ class TrackInfo(Map): may be None. The indices ``index``, ``medium``, and ``medium_index`` are all 1-based. """ - def __init__(self, **kwargs): + def __init__(self, title=None, track_id=None, release_track_id=None, + artist=None, artist_id=None, length=None, index=None, + medium=None, medium_index=None, medium_total=None, + artist_sort=None, disctitle=None, artist_credit=None, + data_source=None, data_url=None, media=None, lyricist=None, + composer=None, composer_sort=None, arranger=None, + performer=None, track_alt=None, work=None, mb_workid=None, + work_disambig=None, bpm=None, initial_key=None, genre=None, + **kwargs): + self.title = title + self.track_id = track_id + self.release_track_id = release_track_id + self.artist = artist + self.artist_id = artist_id + self.length = length + self.index = index + self.media = media + self.medium = medium + self.medium_index = medium_index + self.medium_total = medium_total + self.artist_sort = artist_sort + self.disctitle = disctitle + self.artist_credit = artist_credit + self.data_source = data_source + self.data_url = data_url + self.lyricist = lyricist + self.composer = composer + self.composer_sort = composer_sort + self.arranger = arranger + self.performer = performer + self.track_alt = track_alt + self.work = work + self.mb_workid = mb_workid + self.work_disambig = work_disambig + self.bpm = bpm + self.initial_key = initial_key + self.genre = genre for arg in kwargs: self.__setattr__(arg, kwargs[arg]) diff --git a/beetsplug/cue.py b/beetsplug/cue.py index 92ca8784a..1ff817b2b 100644 --- a/beetsplug/cue.py +++ b/beetsplug/cue.py @@ -53,5 +53,6 @@ class CuePlugin(BeetsPlugin): title = "dunno lol" track_id = "wtf" index = int(path.basename(t)[len("split-track"):-len(".wav")]) - yield TrackInfo(title, track_id, index=index, artist=artist) + yield TrackInfo(title=title, track_id=track_id, index=index, + artist=artist) # generate TrackInfo instances diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 86ace9aa8..a0a6ea654 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -356,7 +356,8 @@ class DiscogsPlugin(BeetsPlugin): # a master release, otherwise fetch the master release. original_year = self.get_master_year(master_id) if master_id else year - return AlbumInfo(album, album_id, artist, artist_id, tracks, asin=None, + return AlbumInfo(album=album, album_id=album_id, artist=artist, + artist_id=artist_id, tracks=tracks, asin=None, albumtype=albumtype, va=va, year=year, month=None, day=None, label=label, mediums=len(set(mediums)), artist_sort=None, releasegroup_id=master_id, @@ -567,8 +568,8 @@ class DiscogsPlugin(BeetsPlugin): track.get('artists', []) ) length = self.get_track_length(track['duration']) - return TrackInfo(title, track_id, artist=artist, artist_id=artist_id, - length=length, index=index, + return TrackInfo(title=title, track_id=track_id, artist=artist, + artist_id=artist_id, length=length, index=index, medium=medium, medium_index=medium_index, artist_sort=None, disctitle=None, artist_credit=None) diff --git a/test/test_autotag.py b/test/test_autotag.py index a11bc8fac..3b9ff3e67 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -106,9 +106,9 @@ def _make_item(title, track, artist=u'some artist'): def _make_trackinfo(): return [ - TrackInfo(u'one', None, artist=u'some artist', length=1, index=1), - TrackInfo(u'two', None, artist=u'some artist', length=1, index=2), - TrackInfo(u'three', None, artist=u'some artist', length=1, index=3), + TrackInfo(title=u'one', track_id=one, artist=u'some artist', length=1, index=1), + TrackInfo(title=u'two', track_id=None, artist=u'some artist', length=1, index=2), + TrackInfo(title=u'three', track_id=None, artist=u'some artist', length=1, index=3), ] @@ -503,9 +503,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 2)) items.append(self.item(u'two', 3)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'two', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'two', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -522,9 +522,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 1)) items.append(self.item(u'two', 1)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'two', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'two', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -540,9 +540,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'one', 1)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'two', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'two', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -558,8 +558,8 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'two', 2)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(u'one', None)) - trackinfo.append(TrackInfo(u'three', None)) + trackinfo.append(TrackInfo(title=u'one', track_id=None)) + trackinfo.append(TrackInfo(title=u'three', track_id=None)) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, [items[1]]) @@ -595,7 +595,8 @@ class AssignmentTest(unittest.TestCase): items.append(item(12, 186.45916150485752)) def info(index, title, length): - return TrackInfo(title, None, length=length, index=index) + return TrackInfo(title=title, track_id=None, length=length, + index=index) trackinfo = [] trackinfo.append(info(1, u'Alone', 238.893)) trackinfo.append(info(2, u'The Woman in You', 341.44)) @@ -638,8 +639,8 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item({})) trackinfo = [] trackinfo.append(TrackInfo( - u'oneNew', - u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', medium=1, medium_index=1, medium_total=1, @@ -648,8 +649,8 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): artist_sort='trackArtistSort', )) trackinfo.append(TrackInfo( - u'twoNew', - u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', medium=2, medium_index=1, index=2, @@ -828,15 +829,15 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): self.items.append(Item({})) trackinfo = [] trackinfo.append(TrackInfo( - u'oneNew', - u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', artist=u'artistOneNew', artist_id=u'a05686fc-9db2-4c23-b99e-77f5db3e5282', index=1, )) trackinfo.append(TrackInfo( - u'twoNew', - u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', artist=u'artistTwoNew', artist_id=u'80b3cf5e-18fe-4c59-98c7-e5bb87210710', index=2, diff --git a/test/test_ui.py b/test/test_ui.py index 110e80782..c4d502c73 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1051,8 +1051,10 @@ class ShowChangeTest(_common.TestCase): self.items[0].track = 1 self.items[0].path = b'/path/to/file.mp3' self.info = autotag.AlbumInfo( - u'the album', u'album id', u'the artist', u'artist id', [ - autotag.TrackInfo(u'the title', u'track id', index=1) + album=u'the album', album_id=u'album id', artist=u'the artist', + artist_id=u'artist id', tracks=[ + autotag.TrackInfo(title=u'the title', track_id=u'track id', + index=1) ] ) From 32d81d801bd7d9eea9a6887cdf443e0675ce6215 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:25:42 +0200 Subject: [PATCH 115/293] forgot a positional argument --- beets/autotag/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index fcb4ba161..0017f9dd6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -101,7 +101,7 @@ class AlbumInfo(Map): The others are optional and may be None. """ def __init__(self, album=None, album_id=None, artist=None, artist_id=None, - tracks, asin=None, albumtype=None, va=False, year=None, + tracks=None, asin=None, albumtype=None, va=False, year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, language=None, country=None, style=None, From 805f4e801f36cf73be8735daf601647b45b55770 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:31:39 +0200 Subject: [PATCH 116/293] typo in tests --- test/test_autotag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 3b9ff3e67..b894a65f3 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -106,7 +106,7 @@ def _make_item(title, track, artist=u'some artist'): def _make_trackinfo(): return [ - TrackInfo(title=u'one', track_id=one, artist=u'some artist', length=1, index=1), + TrackInfo(title=u'one', track_id=None, artist=u'some artist', length=1, index=1), TrackInfo(title=u'two', track_id=None, artist=u'some artist', length=1, index=2), TrackInfo(title=u'three', track_id=None, artist=u'some artist', length=1, index=3), ] From bd543136d66037e13400793d518ee74593d7f5f5 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:37:19 +0200 Subject: [PATCH 117/293] scale back some changes in __init__.py and hooks.py --- beets/autotag/__init__.py | 155 +++++++++++++++++--------------------- beets/autotag/hooks.py | 10 ++- 2 files changed, 76 insertions(+), 89 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 530dcaf6e..05b341a2d 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -41,8 +41,33 @@ log = logging.getLogger('beets') def apply_item_metadata(item, track_info): """Set an item's metadata from its matched TrackInfo object. """ - for attr in track_info: - item.__setattr__(attr, getattr(track_info, attr)) + item.artist = track_info.artist + item.artist_sort = track_info.artist_sort + item.artist_credit = track_info.artist_credit + item.title = track_info.title + item.mb_trackid = track_info.track_id + item.mb_releasetrackid = track_info.release_track_id + if track_info.artist_id: + item.mb_artistid = track_info.artist_id + if track_info.data_source: + item.data_source = track_info.data_source + + if track_info.lyricist is not None: + item.lyricist = track_info.lyricist + if track_info.composer is not None: + item.composer = track_info.composer + if track_info.composer_sort is not None: + item.composer_sort = track_info.composer_sort + if track_info.arranger is not None: + item.arranger = track_info.arranger + if track_info.performer is not None: + item.performer = track_info.performer + if track_info.work is not None: + item.work = track_info.work + if track_info.mb_workid is not None: + item.mb_workid = track_info.mb_workid + if track_info.work_disambig is not None: + item.work_disambig = track_info.work_disambig # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? @@ -55,50 +80,25 @@ def apply_metadata(album_info, mapping): for item, track_info in mapping.items(): # Artist or artist credit. if config['artist_credit']: - - if 'artist_credit' in track_info: - item.artist = track_info.artist_credit - elif 'artist' in track_info: - item.artist = track_info.artist - elif 'artist_credit' in album_info: - item.artist = album_info.artist_credit - elif 'artist' in album_info: - item.artist = album_info.artist - - if 'artist_credit' in album_info: - item.albumartist = album_info.artist_credit - elif 'artist' in album_info: - item.albumartist = album_info.artist - + item.artist = (track_info.artist_credit or + track_info.artist or + album_info.artist_credit or + album_info.artist) + item.albumartist = (album_info.artist_credit or + album_info.artist) else: - if 'artist' in track_info: - item.artist = track_info.artist - elif 'artist' in album_info: - item.artist = album_info.artist - - if 'artist' in album_info: - item.albumartist = album_info.artist + item.artist = (track_info.artist or album_info.artist) + item.albumartist = album_info.artist # Album. - if 'album' in album_info: - item.album = album_info.album + item.album = album_info.album # Artist sort and credit names. - if 'artist_sort' in track_info: - item.artist_sort = track_info.artist_sort - elif 'artist_sort' in album_info: - item.artist_sort = album_info.artist_sort - - if 'artist_credit' in track_info: - item.artist_credit = track_info.artist_credit - elif 'artist_credit' in album_info: - item.artist_credit = album_info.artist_credit - - if 'albumartist_sort' in album_info: - item.albumartist_sort = album_info.artist_sort - - if 'albumartist_credit' in album_info: - item.albumartist_credit = album_info.artist_credit + item.artist_sort = track_info.artist_sort or album_info.artist_sort + item.artist_credit = (track_info.artist_credit or + album_info.artist_credit) + item.albumartist_sort = album_info.artist_sort + item.albumartist_credit = album_info.artist_credit # Release date. for prefix in '', 'original_': @@ -108,10 +108,7 @@ def apply_metadata(album_info, mapping): for suffix in 'year', 'month', 'day': key = prefix + suffix - if key in album_info: - value = getattr(album_info, key) - else: - value = 0 + value = getattr(album_info, key) or 0 # If we don't even have a year, apply nothing. if suffix == 'year' and not value: @@ -127,55 +124,40 @@ def apply_metadata(album_info, mapping): item[suffix] = value # Title. - if 'title' in track_info: - item.title = track_info.title + item.title = track_info.title if config['per_disc_numbering']: # We want to let the track number be zero, but if the medium index # is not provided we need to fall back to the overall index. - if 'medium_index' in track_info: + if track_info.medium_index is not None: item.track = track_info.medium_index - elif 'index' in track_info: + else: item.track = track_info.index - if 'medium_total' in track_info: - item.tracktotal = track_info.medium_total - elif 'tracks' in album_info: - item.tracktotal = len(album_info.tracks) + item.tracktotal = track_info.medium_total or len(album_info.tracks) else: - if 'index' in track_info: - item.track = track_info.index - if 'tracks' in album_info: - item.tracktotal = len(album_info.tracks) + item.track = track_info.index + item.tracktotal = len(album_info.tracks) # Disc and disc count. - if 'medium' in track_info: - item.disc = track_info.medium - if 'mediums' in album_info: - item.disctotal = album_info.mediums + item.disc = track_info.medium + item.disctotal = album_info.mediums # MusicBrainz IDs. - if 'track_id' in track_info: - item.mb_trackid = track_info.track_id - if 'release_track_id' in track_info: - item.mb_releasetrackid = track_info.release_track_id - if 'album_id' in album_info: - item.mb_albumid = album_info.album_id - if 'artist_id' in track_info: + item.mb_trackid = track_info.track_id + item.mb_releasetrackid = track_info.release_track_id + item.mb_albumid = album_info.album_id + if track_info.artist_id: item.mb_artistid = track_info.artist_id - elif 'artist_id' in album_info: + else: item.mb_artistid = album_info.artist_id - if 'artist_id' in album_info: - item.mb_albumartistid = album_info.artist_id - if 'releasegroup_id' in album_info: - item.mb_releasegroupid = album_info.releasegroup_id + item.mb_albumartistid = album_info.artist_id + item.mb_releasegroupid = album_info.releasegroup_id # Compilation flag. - if 'va' in album_info: - item.comp = album_info.va + item.comp = album_info.va # Track alt. - if 'track_alt' in track_info: - item.track_alt = track_info.track_alt + item.track_alt = track_info.track_alt # Miscellaneous/nullable metadata. misc_fields = { @@ -204,6 +186,7 @@ def apply_metadata(album_info, mapping): 'composer', 'composer_sort', 'arranger', + 'performer', 'work', 'mb_workid', 'work_disambig', @@ -217,16 +200,14 @@ def apply_metadata(album_info, mapping): # field is explicitly allowed to be overwritten for field in misc_fields['album']: clobber = field in config['overwrite_null']['album'].as_str_seq() - if field in album_info: - value = getattr(album_info, field) - if value is None and not clobber: - continue - item[field] = value + value = getattr(album_info, field) + if value is None and not clobber: + continue + item[field] = value for field in misc_fields['track']: clobber = field in config['overwrite_null']['track'].as_str_seq() - if field in track_info: - value = getattr(track_info, field) - if value is None and not clobber: - continue - item[field] = value + value = getattr(track_info, field) + if value is None and not clobber: + continue + item[field] = value diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 0017f9dd6..1e856ca0a 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -155,7 +155,12 @@ class AlbumInfo(Map): """Ensure that all string attributes on this object, and the constituent `TrackInfo` objects, are decoded to Unicode. """ - for fld in self: + for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort', + 'catalognum', 'script', 'language', 'country', 'style', + 'genre', 'albumstatus', 'albumdisambig', + 'releasegroupdisambig', 'artist_credit', + 'media', 'discogs_albumid', 'discogs_labelid', + 'discogs_artistid']: value = getattr(self, fld) if type(value) == str: if isinstance(value, bytes): @@ -218,7 +223,8 @@ class TrackInfo(Map): """Ensure that all string attributes on this object are decoded to Unicode. """ - for fld in self: + for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', + 'artist_credit', 'media']: value = getattr(self, fld) if type(value) == str: if isinstance(value, bytes): From 363cbf8147940f471558e3545cc58ee2f491fb3e Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:39:20 +0200 Subject: [PATCH 118/293] mixed two PR --- beets/autotag/__init__.py | 3 --- beets/autotag/hooks.py | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 05b341a2d..f9e38413e 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -60,8 +60,6 @@ def apply_item_metadata(item, track_info): item.composer_sort = track_info.composer_sort if track_info.arranger is not None: item.arranger = track_info.arranger - if track_info.performer is not None: - item.performer = track_info.performer if track_info.work is not None: item.work = track_info.work if track_info.mb_workid is not None: @@ -186,7 +184,6 @@ def apply_metadata(album_info, mapping): 'composer', 'composer_sort', 'arranger', - 'performer', 'work', 'mb_workid', 'work_disambig', diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 1e856ca0a..48d1cae52 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -184,7 +184,7 @@ class TrackInfo(Map): artist_sort=None, disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, - performer=None, track_alt=None, work=None, mb_workid=None, + track_alt=None, work=None, mb_workid=None, work_disambig=None, bpm=None, initial_key=None, genre=None, **kwargs): self.title = title @@ -207,7 +207,6 @@ class TrackInfo(Map): self.composer = composer self.composer_sort = composer_sort self.arranger = arranger - self.performer = performer self.track_alt = track_alt self.work = work self.mb_workid = mb_workid From 14e1b33839afbf1c8fb1c4f94caf77bb99d82b96 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 11:44:51 +0200 Subject: [PATCH 119/293] lines too long --- test/test_autotag.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index b894a65f3..fcf2ee5b1 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -106,9 +106,12 @@ def _make_item(title, track, artist=u'some artist'): def _make_trackinfo(): return [ - TrackInfo(title=u'one', track_id=None, artist=u'some artist', length=1, index=1), - TrackInfo(title=u'two', track_id=None, artist=u'some artist', length=1, index=2), - TrackInfo(title=u'three', track_id=None, artist=u'some artist', length=1, index=3), + TrackInfo(title=u'one', track_id=None, artist=u'some artist', + length=1, index=1), + TrackInfo(title=u'two', track_id=None, artist=u'some artist', + length=1, index=2), + TrackInfo(title=u'three', track_id=None, artist=u'some artist', + length=1, index=3), ] From 8df2e5b8f5affc9a5472886e819a31789c8fa280 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 12:01:35 +0200 Subject: [PATCH 120/293] arrange decoder --- beets/autotag/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 48d1cae52..95533d11d 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -162,7 +162,7 @@ class AlbumInfo(Map): 'media', 'discogs_albumid', 'discogs_labelid', 'discogs_artistid']: value = getattr(self, fld) - if type(value) == str: + if isinstance(value, bytes): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) @@ -225,7 +225,7 @@ class TrackInfo(Map): for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle', 'artist_credit', 'media']: value = getattr(self, fld) - if type(value) == str: + if isinstance(value, bytes): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) From 98389b6161bcdad8cbbd8638eeedd01d877b25a3 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 12:02:27 +0200 Subject: [PATCH 121/293] dupe test --- beets/autotag/hooks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 95533d11d..83abd8736 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -163,8 +163,7 @@ class AlbumInfo(Map): 'discogs_artistid']: value = getattr(self, fld) if isinstance(value, bytes): - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) + setattr(self, fld, value.decode(codec, 'ignore')) class TrackInfo(Map): @@ -226,8 +225,7 @@ class TrackInfo(Map): 'artist_credit', 'media']: value = getattr(self, fld) if isinstance(value, bytes): - if isinstance(value, bytes): - setattr(self, fld, value.decode(codec, 'ignore')) + setattr(self, fld, value.decode(codec, 'ignore')) # Candidate distance scoring. From 6a03afc65df168cf60b83bc34c1b9b94c648c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Tempel?= Date: Sat, 25 Apr 2020 18:26:58 +0200 Subject: [PATCH 122/293] web plugin: support path queries by separating them with a backslash Without this change the web plugin does not support path queries as slashes are currently used for joining keywords in the QueryConverter. Moreover, flask cannot distinguish between an URL encoded and a plain '/' character during routing [0]. To work around this issue without introducing a breaking change (i.e. removing the QueryConverter) use the backslash character for path queries and convert it later on. Fixes #3566 [0]: https://github.com/pallets/flask/issues/900 --- beetsplug/web/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 21ff5d94e..49149772d 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -177,10 +177,11 @@ class QueryConverter(PathConverter): """ def to_python(self, value): - return value.split('/') + queries = value.split('/') + return [query.replace('\\', os.sep) for query in queries] def to_url(self, value): - return ','.join(value) + return ','.join([v.replace(os.sep, '\\') for v in value]) class EverythingConverter(PathConverter): From 57a759c8175eedabfddbd3a02e33b3af73fd13e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Tempel?= Date: Sun, 26 Apr 2020 17:39:16 +0200 Subject: [PATCH 123/293] docs: document joining of paths performed by web plugin for queries --- docs/plugins/web.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index d416b1b7d..99214ef01 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -210,7 +210,10 @@ If the server runs UNIX, you'll need to include an extra leading slash: ``GET /item/query/querystring`` +++++++++++++++++++++++++++++++ -Returns a list of tracks matching the query. The *querystring* must be a valid query as described in :doc:`/reference/query`. :: +Returns a list of tracks matching the query. The *querystring* must be a +valid query as described in :doc:`/reference/query`. Path elements are +joined as query keywords. For example, ``/item/query/foo/bar`` will be +converted to the query ``foo,bar``. :: { "results": [ From 226d8089e1ff16687bff1731f048e8e4ba130db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Tempel?= Date: Sun, 26 Apr 2020 17:45:37 +0200 Subject: [PATCH 124/293] docs: document special handling of backlash in web plugin queries --- docs/plugins/web.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 99214ef01..481727475 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -213,7 +213,11 @@ If the server runs UNIX, you'll need to include an extra leading slash: Returns a list of tracks matching the query. The *querystring* must be a valid query as described in :doc:`/reference/query`. Path elements are joined as query keywords. For example, ``/item/query/foo/bar`` will be -converted to the query ``foo,bar``. :: +converted to the query ``foo,bar``. As this conflicts with using a slash +character as a path seperator in path queries, a backlash character +should be used in these queries instead. This character is converted to +the path seperator actually used by the operating system before the +query is performed. :: { "results": [ From 969a2074186d277054c364d249320f3518edfaf0 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 27 Apr 2020 07:45:31 -0400 Subject: [PATCH 125/293] Simplify docs for #3567 --- docs/plugins/web.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 481727475..65d4743fb 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -211,13 +211,7 @@ If the server runs UNIX, you'll need to include an extra leading slash: +++++++++++++++++++++++++++++++ Returns a list of tracks matching the query. The *querystring* must be a -valid query as described in :doc:`/reference/query`. Path elements are -joined as query keywords. For example, ``/item/query/foo/bar`` will be -converted to the query ``foo,bar``. As this conflicts with using a slash -character as a path seperator in path queries, a backlash character -should be used in these queries instead. This character is converted to -the path seperator actually used by the operating system before the -query is performed. :: +valid query as described in :doc:`/reference/query`. :: { "results": [ @@ -226,6 +220,11 @@ query is performed. :: ] } +Path elements are joined as parts of a query. For example, +``/item/query/foo/bar`` will be converted to the query ``foo,bar``. +To specify literal path separators in a query, use a backslash instead of a +slash. + ``GET /item/6/file`` ++++++++++++++++++++ From f43a0c3270d18af19affea04fae9fb39ab02d784 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 27 Apr 2020 07:46:28 -0400 Subject: [PATCH 126/293] Changelog/thanks for #3567 --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2f1521eb..5d8c26bb9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -115,6 +115,10 @@ New features: :bug:`3459` * :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. :bug:`3530` +* :doc:`/plugins/web`: The query API now interprets backslashes as path + separators to support path queries. + Thanks to :user:`nmeum`. + :bug:`3567` Fixes: From 63df7cf3edf9fc45fad4186d4332f30afe66a952 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 15:03:55 +0200 Subject: [PATCH 127/293] forgot to decode all tracks of an album --- beets/autotag/hooks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 83abd8736..a5d1ae428 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -165,6 +165,10 @@ class AlbumInfo(Map): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) + if 'tracks' in self: + for track in self.tracks: + track.decode(codec) + class TrackInfo(Map): """Describes a canonical track present on a release. Appears as part From 048b5c21517b88089723ecbca6282e337c976244 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 15:55:12 +0200 Subject: [PATCH 128/293] remove need for deepcopy, simplify __init__ --- beets/autotag/hooks.py | 18 +--- test/test_autotag.py | 199 +++++++++++++++++++++++++++++++++++++++-- test/test_ui.py | 7 +- 3 files changed, 202 insertions(+), 22 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index a5d1ae428..0c6e1eb80 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -45,13 +45,8 @@ class Map(dict): m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) """ - def __init__(self, *args, **kwargs): - super(Map, self).__init__(*args, **kwargs) - for arg in args: - if isinstance(arg, dict): - for k, v in arg.iteritems(): - self[k] = v - + def __init__(self, **kwargs): + super(Map, self).__init__(**kwargs) if kwargs: for k, v in kwargs.iteritems(): self[k] = v @@ -76,15 +71,8 @@ class Map(dict): super(Map, self).__delitem__(key) del self.__dict__[key] - def __getstate__(self): - return self.__dict__ - - def __setstate__(self, state): - for key in state: - self.__setattr__(key, state[key]) - def __hash__(self): - return hash(tuple(sorted(self.items()))) + return self.id class AlbumInfo(Map): diff --git a/test/test_autotag.py b/test/test_autotag.py index fcf2ee5b1..97030785b 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -753,13 +753,75 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].albumtype, 'album') def test_album_artist_overrides_empty_track_artist(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') def test_album_artist_overridden_by_nonempty_track_artist(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -781,7 +843,38 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].artist_sort, 'albumArtistSort') def test_full_date_applied(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -796,7 +889,38 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item(year=1, month=2, day=3)) self.items.append(Item(year=4, month=5, day=6)) - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.year = 2013 self._apply(info=my_info) @@ -816,7 +940,38 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[0].day, 3) def test_data_source_applied(self): - my_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + my_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -827,6 +982,7 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def setUp(self): super(ApplyCompilationTest, self).setUp() + # make a deepcopy of self.info self.items = [] self.items.append(Item({})) self.items.append(Item({})) @@ -878,7 +1034,38 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): self.assertFalse(self.items[1].comp) def test_va_flag_sets_comp(self): - va_info = copy.deepcopy(self.info) + # make a deepcopy of self.info + trackinfo = [] + trackinfo.append(TrackInfo( + title=u'oneNew', + track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', + medium=1, + medium_index=1, + medium_total=1, + index=1, + artist_credit='trackArtistCredit', + artist_sort='trackArtistSort', + )) + trackinfo.append(TrackInfo( + title=u'twoNew', + track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', + medium=2, + medium_index=1, + index=2, + medium_total=1, + )) + va_info = AlbumInfo( + tracks=trackinfo, + artist=u'artistNew', + album=u'albumNew', + album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', + artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', + artist_credit=u'albumArtistCredit', + artist_sort=u'albumArtistSort', + albumtype=u'album', + va=False, + mediums=2, + ) va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) diff --git a/test/test_ui.py b/test/test_ui.py index c4d502c73..608c1d7fb 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1138,7 +1138,12 @@ class SummarizeItemsTest(_common.TestCase): summary = commands.summarize_items([self.item], False) self.assertEqual(summary, u"1 items, F, 4kbps, 10:54, 987.0 B") - i2 = deepcopy(self.item) + # make a copy of self.item + i2 = library.Item() + i2.bitrate = 4321 + i2.length = 10 * 60 + 54 + i2.format = "F" + summary = commands.summarize_items([self.item, i2], False) self.assertEqual(summary, u"2 items, F, 4kbps, 21:48, 1.9 KiB") From a51ef113d3fd1642f5c7298895697dae76fc9b1c Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 15:59:21 +0200 Subject: [PATCH 129/293] arranged hash --- beets/autotag/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 0c6e1eb80..350ca065c 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -72,7 +72,7 @@ class Map(dict): del self.__dict__[key] def __hash__(self): - return self.id + return id(self) class AlbumInfo(Map): From 2bc0027adf9824d7a312ae65ba0c045965eb4e60 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:08:47 +0200 Subject: [PATCH 130/293] remove some prints, unused libraries, __setitem__ method --- beets/autotag/hooks.py | 4 ---- test/test_autotag.py | 1 - test/test_ui.py | 1 - 3 files changed, 6 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 350ca065c..74145e78d 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -60,10 +60,6 @@ class Map(dict): def __setattr__(self, key, value): self.__setitem__(key, value) - def __setitem__(self, key, value): - super(Map, self).__setitem__(key, value) - self.__dict__.update({key: value}) - def __delattr__(self, item): self.__delitem__(item) diff --git a/test/test_autotag.py b/test/test_autotag.py index 97030785b..385b793ee 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -18,7 +18,6 @@ from __future__ import division, absolute_import, print_function import re -import copy import unittest from test import _common diff --git a/test/test_ui.py b/test/test_ui.py index 608c1d7fb..d0be10060 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -22,7 +22,6 @@ import shutil import re import subprocess import platform -from copy import deepcopy import six import unittest From eec7994dbe4cc3e12495de359989c1f6884f312c Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:16:37 +0200 Subject: [PATCH 131/293] corrected on deepcopy replacement --- test/test_autotag.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 385b793ee..af74b5c8f 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -1035,35 +1035,28 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info trackinfo = [] + trackinfo = [] trackinfo.append(TrackInfo( title=u'oneNew', track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, + artist=u'artistOneNew', + artist_id=u'a05686fc-9db2-4c23-b99e-77f5db3e5282', index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', )) trackinfo.append(TrackInfo( title=u'twoNew', track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, + artist=u'artistTwoNew', + artist_id=u'80b3cf5e-18fe-4c59-98c7-e5bb87210710', index=2, - medium_total=1, )) va_info = AlbumInfo( tracks=trackinfo, - artist=u'artistNew', + artist=u'variousNew', album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, + album_id='3b69ea40-39b8-487f-8818-04b6eff8c21a', + artist_id='89ad4ac3-39f7-470e-963a-56509c546377', + albumtype=u'compilation', ) va_info.va = True self._apply(info=va_info) From d07a3e36970aa785a8085afd56eaa4062be93787 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:17:11 +0200 Subject: [PATCH 132/293] remove __delitem__ and __delattr__ methods --- beets/autotag/hooks.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 74145e78d..36ea8db1e 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -60,13 +60,6 @@ class Map(dict): def __setattr__(self, key, value): self.__setitem__(key, value) - def __delattr__(self, item): - self.__delitem__(item) - - def __delitem__(self, key): - super(Map, self).__delitem__(key) - del self.__dict__[key] - def __hash__(self): return id(self) From 13db4063fa0d5515f8a479e90c1a2235c6e4cd06 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 27 Apr 2020 16:31:38 +0200 Subject: [PATCH 133/293] actually, I don't need __init__ --- beets/autotag/hooks.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 36ea8db1e..1500538d3 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -45,11 +45,6 @@ class Map(dict): m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer']) """ - def __init__(self, **kwargs): - super(Map, self).__init__(**kwargs) - if kwargs: - for k, v in kwargs.iteritems(): - self[k] = v def __getattr__(self, attr): if attr in self: From b39ef0b8f949ea41d0d9e40674b20d9201c370a3 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 12:16:19 +0200 Subject: [PATCH 134/293] better deepcopy, docstring, minor improvements --- beets/autotag/hooks.py | 26 +++--- test/test_autotag.py | 198 +++++------------------------------------ 2 files changed, 40 insertions(+), 184 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 1500538d3..81c2fadb1 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -39,11 +39,19 @@ except AttributeError: # Classes used to represent candidate options. -class Map(dict): +class AttrDict(dict): """ - Example: - m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, - sports=['Soccer']) + Dictionary with flexible attributes + to get an tag value: + value = info.tag + or value = info[tag] + or value = info.get(tag) + or value = getattr(info, tag) + all raise AttributeError when info doesn't have tag + to set a tag value: + info.tag = value + or info[tag] = value + or setattr(info, tag, value) """ def __getattr__(self, attr): @@ -59,7 +67,7 @@ class Map(dict): return id(self) -class AlbumInfo(Map): +class AlbumInfo(AttrDict): """Describes a canonical release that may be used to match a release in the library. Consists of these data members: @@ -117,8 +125,7 @@ class AlbumInfo(Map): self.discogs_albumid = discogs_albumid self.discogs_labelid = discogs_labelid self.discogs_artistid = discogs_artistid - for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.update(kwargs) # Work around a bug in python-musicbrainz-ngs that causes some # strings to be bytes rather than Unicode. @@ -142,7 +149,7 @@ class AlbumInfo(Map): track.decode(codec) -class TrackInfo(Map): +class TrackInfo(AttrDict): """Describes a canonical track present on a release. Appears as part of an AlbumInfo's ``tracks`` list. Consists of these data members: @@ -189,8 +196,7 @@ class TrackInfo(Map): self.bpm = bpm self.initial_key = initial_key self.genre = genre - for arg in kwargs: - self.__setattr__(arg, kwargs[arg]) + self.update(kwargs) # As above, work around a bug in python-musicbrainz-ngs. def decode(self, codec='utf-8'): diff --git a/test/test_autotag.py b/test/test_autotag.py index af74b5c8f..3532a80ec 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -754,36 +754,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overrides_empty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') @@ -791,36 +765,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overridden_by_nonempty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -844,36 +792,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_full_date_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -890,36 +812,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.year = 2013 self._apply(info=my_info) @@ -941,36 +837,10 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_data_source_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - medium=1, - medium_index=1, - medium_total=1, - index=1, - artist_credit='trackArtistCredit', - artist_sort='trackArtistSort', - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - medium=2, - medium_index=1, - index=2, - medium_total=1, - )) - my_info = AlbumInfo( - tracks=trackinfo, - artist=u'artistNew', - album=u'albumNew', - album_id='7edb51cb-77d6-4416-a23c-3a8c2994a2c7', - artist_id='a6623d39-2d8e-4f70-8242-0a9553b91e50', - artist_credit=u'albumArtistCredit', - artist_sort=u'albumArtistSort', - albumtype=u'album', - va=False, - mediums=2, - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + my_info = AlbumInfo(self.info) + my_info.tracks = trackinfo my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -981,7 +851,6 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def setUp(self): super(ApplyCompilationTest, self).setUp() - # make a deepcopy of self.info self.items = [] self.items.append(Item({})) self.items.append(Item({})) @@ -1035,29 +904,10 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info trackinfo = [] - trackinfo = [] - trackinfo.append(TrackInfo( - title=u'oneNew', - track_id=u'dfa939ec-118c-4d0f-84a0-60f3d1e6522c', - artist=u'artistOneNew', - artist_id=u'a05686fc-9db2-4c23-b99e-77f5db3e5282', - index=1, - )) - trackinfo.append(TrackInfo( - title=u'twoNew', - track_id=u'40130ed1-a27c-42fd-a328-1ebefb6caef4', - artist=u'artistTwoNew', - artist_id=u'80b3cf5e-18fe-4c59-98c7-e5bb87210710', - index=2, - )) - va_info = AlbumInfo( - tracks=trackinfo, - artist=u'variousNew', - album=u'albumNew', - album_id='3b69ea40-39b8-487f-8818-04b6eff8c21a', - artist_id='89ad4ac3-39f7-470e-963a-56509c546377', - albumtype=u'compilation', - ) + trackinfo.append(TrackInfo(self.info.tracks[0])) + trackinfo.append(TrackInfo(self.info.tracks[1])) + va_info = AlbumInfo(self.info) + va_info.tracks = trackinfo va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) From ba2b22cac581d4cb304457b57fe5e1555bc18e45 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 12:59:52 +0200 Subject: [PATCH 135/293] try to correct deepcopy --- test/test_autotag.py | 66 ++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/test/test_autotag.py b/test/test_autotag.py index 3532a80ec..06e6b7e72 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -754,9 +754,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overrides_empty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') @@ -765,9 +770,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overridden_by_nonempty_track_artist(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' @@ -792,9 +802,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_full_date_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.year = 2013 my_info.month = 12 @@ -812,9 +827,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.year = 2013 self._apply(info=my_info) @@ -837,9 +857,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_data_source_applied(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - my_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + my_info = AlbumInfo() + my_info.update(self.info) my_info.tracks = trackinfo my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -904,9 +929,14 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info trackinfo = [] - trackinfo.append(TrackInfo(self.info.tracks[0])) - trackinfo.append(TrackInfo(self.info.tracks[1])) - va_info = AlbumInfo(self.info) + track = TrackInfo() + track.update(self.info.tracks[0]) + trackinfo.append(track) + track = TrackInfo() + track.update(self.info.tracks[1]) + trackinfo.append(track) + va_info = AlbumInfo() + va_info.update(self.info) va_info.tracks = trackinfo va_info.va = True self._apply(info=va_info) From 84bbd14c767d3a720e57b5572f9c8f443c18896a Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 13:09:04 +0200 Subject: [PATCH 136/293] trailing whitespace --- beets/autotag/hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 81c2fadb1..2c78749e6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -42,13 +42,13 @@ except AttributeError: class AttrDict(dict): """ Dictionary with flexible attributes - to get an tag value: + to get an tag value: value = info.tag or value = info[tag] or value = info.get(tag) or value = getattr(info, tag) all raise AttributeError when info doesn't have tag - to set a tag value: + to set a tag value: info.tag = value or info[tag] = value or setattr(info, tag, value) From d7ed84646ef2b5a91cda234c0284433ff6c67bd5 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 14:18:27 +0200 Subject: [PATCH 137/293] create method for deepcopy --- beets/autotag/hooks.py | 14 +++++++++ test/test_autotag.py | 66 ++++-------------------------------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 2c78749e6..444ae7b59 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -148,6 +148,16 @@ class AlbumInfo(AttrDict): for track in self.tracks: track.decode(codec) + def dup_albuminfo(self): + dupe = AlbumInfo() + dupe.update(self) + if 'tracks' in self: + tracks = [] + for track in self.tracks: + tracks.append(track.dup_trackinfo()) + dupe.tracks = tracks + return dupe + class TrackInfo(AttrDict): """Describes a canonical track present on a release. Appears as part @@ -209,6 +219,10 @@ class TrackInfo(AttrDict): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) + def dup_trackinfo(self): + dupe = TrackInfo() + dupe.update(self) + # Candidate distance scoring. diff --git a/test/test_autotag.py b/test/test_autotag.py index 06e6b7e72..cef6364d0 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -753,32 +753,14 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_album_artist_overrides_empty_track_artist(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') def test_album_artist_overridden_by_nonempty_track_artist(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -801,16 +783,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_full_date_applied(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -826,16 +799,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item(year=4, month=5, day=6)) # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.year = 2013 self._apply(info=my_info) @@ -856,16 +820,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): def test_data_source_applied(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - my_info = AlbumInfo() - my_info.update(self.info) - my_info.tracks = trackinfo + my_info = self.info.dup_albuminfo() my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -928,16 +883,7 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): def test_va_flag_sets_comp(self): # make a deepcopy of self.info - trackinfo = [] - track = TrackInfo() - track.update(self.info.tracks[0]) - trackinfo.append(track) - track = TrackInfo() - track.update(self.info.tracks[1]) - trackinfo.append(track) - va_info = AlbumInfo() - va_info.update(self.info) - va_info.tracks = trackinfo + va_info = self.info.dup_albuminfo() va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) From 370df6253d4c4da48183a018332c8c2778877979 Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 28 Apr 2020 14:19:59 +0200 Subject: [PATCH 138/293] forgot a return --- beets/autotag/hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 444ae7b59..c2775bf32 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -222,6 +222,7 @@ class TrackInfo(AttrDict): def dup_trackinfo(self): dupe = TrackInfo() dupe.update(self) + return dupe # Candidate distance scoring. From 499fcb8315e40e1554cfef5ccb4e8437fbfeb5b8 Mon Sep 17 00:00:00 2001 From: Joris Date: Sun, 3 May 2020 12:56:54 +0200 Subject: [PATCH 139/293] Update docs/plugins/subsonicplaylist.rst Co-authored-by: Adrian Sampson --- docs/plugins/subsonicplaylist.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index 22fe16816..c712a4823 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -23,11 +23,11 @@ flag. This resets all `subsonic_playlist` tag before importing playlists. Options to be defined in your config with their default value:: subsonicplaylist: - 'base_url': "https://your.subsonic.server" - 'delete': False, - 'playlist_ids': [], - 'playlist_names': [], - 'username': '', - 'password': '' + base_url: "https://your.subsonic.server" + delete: no + playlist_ids: [] + playlist_names: [] + username: '' + password: '' -Parameters `base_url`, `username` and `password` must be defined! \ No newline at end of file +Parameters `base_url`, `username` and `password` must be defined! From 82f0c59f46d6a0bd164313bc83540d2c4dcc442b Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 13:10:24 +0200 Subject: [PATCH 140/293] review: updated subsonicplaylist.rst --- docs/plugins/subsonicplaylist.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index c712a4823..b9a54c3de 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -2,11 +2,11 @@ Subsonic Playlist Plugin ======================== The ``subsonicplaylist`` plugin allows to import playlists from a subsonic server. -This is done by retrieving the track infos from the subsonic server, searching -them in the beets library and adding the playlist names to the +This is done by retrieving the track info from the subsonic server, searching +for them in the beets library, and adding the playlist names to the `subsonic_playlist` tag of the found items. The content of the tag has the format: - subsonic_playlist: ";first playlist;second playlist" + subsonic_playlist: ";first playlist;second playlist;" To get all items in a playlist use the query `;playlist name;`. @@ -15,12 +15,22 @@ Command Line Usage To use the ``subsonicplaylist`` plugin, enable it in your configuration (see :ref:`using-plugins`). Then use it by invoking the ``subsonicplaylist`` command. +Next, configure the plugin to connect to your Subsonic server, like this:: + + subsonicplaylist: + base_url: http://subsonic.example.com + username: someUser + password: somePassword + +After this you can import your playlists by invoking the `subsonicplaylist` command. + By default only the tags of the items found for playlists will be updated. This means that, if one imported a playlist, then delete one song from it and imported the playlist again, the deleted song will still have the playlist set in its `subsonic_playlist` tag. To solve this problem one can use the `-d/--delete` flag. This resets all `subsonic_playlist` tag before importing playlists. -Options to be defined in your config with their default value:: + +Here's an example configuration with all the available options and their default values:: subsonicplaylist: base_url: "https://your.subsonic.server" From cb7dfe3f6f9fa37045cf6741e6909003784c62b0 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 14:05:22 +0200 Subject: [PATCH 141/293] review: updated subsonicplaylist.py --- beetsplug/subsonicplaylist.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index fbbcd8ac3..561b6032d 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -19,7 +19,13 @@ from hashlib import md5 import xml.etree.ElementTree as ET import random import string +from urllib.parse import urlencode + import requests +from beets.dbcore.query import SubstringQuery + +from beets.dbcore import AndQuery, Query, MatchQuery + from beets import plugins from beets.ui import Subcommand @@ -46,19 +52,21 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def update_tags(self, playlist_dict, lib): for query, playlist_tag in playlist_dict.items(): - query = 'artist:"{}" album:"{}" title:"{}"'.format(*query) + query = AndQuery([SubstringQuery("artist", query[0]), + SubstringQuery("album", query[1]), + SubstringQuery("title", query[2])]) items = lib.items(query) - if len(items) <= 0: + if not items: self._log.warn(u"{} | track not found ({})", playlist_tag, query) continue for item in items: - item.update({'subsonic_playlist': playlist_tag}) + item.subsonic_playlist = playlist_tag with lib.transaction(): item.try_sync(write=True, move=False) def get_playlist(self, playlist_id): - xml = self.send('getPlaylist', '&id={}'.format(playlist_id)).text + xml = self.send('getPlaylist', {'id': playlist_id}).text playlist = ET.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' @@ -72,7 +80,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def commands(self): def build_playlist(lib, opts, args): - self._parse_opts(opts) + self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() if len(self.config['playlist_names'].as_str_seq()) > 0: playlists = ET.fromstring(self.send('getPlaylists').text)[ @@ -102,8 +110,6 @@ class SubsonicPlaylistPlugin(BeetsPlugin): item.try_sync(write=True, move=False) self.update_tags(playlist_dict, lib) - ## notify plugins - plugins.send('database_change', lib=lib, model=self) subsonicplaylist_cmds = Subcommand( 'subsonicplaylist', help=u'import a subsonic playlist' @@ -117,24 +123,18 @@ class SubsonicPlaylistPlugin(BeetsPlugin): subsonicplaylist_cmds.func = build_playlist return [subsonicplaylist_cmds] - def _parse_opts(self, opts): - - if opts.delete: - self.config['delete'].set(True) - - self.opts = opts - return True - def generate_token(self): salt = ''.join(random.choices(string.ascii_lowercase + string.digits)) return md5( (self.config['password'].get() + salt).encode()).hexdigest(), salt - def send(self, endpoint, params=''): + def send(self, endpoint, params=None): + if params is None: + params = dict() url = '{}/rest/{}?u={}&t={}&s={}&v=1.12.0&c=beets'.format( self.config['base_url'].get(), endpoint, self.config['username'], *self.generate_token()) - resp = requests.get(url + params) + resp = requests.get(url + urlencode(params)) return resp From 13b4a1413d1c69e79dcb497fb32675c7d96fb302 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 14:58:34 +0200 Subject: [PATCH 142/293] update all songs in one transaction --- beetsplug/subsonicplaylist.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 561b6032d..9c9c74974 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -51,18 +51,18 @@ class SubsonicPlaylistPlugin(BeetsPlugin): self.config['password'].redact = True def update_tags(self, playlist_dict, lib): - for query, playlist_tag in playlist_dict.items(): - query = AndQuery([SubstringQuery("artist", query[0]), - SubstringQuery("album", query[1]), - SubstringQuery("title", query[2])]) - items = lib.items(query) - if not items: - self._log.warn(u"{} | track not found ({})", playlist_tag, - query) - continue - for item in items: - item.subsonic_playlist = playlist_tag - with lib.transaction(): + with lib.transaction(): + for query, playlist_tag in playlist_dict.items(): + query = AndQuery([SubstringQuery("artist", query[0]), + SubstringQuery("album", query[1]), + SubstringQuery("title", query[2])]) + items = lib.items(query) + if not items: + self._log.warn(u"{} | track not found ({})", playlist_tag, + query) + continue + for item in items: + item.subsonic_playlist = playlist_tag item.try_sync(write=True, move=False) def get_playlist(self, playlist_id): From cb7ad191a35d6deda8872f633a7a44c8b26673d4 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 15:02:51 +0200 Subject: [PATCH 143/293] removed duplicate line from merge --- docs/changelog.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cbd9f3ce2..88d289e35 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,6 @@ Changelog New features: * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. -* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets From 5c3debe2369cc161c2c9a875e3f2c20d8a2f5903 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sun, 3 May 2020 16:07:27 +0200 Subject: [PATCH 144/293] removed unused imports --- beetsplug/subsonicplaylist.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 9c9c74974..14cd855b1 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -24,9 +24,7 @@ from urllib.parse import urlencode import requests from beets.dbcore.query import SubstringQuery -from beets.dbcore import AndQuery, Query, MatchQuery - -from beets import plugins +from beets.dbcore import AndQuery from beets.ui import Subcommand From a553693677e8cc11c645efa94d11b1669fd0e35c Mon Sep 17 00:00:00 2001 From: pants108 Date: Sun, 3 May 2020 00:06:55 +0000 Subject: [PATCH 145/293] fetchart: clean up invalid tmp files --- beetsplug/fetchart.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 763440238..d4ab714de 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -210,6 +210,9 @@ class ArtSource(RequestMixin): def fetch_image(self, candidate, plugin): raise NotImplementedError() + def cleanup_tmp(self, candidate): + raise NotImplementedError() + class LocalArtSource(ArtSource): IS_LOCAL = True @@ -218,6 +221,10 @@ class LocalArtSource(ArtSource): def fetch_image(self, candidate, plugin): pass + def cleanup_tmp(self, candidate): + # local art source does not create tmp files + pass + class RemoteArtSource(ArtSource): IS_LOCAL = False @@ -291,6 +298,13 @@ class RemoteArtSource(ArtSource): self._log.debug(u'error fetching art: {}', exc) return + def cleanup_tmp(self, candidate): + if candidate.path: + try: + util.remove(path=candidate.path) + except util.FilesystemError as exc: + self._log.debug(u'error cleaning up tmp art: {}', exc) + class CoverArtArchive(RemoteArtSource): NAME = u"Cover Art Archive" @@ -1017,6 +1031,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): u'using {0.LOC_STR} image {1}'.format( source, util.displayable_path(out.path))) break + # else: remove tmp images created by this invalid candidate + source.cleanup_tmp(candidate) if out: break From 4102aae100d07344a67432ee00dfe8ef30f3743b Mon Sep 17 00:00:00 2001 From: Joris Date: Tue, 5 May 2020 10:45:43 +0200 Subject: [PATCH 146/293] Update docs/plugins/subsonicplaylist.rst Co-authored-by: Adrian Sampson --- docs/plugins/subsonicplaylist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/subsonicplaylist.rst b/docs/plugins/subsonicplaylist.rst index b9a54c3de..98c83ebe1 100644 --- a/docs/plugins/subsonicplaylist.rst +++ b/docs/plugins/subsonicplaylist.rst @@ -40,4 +40,4 @@ Here's an example configuration with all the available options and their default username: '' password: '' -Parameters `base_url`, `username` and `password` must be defined! +The `base_url`, `username`, and `password` options are required. From 1a79ae5a9f511394ed5767e270bd9737110717f6 Mon Sep 17 00:00:00 2001 From: pants108 Date: Tue, 5 May 2020 18:39:05 +0000 Subject: [PATCH 147/293] code review 1 --- beetsplug/fetchart.py | 14 +++++--------- docs/changelog.rst | 4 +++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index d4ab714de..70477a624 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -210,8 +210,8 @@ class ArtSource(RequestMixin): def fetch_image(self, candidate, plugin): raise NotImplementedError() - def cleanup_tmp(self, candidate): - raise NotImplementedError() + def cleanup(self, candidate): + pass class LocalArtSource(ArtSource): @@ -221,10 +221,6 @@ class LocalArtSource(ArtSource): def fetch_image(self, candidate, plugin): pass - def cleanup_tmp(self, candidate): - # local art source does not create tmp files - pass - class RemoteArtSource(ArtSource): IS_LOCAL = False @@ -298,7 +294,7 @@ class RemoteArtSource(ArtSource): self._log.debug(u'error fetching art: {}', exc) return - def cleanup_tmp(self, candidate): + def cleanup(self, candidate): if candidate.path: try: util.remove(path=candidate.path) @@ -1031,8 +1027,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): u'using {0.LOC_STR} image {1}'.format( source, util.displayable_path(out.path))) break - # else: remove tmp images created by this invalid candidate - source.cleanup_tmp(candidate) + # Remove temporary files for invalid candidates. + source.cleanup(candidate) if out: break diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d8c26bb9..c09b1b8ab 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -122,9 +122,11 @@ New features: Fixes: -* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take +* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take environment variables such as proxy servers into account when making requests :bug:`3450` +* :doc:`/plugins/fetchart`: Temporary files for fetched album art that fail + validation are now removed * :doc:`/plugins/inline`: In function-style field definitions that refer to flexible attributes, values could stick around from one function invocation to the next. This meant that, when displaying a list of objects, later From 7c71bb87a2f549c063af979b50e2160cd32058ed Mon Sep 17 00:00:00 2001 From: soergeld Date: Fri, 8 May 2020 16:32:12 +0200 Subject: [PATCH 148/293] cleaning up, renaming dup_XXInfo() to copy() --- beets/autotag/hooks.py | 23 +++++-------- beetsplug/discogs.py | 19 ++++------ test/test_autotag.py | 78 +++++++++++++++--------------------------- test/test_ui.py | 5 +-- 4 files changed, 44 insertions(+), 81 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index c2775bf32..6e0cada53 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -80,9 +80,9 @@ class AlbumInfo(AttrDict): ``mediums`` along with the fields up through ``tracks`` are required. The others are optional and may be None. """ - def __init__(self, album=None, album_id=None, artist=None, artist_id=None, - tracks=None, asin=None, albumtype=None, va=False, year=None, - month=None, day=None, label=None, mediums=None, + def __init__(self, tracks, album=None, album_id=None, artist=None, + artist_id=None, asin=None, albumtype=None, va=False, + year=None, month=None, day=None, label=None, mediums=None, artist_sort=None, releasegroup_id=None, catalognum=None, script=None, language=None, country=None, style=None, genre=None, albumstatus=None, media=None, albumdisambig=None, @@ -144,18 +144,13 @@ class AlbumInfo(AttrDict): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) - if 'tracks' in self: - for track in self.tracks: - track.decode(codec) + for track in self.tracks: + track.decode(codec) - def dup_albuminfo(self): - dupe = AlbumInfo() + def copy(self): + dupe = AlbumInfo([]) dupe.update(self) - if 'tracks' in self: - tracks = [] - for track in self.tracks: - tracks.append(track.dup_trackinfo()) - dupe.tracks = tracks + dupe.tracks = [track.copy() for track in self.tracks] return dupe @@ -219,7 +214,7 @@ class TrackInfo(AttrDict): if isinstance(value, bytes): setattr(self, fld, value.decode(codec, 'ignore')) - def dup_trackinfo(self): + def copy(self): dupe = TrackInfo() dupe.update(self) return dupe diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index a0a6ea654..038fa809a 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -357,17 +357,13 @@ class DiscogsPlugin(BeetsPlugin): original_year = self.get_master_year(master_id) if master_id else year return AlbumInfo(album=album, album_id=album_id, artist=artist, - artist_id=artist_id, tracks=tracks, asin=None, - albumtype=albumtype, va=va, year=year, month=None, - day=None, label=label, mediums=len(set(mediums)), - artist_sort=None, releasegroup_id=master_id, - catalognum=catalogno, script=None, language=None, + artist_id=artist_id, tracks=tracks, + albumtype=albumtype, va=va, year=year, + label=label, mediums=len(set(mediums)), + releasegroup_id=master_id, catalognum=catalogno, country=country, style=style, genre=genre, - albumstatus=None, media=media, - albumdisambig=None, artist_credit=None, - original_year=original_year, original_month=None, - original_day=None, data_source='Discogs', - data_url=data_url, + media=media, original_year=original_year, + data_source='Discogs', data_url=data_url, discogs_albumid=discogs_albumid, discogs_labelid=labelid, discogs_artistid=artist_id) @@ -570,8 +566,7 @@ class DiscogsPlugin(BeetsPlugin): length = self.get_track_length(track['duration']) return TrackInfo(title=title, track_id=track_id, artist=artist, artist_id=artist_id, length=length, index=index, - medium=medium, medium_index=medium_index, - artist_sort=None, disctitle=None, artist_credit=None) + medium=medium, medium_index=medium_index) def get_track_index(self, position): """Returns the medium, medium index and subtrack index for a discogs diff --git a/test/test_autotag.py b/test/test_autotag.py index cef6364d0..febd1641d 100644 --- a/test/test_autotag.py +++ b/test/test_autotag.py @@ -350,9 +350,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) self.assertEqual(self._dist(items, info), 0) @@ -364,9 +362,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) dist = self._dist(items, info) self.assertNotEqual(dist, 0) @@ -382,9 +378,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'someone else', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) self.assertNotEqual(self._dist(items, info), 0) @@ -397,9 +391,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'should be ignored', album=u'some album', tracks=_make_trackinfo(), - va=True, - album_id=None, - artist_id=None, + va=True ) self.assertEqual(self._dist(items, info), 0) @@ -413,9 +405,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'should be ignored', album=u'some album', tracks=_make_trackinfo(), - va=True, - album_id=None, - artist_id=None, + va=True ) info.tracks[0].artist = None info.tracks[1].artist = None @@ -431,9 +421,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=True, - album_id=None, - artist_id=None, + va=True ) self.assertNotEqual(self._dist(items, info), 0) @@ -446,9 +434,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) dist = self._dist(items, info) self.assertTrue(0 < dist < 0.2) @@ -462,9 +448,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) info.tracks[0].medium_index = 1 info.tracks[1].medium_index = 2 @@ -481,9 +465,7 @@ class AlbumDistanceTest(_common.TestCase): artist=u'some artist', album=u'some album', tracks=_make_trackinfo(), - va=False, - album_id=None, - artist_id=None, + va=False ) info.tracks[0].medium_index = 1 info.tracks[1].medium_index = 2 @@ -505,9 +487,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 2)) items.append(self.item(u'two', 3)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'two', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'two')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -524,9 +506,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'three', 1)) items.append(self.item(u'two', 1)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'two', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'two')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -542,9 +524,9 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'one', 1)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'two', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'two')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, []) @@ -560,8 +542,8 @@ class AssignmentTest(unittest.TestCase): items.append(self.item(u'two', 2)) items.append(self.item(u'three', 3)) trackinfo = [] - trackinfo.append(TrackInfo(title=u'one', track_id=None)) - trackinfo.append(TrackInfo(title=u'three', track_id=None)) + trackinfo.append(TrackInfo(title=u'one')) + trackinfo.append(TrackInfo(title=u'three')) mapping, extra_items, extra_tracks = \ match.assign_items(items, trackinfo) self.assertEqual(extra_items, [items[1]]) @@ -597,7 +579,7 @@ class AssignmentTest(unittest.TestCase): items.append(item(12, 186.45916150485752)) def info(index, title, length): - return TrackInfo(title=title, track_id=None, length=length, + return TrackInfo(title=title, length=length, index=index) trackinfo = [] trackinfo.append(info(1, u'Alone', 238.893)) @@ -752,15 +734,13 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].albumtype, 'album') def test_album_artist_overrides_empty_track_artist(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() self._apply(info=my_info) self.assertEqual(self.items[0].artist, 'artistNew') self.assertEqual(self.items[1].artist, 'artistNew') def test_album_artist_overridden_by_nonempty_track_artist(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.tracks[0].artist = 'artist1!' my_info.tracks[1].artist = 'artist2!' self._apply(info=my_info) @@ -782,8 +762,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[1].artist_sort, 'albumArtistSort') def test_full_date_applied(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.year = 2013 my_info.month = 12 my_info.day = 18 @@ -798,8 +777,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.items.append(Item(year=1, month=2, day=3)) self.items.append(Item(year=4, month=5, day=6)) - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.year = 2013 self._apply(info=my_info) @@ -819,8 +797,7 @@ class ApplyTest(_common.TestCase, ApplyTestUtil): self.assertEqual(self.items[0].day, 3) def test_data_source_applied(self): - # make a deepcopy of self.info - my_info = self.info.dup_albuminfo() + my_info = self.info.copy() my_info.data_source = 'MusicBrainz' self._apply(info=my_info) @@ -882,8 +859,7 @@ class ApplyCompilationTest(_common.TestCase, ApplyTestUtil): self.assertFalse(self.items[1].comp) def test_va_flag_sets_comp(self): - # make a deepcopy of self.info - va_info = self.info.dup_albuminfo() + va_info = self.info.copy() va_info.va = True self._apply(info=va_info) self.assertTrue(self.items[0].comp) diff --git a/test/test_ui.py b/test/test_ui.py index d0be10060..b1e7e8fad 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -1138,10 +1138,7 @@ class SummarizeItemsTest(_common.TestCase): self.assertEqual(summary, u"1 items, F, 4kbps, 10:54, 987.0 B") # make a copy of self.item - i2 = library.Item() - i2.bitrate = 4321 - i2.length = 10 * 60 + 54 - i2.format = "F" + i2 = self.item.copy() summary = commands.summarize_items([self.item, i2], False) self.assertEqual(summary, u"2 items, F, 4kbps, 21:48, 1.9 KiB") From 4933671c10b955a6ce7a732b520d0b6c4a9994a0 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Fri, 8 May 2020 21:36:56 +0200 Subject: [PATCH 149/293] review: updated subsonicplaylist.py --- beetsplug/subsonicplaylist.py | 77 ++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 14cd855b1..d02a8919d 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -15,24 +15,44 @@ from __future__ import absolute_import, division, print_function -from hashlib import md5 -import xml.etree.ElementTree as ET import random import string +import xml.etree.ElementTree as ET +from hashlib import md5 from urllib.parse import urlencode import requests -from beets.dbcore.query import SubstringQuery from beets.dbcore import AndQuery - +from beets.dbcore.query import SubstringQuery +from beets.plugins import BeetsPlugin from beets.ui import Subcommand -from beets.plugins import BeetsPlugin - __author__ = 'https://github.com/MrNuggelz' +def filter_to_be_removed(items, keys): + if len(items) > len(keys): + dont_remove = [] + for artist, album, title in keys: + for item in items: + if artist == item['artist'] and \ + album == item['album'] and \ + title == item['title']: + dont_remove.append(item) + return [item for item in items if item not in dont_remove] + else: + def to_be_removed(item): + for artist, album, title in keys: + if artist == item['artist'] and\ + album == item['album'] and\ + title == item['title']: + return False + return True + + return [item for item in items if to_be_removed(item)] + + class SubsonicPlaylistPlugin(BeetsPlugin): def __init__(self): @@ -80,12 +100,12 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def build_playlist(lib, opts, args): self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() - if len(self.config['playlist_names'].as_str_seq()) > 0: + if self.config['playlist_names'].as_str_seq(): playlists = ET.fromstring(self.send('getPlaylists').text)[ 0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ - ' but no erro message found' + ' but no error message found' self._log.warn( playlists.attrib.get('message', alt_error)) return @@ -93,17 +113,17 @@ class SubsonicPlaylistPlugin(BeetsPlugin): for playlist in playlists: if name == playlist.attrib['name']: ids.append(playlist.attrib['id']) - playlist_dict = dict() - for playlist_id in ids: - name, tracks = self.get_playlist(playlist_id) - for track in tracks: - if track not in playlist_dict: - playlist_dict[track] = ';' - playlist_dict[track] += name + ';' + + playlist_dict = self.get_playlists(ids) + # delete old tags if self.config['delete']: - for item in lib.items('subsonic_playlist:";"'): - item.update({'subsonic_playlist': ''}) + existing = list(lib.items('subsonic_playlist:";"')) + to_be_removed = filter_to_be_removed( + existing, + playlist_dict.keys()) + for item in to_be_removed: + item['subsonic_playlist'] = '' with lib.transaction(): item.try_sync(write=True, move=False) @@ -129,10 +149,21 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def send(self, endpoint, params=None): if params is None: params = dict() - url = '{}/rest/{}?u={}&t={}&s={}&v=1.12.0&c=beets'.format( - self.config['base_url'].get(), - endpoint, - self.config['username'], - *self.generate_token()) - resp = requests.get(url + urlencode(params)) + a, b = self.generate_token() + params['u'] = self.config['username'] + params['t'] = a + params['s'] = b + params['v'] = '1.12.0' + params['c'] = 'beets' + resp = requests.get('{}/rest/{}?{}'.format(self.config['base_url'].get(),endpoint,urlencode(params))) return resp + + def get_playlists(self, ids): + output = dict() + for playlist_id in ids: + name, tracks = self.get_playlist(playlist_id) + for track in tracks: + if track not in output: + output[track] = ';' + output[track] += name + ';' + return output \ No newline at end of file From 4b7f42d21458166ef5eefb9b3068d8ff0447016c Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Sat, 9 May 2020 12:29:52 +0200 Subject: [PATCH 150/293] Update AttrDict docstring Co-authored-by: Adrian Sampson --- beets/autotag/hooks.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 6e0cada53..9bdf6b001 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -40,18 +40,8 @@ except AttributeError: # Classes used to represent candidate options. class AttrDict(dict): - """ - Dictionary with flexible attributes - to get an tag value: - value = info.tag - or value = info[tag] - or value = info.get(tag) - or value = getattr(info, tag) - all raise AttributeError when info doesn't have tag - to set a tag value: - info.tag = value - or info[tag] = value - or setattr(info, tag, value) + """A dictionary that supports attribute ("dot") access, so `d.field` + is equivalent to `d['field']`. """ def __getattr__(self, attr): From 8fa103e0de4de0ebb01685345f44e229cf4dab21 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sat, 9 May 2020 12:44:36 +0200 Subject: [PATCH 151/293] changelog entry --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2f1521eb..5434e8b13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -115,6 +115,9 @@ New features: :bug:`3459` * :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. :bug:`3530` +* The classes ``AlbumInfo`` and ``TrackInfo`` now have flexible attributes, + allowing to solve :bug:`1547`. + Thanks to :user:`dosoe`. Fixes: From 5d90296a20b336349261e1d39a95503a605112f9 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Sat, 9 May 2020 13:16:56 +0200 Subject: [PATCH 152/293] fixed flake8 issues --- beetsplug/subsonicplaylist.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index d02a8919d..95bdaa886 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -155,7 +155,11 @@ class SubsonicPlaylistPlugin(BeetsPlugin): params['s'] = b params['v'] = '1.12.0' params['c'] = 'beets' - resp = requests.get('{}/rest/{}?{}'.format(self.config['base_url'].get(),endpoint,urlencode(params))) + resp = requests.get('{}/rest/{}?{}'.format( + self.config['base_url'].get(), + endpoint, + urlencode(params)) + ) return resp def get_playlists(self, ids): @@ -166,4 +170,4 @@ class SubsonicPlaylistPlugin(BeetsPlugin): if track not in output: output[track] = ';' output[track] += name + ';' - return output \ No newline at end of file + return output From c948530279f6488964b7715e131946484f0fa5dc Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 13:05:55 +0200 Subject: [PATCH 153/293] Fetching parentwork if changed, even if not force --- beetsplug/parentwork.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index d40254696..e2460e875 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -173,13 +173,17 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return hasparent = hasattr(item, 'parentwork') - if force or not hasparent: + workfalse = True + if hasattr(item, 'mb_workid_current'): + workfalse = item.mb_workid_current != item.mb_workid + if force or (not hasparent) or workfalse: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) + parent_info['mb_workid_current'] = item.mb_workid if 'parent_composer' in parent_info: self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], @@ -202,4 +206,5 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return ui.show_model_changes( item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', - 'parent_composer_sort', 'work_date']) + 'parent_composer_sort', 'work_date', + 'mb_workid_current']) From 42e6fdbc9bddb047f94a26e2ae53987a938c8a6c Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:00:50 +0200 Subject: [PATCH 154/293] Instead of applying all tags from a list, apply all not in a list --- beets/autotag/__init__.py | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index f9e38413e..d3bb48117 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -157,52 +157,52 @@ def apply_metadata(album_info, mapping): # Track alt. item.track_alt = track_info.track_alt - # Miscellaneous/nullable metadata. + # Metadata that has already been set misc_fields = { 'album': ( - 'albumtype', - 'label', - 'asin', - 'catalognum', - 'script', - 'language', - 'country', - 'style', - 'genre', - 'discogs_albumid', - 'discogs_artistid', - 'discogs_labelid', - 'albumstatus', - 'albumdisambig', - 'releasegroupdisambig', - 'data_source', + 'va', + 'releasegroup_id', + 'artist_id', + 'album_id', + 'mediums', + 'tracks', + 'year', + 'month', + 'day', + 'artist', + 'artist_credit', ), 'track': ( - 'disctitle', - 'lyricist', - 'media', - 'composer', - 'composer_sort', - 'arranger', - 'work', - 'mb_workid', - 'work_disambig', - 'bpm', - 'initial_key', - 'genre' + 'track_alt', + 'artist_id', + 'release_track_id', + 'medium', + 'index', + 'medium_index', + 'title', + 'artist_credit', + 'artist_sort', + 'artist', + 'track_id', + 'medium_total', + 'data_url' ) } # Don't overwrite fields with empty values unless the # field is explicitly allowed to be overwritten - for field in misc_fields['album']: + for field in album_info.keys(): + if field in misc_fields['album']: + continue clobber = field in config['overwrite_null']['album'].as_str_seq() value = getattr(album_info, field) if value is None and not clobber: continue item[field] = value - for field in misc_fields['track']: + for field in track_info.keys(): + if field in misc_fields['track']: + continue clobber = field in config['overwrite_null']['track'].as_str_seq() value = getattr(track_info, field) if value is None and not clobber: From f39bc1f894606f0e13e8a09b060d9b6991f80741 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:09:51 +0200 Subject: [PATCH 155/293] forgot a tag in the exclude list --- beets/autotag/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index d3bb48117..9cde24e27 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -171,6 +171,7 @@ def apply_metadata(album_info, mapping): 'day', 'artist', 'artist_credit', + 'artist_sort' ), 'track': ( 'track_alt', From e7d7c72ae151f086070c27b88be926835c9e22ef Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:16:57 +0200 Subject: [PATCH 156/293] forgot another one --- beets/autotag/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 9cde24e27..945dd06e9 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -171,7 +171,8 @@ def apply_metadata(album_info, mapping): 'day', 'artist', 'artist_credit', - 'artist_sort' + 'artist_sort', + 'data_url' ), 'track': ( 'track_alt', From cef8ab65367575b59559795f893b5f957ddf79c1 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 17:27:52 +0200 Subject: [PATCH 157/293] Same for tracks --- beets/autotag/__init__.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 945dd06e9..f79606aaa 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -52,20 +52,24 @@ def apply_item_metadata(item, track_info): if track_info.data_source: item.data_source = track_info.data_source - if track_info.lyricist is not None: - item.lyricist = track_info.lyricist - if track_info.composer is not None: - item.composer = track_info.composer - if track_info.composer_sort is not None: - item.composer_sort = track_info.composer_sort - if track_info.arranger is not None: - item.arranger = track_info.arranger - if track_info.work is not None: - item.work = track_info.work - if track_info.mb_workid is not None: - item.mb_workid = track_info.mb_workid - if track_info.work_disambig is not None: - item.work_disambig = track_info.work_disambig + misc_fields = ['artist_id', + 'release_track_id', + 'title', + 'artist_credit', + 'artist_sort', + 'artist', + 'track_id', + 'data_url' + ] + + for field in track_info.keys(): + if field in misc_fields: + continue + clobber = field in config['overwrite_null']['track'].as_str_seq() + value = getattr(track_info, field) + if value is None and not clobber: + continue + item[field] = value # At the moment, the other metadata is left intact (including album # and track number). Perhaps these should be emptied? From 2d15060bf8d4fbb4a677f7cc68e6c7a2fd138676 Mon Sep 17 00:00:00 2001 From: soergeld Date: Sun, 10 May 2020 20:02:54 +0200 Subject: [PATCH 158/293] more forgotten tags --- beets/autotag/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index f79606aaa..670d55212 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -59,7 +59,8 @@ def apply_item_metadata(item, track_info): 'artist_sort', 'artist', 'track_id', - 'data_url' + 'data_url', + 'length' ] for field in track_info.keys(): @@ -191,7 +192,8 @@ def apply_metadata(album_info, mapping): 'artist', 'track_id', 'medium_total', - 'data_url' + 'data_url', + 'length' ) } From 1f93ce5a49fc356566b00da165bc4d35d1ca4bfc Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 11 May 2020 16:14:52 +0200 Subject: [PATCH 159/293] small changes, docs --- beetsplug/parentwork.py | 8 ++++---- docs/plugins/parentwork.rst | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index e2460e875..e4978ebf4 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -174,16 +174,16 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) hasparent = hasattr(item, 'parentwork') workfalse = True - if hasattr(item, 'mb_workid_current'): + if hasattr(item, 'parentwork_workid_current'): workfalse = item.mb_workid_current != item.mb_workid - if force or (not hasparent) or workfalse: + if force or not hasparent or workfalse: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: self._log.debug("error fetching work: {}", e) return parent_info = self.get_info(item, work_info) - parent_info['mb_workid_current'] = item.mb_workid + parent_info['parentwork_workid_current'] = item.mb_workid if 'parent_composer' in parent_info: self._log.debug("Work fetched: {} - {}", parent_info['parentwork'], @@ -207,4 +207,4 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date', - 'mb_workid_current']) + 'parentwork_workid_current']) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index 9707650b4..e25f01395 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -18,7 +18,7 @@ example, all the movements of a symphony. This plugin aims to solve this problem by also fetching the parent work, which would be the whole symphony in this example. -This plugin adds five tags: +This plugin adds six tags: - **parentwork**: The title of the parent work. - **mb_parentworkid**: The musicbrainz id of the parent work. @@ -27,6 +27,9 @@ This plugin adds five tags: - **parent_composer_sort**: The sort name of the parent work composer. - **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. +- **parentwork_workid_current**: The musicbrainz id of the work as it was when + the parentwork was retrieved. This tag exists only for internal bookkeeping, + to keep track of recordings whose work have changed. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From dda7e41aebf45b177e471f5f65d5ca91725aa8c0 Mon Sep 17 00:00:00 2001 From: soergeld Date: Mon, 11 May 2020 16:41:20 +0200 Subject: [PATCH 160/293] various small changes --- beets/autotag/__init__.py | 101 +++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 56 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 670d55212..dea1fe756 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -35,6 +35,41 @@ from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') +# metadata that is already hardcoded +MISC_FIELDS = { + 'album': ( + 'va', + 'releasegroup_id', + 'artist_id', + 'album_id', + 'mediums', + 'tracks', + 'year', + 'month', + 'day', + 'artist', + 'artist_credit', + 'artist_sort', + 'data_url' + ), + 'track': ( + 'track_alt', + 'artist_id', + 'release_track_id', + 'medium', + 'index', + 'medium_index', + 'title', + 'artist_credit', + 'artist_sort', + 'artist', + 'track_id', + 'medium_total', + 'data_url', + 'length' + ) +} + # Additional utilities for the main interface. @@ -52,23 +87,11 @@ def apply_item_metadata(item, track_info): if track_info.data_source: item.data_source = track_info.data_source - misc_fields = ['artist_id', - 'release_track_id', - 'title', - 'artist_credit', - 'artist_sort', - 'artist', - 'track_id', - 'data_url', - 'length' - ] - - for field in track_info.keys(): - if field in misc_fields: + for field, value in track_info.items(): + # only overwrite fields that are not already hardcoded + if field in MISC_FIELDS['track']: continue - clobber = field in config['overwrite_null']['track'].as_str_seq() - value = getattr(track_info, field) - if value is None and not clobber: + if value is None: continue item[field] = value @@ -162,54 +185,20 @@ def apply_metadata(album_info, mapping): # Track alt. item.track_alt = track_info.track_alt - # Metadata that has already been set - misc_fields = { - 'album': ( - 'va', - 'releasegroup_id', - 'artist_id', - 'album_id', - 'mediums', - 'tracks', - 'year', - 'month', - 'day', - 'artist', - 'artist_credit', - 'artist_sort', - 'data_url' - ), - 'track': ( - 'track_alt', - 'artist_id', - 'release_track_id', - 'medium', - 'index', - 'medium_index', - 'title', - 'artist_credit', - 'artist_sort', - 'artist', - 'track_id', - 'medium_total', - 'data_url', - 'length' - ) - } - # Don't overwrite fields with empty values unless the # field is explicitly allowed to be overwritten - for field in album_info.keys(): - if field in misc_fields['album']: + for field, value in album_info.items(): + # only overwrite fields that are not already hardcoded + if field in MISC_FIELDS['album']: continue clobber = field in config['overwrite_null']['album'].as_str_seq() - value = getattr(album_info, field) if value is None and not clobber: continue item[field] = value - for field in track_info.keys(): - if field in misc_fields['track']: + for field, value in track_info.items(): + # only overwrite fields that are not already hardcoded + if field in MISC_FIELDS['track']: continue clobber = field in config['overwrite_null']['track'].as_str_seq() value = getattr(track_info, field) From fd1fd2182b3ee3af6aa775f3127ade69594ac1d4 Mon Sep 17 00:00:00 2001 From: MrNuggelz Date: Mon, 11 May 2020 20:09:04 +0200 Subject: [PATCH 161/293] changed "SubstringQuery" to "MatchQuery" to prevent wrong song selection --- beetsplug/subsonicplaylist.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 95bdaa886..c537c4093 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -24,7 +24,7 @@ from urllib.parse import urlencode import requests from beets.dbcore import AndQuery -from beets.dbcore.query import SubstringQuery +from beets.dbcore.query import MatchQuery from beets.plugins import BeetsPlugin from beets.ui import Subcommand @@ -71,9 +71,9 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def update_tags(self, playlist_dict, lib): with lib.transaction(): for query, playlist_tag in playlist_dict.items(): - query = AndQuery([SubstringQuery("artist", query[0]), - SubstringQuery("album", query[1]), - SubstringQuery("title", query[2])]) + query = AndQuery([MatchQuery("artist", query[0]), + MatchQuery("album", query[1]), + MatchQuery("title", query[2])]) items = lib.items(query) if not items: self._log.warn(u"{} | track not found ({})", playlist_tag, From 918fea5ecaa729de335a8da8eb170021559caafb Mon Sep 17 00:00:00 2001 From: soergeld Date: Tue, 12 May 2020 16:05:08 +0200 Subject: [PATCH 162/293] small changes, documentation --- beetsplug/parentwork.py | 6 +++--- docs/plugins/parentwork.rst | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index e4978ebf4..781485ba4 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -173,10 +173,10 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) return hasparent = hasattr(item, 'parentwork') - workfalse = True + work_changed = True if hasattr(item, 'parentwork_workid_current'): - workfalse = item.mb_workid_current != item.mb_workid - if force or not hasparent or workfalse: + work_changed = item.mb_workid_current != item.mb_workid + if force or not hasparent or work_changed: try: work_info, work_date = find_parentwork_info(item.mb_workid) except musicbrainzngs.musicbrainz.WebServiceError as e: diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index e25f01395..a3b203c7b 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -18,6 +18,12 @@ example, all the movements of a symphony. This plugin aims to solve this problem by also fetching the parent work, which would be the whole symphony in this example. +It uses the ``mb_workid`` tag to retrieve the *parent work* and stores the +``mb_workid`` as ``parentwork_workid_current``. When one executes +``beet parentwork`` it checks if ``mb_workid`` has changed by comparing it +with ``parentwork_workid_current``, if it has changed or if ``force`` is +enabled it fetches the *parent work* again. + This plugin adds six tags: - **parentwork**: The title of the parent work. @@ -27,7 +33,7 @@ This plugin adds six tags: - **parent_composer_sort**: The sort name of the parent work composer. - **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. -- **parentwork_workid_current**: The musicbrainz id of the work as it was when +- **parentwork_workid_current**: The Musicbrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, to keep track of recordings whose work have changed. @@ -41,7 +47,8 @@ To configure the plugin, make a ``parentwork:`` section in your configuration file. The available options are: - **force**: As a default, ``parentwork`` only fetches work info for - recordings that do not already have a ``parentwork`` tag. If ``force`` + recordings that do not already have a ``parentwork`` tag or where + ``mb_workid`` differs from ``parentwork_workid_current``. If ``force`` is enabled, it fetches it for all recordings. Default: ``no`` From 2923e90da1f3d0887d582024f2cefee56db5e923 Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 00:16:18 +0200 Subject: [PATCH 163/293] cosmetics --- beets/autotag/__init__.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index dea1fe756..5fc5b1c6c 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -36,7 +36,7 @@ from .match import Recommendation # noqa log = logging.getLogger('beets') # metadata that is already hardcoded -MISC_FIELDS = { +SPECIAL_FIELDS = { 'album': ( 'va', 'releasegroup_id', @@ -84,12 +84,10 @@ def apply_item_metadata(item, track_info): item.mb_releasetrackid = track_info.release_track_id if track_info.artist_id: item.mb_artistid = track_info.artist_id - if track_info.data_source: - item.data_source = track_info.data_source for field, value in track_info.items(): # only overwrite fields that are not already hardcoded - if field in MISC_FIELDS['track']: + if field in SPECIAL_FIELDS['track']: continue if value is None: continue @@ -188,8 +186,7 @@ def apply_metadata(album_info, mapping): # Don't overwrite fields with empty values unless the # field is explicitly allowed to be overwritten for field, value in album_info.items(): - # only overwrite fields that are not already hardcoded - if field in MISC_FIELDS['album']: + if field in SPECIAL_FIELDS['album']: continue clobber = field in config['overwrite_null']['album'].as_str_seq() if value is None and not clobber: @@ -197,8 +194,7 @@ def apply_metadata(album_info, mapping): item[field] = value for field, value in track_info.items(): - # only overwrite fields that are not already hardcoded - if field in MISC_FIELDS['track']: + if field in SPECIAL_FIELDS['track']: continue clobber = field in config['overwrite_null']['track'].as_str_seq() value = getattr(track_info, field) From 5db2b4ccaa7223345b02626df188476aabc838ae Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 13 May 2020 08:20:50 -0400 Subject: [PATCH 164/293] Fix new flake8 warnings --- beetsplug/replaygain.py | 2 +- test/test_player.py | 2 ++ test/test_playlist.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 6f15b454e..0ba04783d 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -613,7 +613,7 @@ class FfmpegBackend(Backend): return float(value) except ValueError: raise ReplayGainError( - u"ffmpeg output: expected float value, found {1}" + u"ffmpeg output: expected float value, found {0}" .format(value) ) diff --git a/test/test_player.py b/test/test_player.py index 021a04fc0..dd47ac62b 100644 --- a/test/test_player.py +++ b/test/test_player.py @@ -241,6 +241,8 @@ def implements(commands, expectedFailure=False): # noqa: N803 bluelet_listener = bluelet.Listener + + @mock.patch("beets.util.bluelet.Listener") def start_server(args, assigned_port, listener_patch): """Start the bpd server, writing the port to `assigned_port`. diff --git a/test/test_playlist.py b/test/test_playlist.py index edd98e711..88abb733f 100644 --- a/test/test_playlist.py +++ b/test/test_playlist.py @@ -120,7 +120,7 @@ class PlaylistQueryTestHelper(PlaylistTestHelper): ])) def test_name_query_with_nonexisting_playlist(self): - q = u'playlist:nonexisting'.format(self.playlist_dir) + q = u'playlist:nonexisting' results = self.lib.items(q) self.assertEqual(set(results), set()) From e8b57dfda6057404906dfffefabf0e562923faf3 Mon Sep 17 00:00:00 2001 From: Dorian Soergel Date: Wed, 13 May 2020 14:55:00 +0200 Subject: [PATCH 165/293] Update docs/plugins/parentwork.rst Co-authored-by: Adrian Sampson --- docs/plugins/parentwork.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index a3b203c7b..1aea9e9ff 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -35,7 +35,7 @@ This plugin adds six tags: that has a composition date. Format: yyyy-mm-dd. - **parentwork_workid_current**: The Musicbrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, - to keep track of recordings whose work have changed. + to keep track of recordings whose works have changed. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From 26a78a0c280c480601258c151e8185e9167a6b3c Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 16:54:04 +0200 Subject: [PATCH 166/293] Cosmetics --- beets/autotag/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 5fc5b1c6c..057daa405 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -35,7 +35,7 @@ from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') -# metadata that is already hardcoded +# Metadata that is already hardcoded. SPECIAL_FIELDS = { 'album': ( 'va', @@ -86,7 +86,7 @@ def apply_item_metadata(item, track_info): item.mb_artistid = track_info.artist_id for field, value in track_info.items(): - # only overwrite fields that are not already hardcoded + # We only overwrite fields that are not already hardcoded. if field in SPECIAL_FIELDS['track']: continue if value is None: From 66085a231f88c1fcec91c7ef1807af91a1f1e084 Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 17:42:49 +0200 Subject: [PATCH 167/293] Cosmetics --- beets/autotag/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 057daa405..7ab0d57fd 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -35,7 +35,7 @@ from .match import Recommendation # noqa # Global logger. log = logging.getLogger('beets') -# Metadata that is already hardcoded. +# Metadata fields that are already hardcoded, or where the tag name changes. SPECIAL_FIELDS = { 'album': ( 'va', From ed97b984b02cc2ff41da9b1ec73931687b11ce79 Mon Sep 17 00:00:00 2001 From: soergeld Date: Wed, 13 May 2020 18:02:14 +0200 Subject: [PATCH 168/293] Documentation --- docs/plugins/parentwork.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index 1aea9e9ff..cc3f21c0c 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -18,22 +18,24 @@ example, all the movements of a symphony. This plugin aims to solve this problem by also fetching the parent work, which would be the whole symphony in this example. -It uses the ``mb_workid`` tag to retrieve the *parent work* and stores the -``mb_workid`` as ``parentwork_workid_current``. When one executes -``beet parentwork`` it checks if ``mb_workid`` has changed by comparing it -with ``parentwork_workid_current``, if it has changed or if ``force`` is -enabled it fetches the *parent work* again. +The plugin can detect changes in ``mb_workid`` so it knows when to re-fetch +other metadata, such as ``parentwork``. To do this, when it runs, it stores a +copy of ``mb_workid`` in the bookkeeping field ``parentwork_workid_current``. +At any later run of ``beet parentwork`` it will check if the tags +``mb_workid`` and ``parentwork_workid_current`` are still identical. If it is +not the case, it means the work has changed and all the tags need to be +fetched again. This plugin adds six tags: - **parentwork**: The title of the parent work. -- **mb_parentworkid**: The musicbrainz id of the parent work. +- **mb_parentworkid**: The MusicBrainz id of the parent work. - **parentwork_disambig**: The disambiguation of the parent work title. - **parent_composer**: The composer of the parent work. - **parent_composer_sort**: The sort name of the parent work composer. - **work_date**: The composition date of the work, or the first parent work that has a composition date. Format: yyyy-mm-dd. -- **parentwork_workid_current**: The Musicbrainz id of the work as it was when +- **parentwork_workid_current**: The MusicBrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, to keep track of recordings whose works have changed. @@ -53,5 +55,5 @@ configuration file. The available options are: Default: ``no`` - **auto**: If enabled, automatically fetches works at import. It takes quite - some time, because beets is restricted to one musicbrainz query per second. + some time, because beets is restricted to one MusicBrainz query per second. Default: ``no`` From fe418b946ee8e2c4fd62c86c36ec8059fb5c57c3 Mon Sep 17 00:00:00 2001 From: soergeld Date: Thu, 14 May 2020 22:40:37 +0200 Subject: [PATCH 169/293] forgot one mb_workid_current occurence --- beetsplug/parentwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index 781485ba4..dea7ba040 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -175,7 +175,7 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) hasparent = hasattr(item, 'parentwork') work_changed = True if hasattr(item, 'parentwork_workid_current'): - work_changed = item.mb_workid_current != item.mb_workid + work_changed = item.parentwork_workid_current != item.mb_workid if force or not hasparent or work_changed: try: work_info, work_date = find_parentwork_info(item.mb_workid) From 05d202760bb75290477ea7be341cb76a3e58751d Mon Sep 17 00:00:00 2001 From: Thorsten Date: Sat, 16 May 2020 00:34:10 +0200 Subject: [PATCH 170/293] Update genres.txt Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres --- beetsplug/lastgenre/genres.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beetsplug/lastgenre/genres.txt b/beetsplug/lastgenre/genres.txt index 914ee1290..7ccd7ad3b 100644 --- a/beetsplug/lastgenre/genres.txt +++ b/beetsplug/lastgenre/genres.txt @@ -450,6 +450,8 @@ emo rap emocore emotronic enka +epic doom metal +epic metal eremwu eu ethereal pop ethereal wave @@ -1024,6 +1026,7 @@ neo-medieval neo-prog neo-psychedelia neoclassical +neoclassical metal neoclassical music neofolk neotraditional country @@ -1176,8 +1179,10 @@ progressive folk progressive folk music progressive house progressive metal +progressive power metal progressive rock progressive trance +progressive thrash metal protopunk psych folk psychedelic music @@ -1396,6 +1401,7 @@ symphonic metal symphonic poem symphonic rock symphony +synthcore synthpop synthpunk t'ong guitar @@ -1428,6 +1434,7 @@ tejano tejano music tekno tembang sunda +teutonic thrash metal texas blues thai pop thillana @@ -1444,6 +1451,7 @@ toeshey togaku trad jazz traditional bluegrass +traditional heavy metal traditional pop music trallalero trance From 852cdb6dfa2e34c112442a2078c2b85c9f2501d0 Mon Sep 17 00:00:00 2001 From: Thorsten Date: Sat, 16 May 2020 00:34:37 +0200 Subject: [PATCH 171/293] Update genres-tree.yaml Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres --- beetsplug/lastgenre/genres-tree.yaml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/beetsplug/lastgenre/genres-tree.yaml b/beetsplug/lastgenre/genres-tree.yaml index a09f7e6b3..c8ae42478 100644 --- a/beetsplug/lastgenre/genres-tree.yaml +++ b/beetsplug/lastgenre/genres-tree.yaml @@ -648,35 +648,51 @@ - glam rock - hard rock - heavy metal: - - alternative metal + - alternative metal: + - funk metal - black metal: - viking metal - christian metal - death metal: + - death/doom - goregrind - melodic death metal - technical death metal - - doom metal + - doom metal: + - epic doom metal + - funeral doom - drone metal + - epic metal - folk metal: - celtic metal - medieval metal + - pagan metal - funk metal - glam metal - gothic metal + - industrial metal: + - industrial death metal - metalcore: - deathcore - mathcore: - djent - - power metal + - synthcore + - neoclassical metal + - post-metal + - power metal: + - progressive power metal - progressive metal - sludge metal - speed metal - - stoner rock + - stoner rock: + - stoner metal - symphonic metal - thrash metal: - crossover thrash - groove metal + - progressive thrash metal + - teutonic thrash metal + - traditional heavy metal - math rock - new wave: - world fusion @@ -719,6 +735,7 @@ - street punk - thrashcore - horror punk + - oi! - pop punk - psychobilly - riot grrrl From cd4d8c7f9c3d8fa3fd1706befc223dd920bdef65 Mon Sep 17 00:00:00 2001 From: Thorsten Date: Sat, 16 May 2020 00:51:30 +0200 Subject: [PATCH 172/293] Update changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 545145f0d..f00380a2c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. From ac043f9be0094089ed29ec8c0552781994945607 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 13:14:39 +0200 Subject: [PATCH 173/293] When fetching lyrics from Genius, search for title and artist. Searching only for the title and just verifying the artist afterwards leads to songs with very common titles not being found, since Genius limits the amount of returned hits. An example would be 'Saviour' by 'Circa Waves'. --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 20e1c8858..3a56cc738 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -390,7 +390,7 @@ class Genius(Backend): def fetch(self, artist, title): search_url = self.base_url + "/search" - data = {'q': title} + data = {'q': title + " " + artist.lower()} try: response = requests.get(search_url, data=data, headers=self.headers) From 46143d97629c48777e82dc147ecfff08f7c5e683 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 16:28:17 +0200 Subject: [PATCH 174/293] Remove unnecessary intermediate web request to genius.com when fetching lyrics. The search results already include the correct song page url, making it superfluous to do another request via the /song api just to get it. --- beetsplug/lyrics.py | 20 ++++---------------- test/test_lyrics.py | 8 ++++---- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 3a56cc738..71969cab3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -359,14 +359,9 @@ class Genius(Backend): 'User-Agent': USER_AGENT, } - def lyrics_from_song_api_path(self, song_api_path): - song_url = self.base_url + song_api_path - response = requests.get(song_url, headers=self.headers) - json = response.json() - path = json["response"]["song"]["path"] - + def lyrics_from_song_page(self, page_url): # Gotta go regular html scraping... come on Genius. - page_url = "https://genius.com" + path + self._log.debug(u'fetching lyrics from: {0}', page_url) try: page = requests.get(page_url) except requests.RequestException as exc: @@ -404,7 +399,6 @@ class Genius(Backend): self._log.debug(u'Genius API request returned invalid JSON') return None - song_info = None for hit in json["response"]["hits"]: # Genius uses zero-width characters to denote lowercase # artist names. @@ -412,15 +406,9 @@ class Genius(Backend): strip(u'\u200b').lower() if hit_artist == artist.lower(): - song_info = hit - break + return self.lyrics_from_song_page(hit["result"]["url"]) - if song_info: - self._log.debug(u'fetched: {0}', song_info["result"]["url"]) - song_api_path = song_info["result"]["api_path"] - return self.lyrics_from_song_api_path(song_api_path) - else: - self._log.debug(u'genius: no matching artist') + self._log.debug(u'genius: no matching artist') class LyricsWiki(SymbolsReplaced): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 81d62fea9..d0507c16f 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -456,7 +456,7 @@ class LyricsGeniusBaseTest(unittest.TestCase): self.skipTest("Python's built-in HTML parser is not good enough") -class LyricsGeniusScrapTest(LyricsGeniusBaseTest): +class LyricsGeniusScrapeTest(LyricsGeniusBaseTest): """Checks that Genius backend works as intended. """ @@ -469,12 +469,12 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): - """Ensure that `lyrics_from_song_api_path` doesn't crash when the html - for a Genius page contain
+ """Ensure that `lyrics_from_song_page` doesn't crash when the html + for a Genius page doesn't contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None - self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + self.assertEqual(genius.lyrics_from_song_page('https://genius.com/sample'), None) From 15402f6aa7bceceef8f3909b5e5939342bd549ca Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 17:15:45 +0200 Subject: [PATCH 175/293] Add alternative scraping algorithm to deal with Genius's new song page html layout. --- beetsplug/lyrics.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 71969cab3..27a29e149 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,13 +373,34 @@ class Genius(Backend): # Remove script tags that they put in the middle of the lyrics. [h.extract() for h in html('script')] - # At least Genius is nice and has a tag called 'lyrics'! - # Updated css where the lyrics are based in HTML. + # Most of the time, the page contains a div with class="lyrics" where + # all of the lyrics can be found already correctly formatted + # Sometimes, though, it packages the lyrics into separate divs, most + # likely for easier ad placement lyrics_div = html.find("div", class_="lyrics") - if lyrics_div is None: - self._log.debug(u'Genius lyrics for {0} not found', - page_url) - return None + if not lyrics_div: + self._log.debug(u'Received unusual song page html') + verse_div = html.find("div", + class_=re.compile("Lyrics__Container")) + if not verse_div: + with open('instrumental.html', 'w') as text_file: + text_file.write(str(html)) + if html.find("div", + class_=re.compile("LyricsPlaceholder__Message"), + string="This song is an instrumental"): + self._log.debug('Detected instrumental') + return "[Instrumental]" + else: + self._log.debug("Couldn't scrape page using known layouts") + return None + + lyrics_div = verse_div.parent + for br in lyrics_div.find_all("br"): + br.replace_with("\n") + ads = lyrics_div.find_all("div", + class_=re.compile("InreadAd__Container")) + for ad in ads: + ad.replace_with("\n") return lyrics_div.get_text() From 5d306d6fd9b48b51cbf25934cd5ded90a83b1b02 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sat, 16 May 2020 17:48:56 +0200 Subject: [PATCH 176/293] Remove debugging statement and conform to line length. --- beetsplug/lyrics.py | 2 -- test/test_lyrics.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 27a29e149..71e1d4ef3 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -383,8 +383,6 @@ class Genius(Backend): verse_div = html.find("div", class_=re.compile("Lyrics__Container")) if not verse_div: - with open('instrumental.html', 'w') as text_file: - text_file.write(str(html)) if html.find("div", class_=re.compile("LyricsPlaceholder__Message"), string="This song is an instrumental"): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index d0507c16f..d31116284 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -474,7 +474,8 @@ class LyricsGeniusScrapeTest(LyricsGeniusBaseTest): """ # https://github.com/beetbox/beets/issues/3535 # expected return value None - self.assertEqual(genius.lyrics_from_song_page('https://genius.com/sample'), + song_url = 'https://genius.com/sample' + self.assertEqual(genius.lyrics_from_song_page(song_url), None) From 5298e1a97656ab5c8f33f95a5a537c645e35cca1 Mon Sep 17 00:00:00 2001 From: stlutz Date: Sun, 17 May 2020 18:42:23 +0200 Subject: [PATCH 177/293] Add changelog entry. --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f00380a2c..bd1a77973 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -202,6 +202,7 @@ Fixes: The log message has also been rewritten for to improve clarity. Thanks to :user:`autrimpo`. :bug:`3533` +* :doc:`/plugins/lyrics`: Reduce the failure rate of the Genius backend. For plugin developers: From 5c91013d1f291b60b295f0ebb3dc0a17d88fd2f4 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 17 May 2020 14:46:10 -0400 Subject: [PATCH 178/293] Expand changelog for #3594 --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bd1a77973..3f2890913 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -202,7 +202,9 @@ Fixes: The log message has also been rewritten for to improve clarity. Thanks to :user:`autrimpo`. :bug:`3533` -* :doc:`/plugins/lyrics`: Reduce the failure rate of the Genius backend. +* :doc:`/plugins/lyrics`: Adapt the Genius backend to changes in markup to + reduce the scraping failure rate. + :bug:`3535` :bug:`3594` For plugin developers: From 1374aa2aea1df83b11c77218c354c54de991df59 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 18 May 2020 10:23:26 -0400 Subject: [PATCH 179/293] feat: uses new subsonic format as default also gets rid of warning that port must be `int` when it's default was a `string` --- beetsplug/subsonicupdate.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 469a9d142..29b01986e 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -17,11 +17,9 @@ Your Beets configuration file should contain a "subsonic" section like the following: subsonic: - host: 192.168.x.y (Subsonic server IP) - port: 4040 (default) - user: - pass: - contextpath: /subsonic + url: https://mydomain.com:443/subsonic + user: username + pass: password """ from __future__ import division, absolute_import, print_function @@ -84,11 +82,8 @@ class SubsonicUpdate(BeetsPlugin): # Set default configuration values config['subsonic'].add({ - 'host': 'localhost', - 'port': '4040', 'user': 'admin', 'pass': 'admin', - 'contextpath': '/', 'url': 'http://localhost:4040', }) From d9603788bcc6e991773c4fec07a57c42c157ad6b Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 22 May 2020 16:24:49 -0400 Subject: [PATCH 180/293] fix: use `endwith()` instead of `endWith()` --- beetsplug/subsonicupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 29b01986e..354eb701f 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -61,7 +61,7 @@ def format_url(): """ url = config['subsonic']['url'].as_str() - if url and url.endsWith('/'): + if url and url.endswith('/'): url = url[:-1] # @deprecated("Use url config option instead") From 66e6b11404842585a26c3241a9c76b919a4d9c4c Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 22 May 2020 20:35:18 -0400 Subject: [PATCH 181/293] refactor: moved static methods inside class made them static private class methods --- beetsplug/subsonicupdate.py | 86 ++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 354eb701f..45fc3a8cb 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -35,47 +35,6 @@ from beets.plugins import BeetsPlugin __author__ = 'https://github.com/maffo999' -def create_token(): - """Create salt and token from given password. - - :return: The generated salt and hashed token - """ - password = config['subsonic']['pass'].as_str() - - # Pick the random sequence and salt the password - r = string.ascii_letters + string.digits - salt = "".join([random.choice(r) for _ in range(6)]) - salted_password = password + salt - token = hashlib.md5().update(salted_password.encode('utf-8')).hexdigest() - - # Put together the payload of the request to the server and the URL - return salt, token - - -def format_url(): - """Get the Subsonic URL to trigger a scan. Uses either the url - config option or the deprecated host, port, and context_path config - options together. - - :return: Endpoint for updating Subsonic - """ - - url = config['subsonic']['url'].as_str() - if url and url.endswith('/'): - url = url[:-1] - - # @deprecated("Use url config option instead") - if not url: - host = config['subsonic']['host'].as_str() - port = config['subsonic']['port'].get(int) - context_path = config['subsonic']['contextpath'].as_str() - if context_path == '/': - context_path = '' - url = "http://{}:{}{}".format(host, port, context_path) - - return url + '/rest/startScan' - - class SubsonicUpdate(BeetsPlugin): def __init__(self): super(SubsonicUpdate, self).__init__() @@ -90,10 +49,51 @@ class SubsonicUpdate(BeetsPlugin): config['subsonic']['pass'].redact = True self.register_listener('import', self.start_scan) + @staticmethod + def __create_token(): + """Create salt and token from given password. + + :return: The generated salt and hashed token + """ + password = config['subsonic']['pass'].as_str() + + # Pick the random sequence and salt the password + r = string.ascii_letters + string.digits + salt = "".join([random.choice(r) for _ in range(6)]) + salted_password = password + salt + token = hashlib.md5(salted_password.encode('utf-8')).hexdigest() + + # Put together the payload of the request to the server and the URL + return salt, token + + @staticmethod + def __format_url(): + """Get the Subsonic URL to trigger a scan. Uses either the url + config option or the deprecated host, port, and context_path config + options together. + + :return: Endpoint for updating Subsonic + """ + + url = config['subsonic']['url'].as_str() + if url and url.endswith('/'): + url = url[:-1] + + # @deprecated("Use url config option instead") + if not url: + host = config['subsonic']['host'].as_str() + port = config['subsonic']['port'].get(int) + context_path = config['subsonic']['contextpath'].as_str() + if context_path == '/': + context_path = '' + url = "http://{}:{}{}".format(host, port, context_path) + + return url + '/rest/startScan' + def start_scan(self): user = config['subsonic']['user'].as_str() - url = format_url() - salt, token = create_token() + url = self.__format_url() + salt, token = self.__create_token() payload = { 'u': user, From 10f589d80ad83b54dd86edf1ceb677d762b4be3b Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Fri, 22 May 2020 20:35:32 -0400 Subject: [PATCH 182/293] feat: added subsonicupdate tests --- test/test_subsonic.py | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 test/test_subsonic.py diff --git a/test/test_subsonic.py b/test/test_subsonic.py new file mode 100644 index 000000000..6d37cdf4f --- /dev/null +++ b/test/test_subsonic.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +"""Tests for the 'subsonic' plugin""" + +from __future__ import division, absolute_import, print_function + +import requests +import responses +import unittest + +from test import _common +from beets import config +from beetsplug import subsonicupdate +from test.helper import TestHelper +from six.moves.urllib.parse import parse_qs, urlparse + + +class ArgumentsMock(object): + def __init__(self, mode, show_failures): + self.mode = mode + self.show_failures = show_failures + self.verbose = 1 + + +def _params(url): + """Get the query parameters from a URL.""" + return parse_qs(urlparse(url).query) + + +class SubsonicPluginTest(_common.TestCase, TestHelper): + @responses.activate + def setUp(self): + config.clear() + self.setup_beets() + + config["subsonic"]["user"] = "admin" + config["subsonic"]["pass"] = "admin" + config["subsonic"]["url"] = "http://localhost:4040" + + self.subsonicupdate = subsonicupdate.SubsonicUpdate() + + def tearDown(self): + self.teardown_beets() + + @responses.activate + def test_start_scan(self): + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_extra_forward_slash_url(self): + config["subsonic"]["url"] = "http://localhost:4040/contextPath" + + responses.add( + responses.POST, + 'http://localhost:4040/contextPath/rest/startScan', + status=200 + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_context_path(self): + config["subsonic"]["url"] = "http://localhost:4040/" + + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_missing_port(self): + config["subsonic"]["url"] = "http://localhost/airsonic" + + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + with self.assertRaises(requests.exceptions.ConnectionError): + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_missing_schema(self): + config["subsonic"]["url"] = "localhost:4040/airsonic" + + responses.add( + responses.POST, + 'http://localhost:4040/rest/startScan', + status=200 + ) + + with self.assertRaises(requests.exceptions.InvalidSchema): + self.subsonicupdate.start_scan() + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From edb31f69d00868c2c70967e634c41fb54ecbe59e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 23 May 2020 09:03:12 -0400 Subject: [PATCH 183/293] Remove explicit jellyfish dep from Tox This is already installed when installing beets, and it's done in a way that actually gets the right package version for the right Python version. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9dac2d59a..19cb3069b 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,6 @@ deps = rarfile responses>=0.3.0 pyxdg - jellyfish python-mpd2 coverage discogs-client From fc4a61c55768f73b54098cb06465b77887ca4f55 Mon Sep 17 00:00:00 2001 From: Martin Kirchgessner Date: Sat, 23 May 2020 17:09:18 +0200 Subject: [PATCH 184/293] add drop2beets to community plugins --- docs/plugins/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index de2f2930c..aab922fcd 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -318,6 +318,10 @@ Here are a few of the plugins written by the beets community: * `beets-originquery`_ augments MusicBrainz queries with locally-sourced data to improve autotagger results. +* `drop2beets`_ automatically imports singles as soon as they are dropped in a + folder (using Linux's ``inotify``). You can also set a sub-folders + hierarchy to set flexible attributes by the way. + .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beets-check: https://github.com/geigerzaehler/beets-check .. _copyartifacts: https://github.com/sbarakat/beets-copyartifacts @@ -347,3 +351,4 @@ Here are a few of the plugins written by the beets community: .. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser .. _beets-originquery: https://github.com/x1ppy/beets-originquery +.. _drop2beets: https://github.com/martinkirch/drop2beets From 0c3201930c13190eebade4141d314640dfe2b2f3 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Sun, 24 May 2020 11:20:19 -0400 Subject: [PATCH 185/293] feat: added github actions only working with linux for now. exact replica of travis basically. should probably try to implement the rest of tox or deprecate some functionality. --- .github/workflows/ci.yaml | 77 ++++++++++++++++++++++++++++++++ beetsplug/export.py | 14 +++--- beetsplug/lyrics.py | 4 +- beetsplug/metasync/amarok.py | 4 +- beetsplug/plexupdate.py | 4 +- beetsplug/smartplaylist.py | 8 ++-- beetsplug/subsonicplaylist.py | 9 ++-- beetsplug/thumbnails.py | 2 +- setup.py | 4 +- test/test_export.py | 4 +- tox.ini | 83 +++++++++++++++++++---------------- 11 files changed, 150 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..cbd6381e1 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,77 @@ +name: ci +on: push +jobs: + test-27: + runs-on: ${{ matrix.platform }} + strategy: + matrix: + platform: [ ubuntu-latest ] + env: + NOSE_SHOW_SKIPPED: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up python 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + pip install tox sphinx + - name: Test with tox + run: | + tox -e py27-test + tox -e docs + test-3x: + runs-on: ${{ matrix.platform }} + strategy: + matrix: + platform: [ ubuntu-latest ] + python-version: [ 5, 6, 7, 8 ] + env: + NOSE_SHOW_SKIPPED: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up python 3.${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.${{ matrix.python-version }} + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + pip install tox sphinx + - name: Test with tox + run: | + if [[ ${{ matrix.python-version }} == '8' ]]; then + tox -e py3${{ matrix.python-version }}-test + tox -e py3${{ matrix.python-version }}-cov + tox -e py3${{ matrix.python-version }}-flake8 + else + tox -e py3${{ matrix.python-version }}-test + fi + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python to build docs with sphinx + uses: actions/setup-python@v2 + with: + python-version: 2.7 + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox sphinx + - name: Build and check docs using tox + run: tox -e docs \ No newline at end of file diff --git a/beetsplug/export.py b/beetsplug/export.py index f7e84a570..0b4876bbf 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -21,7 +21,7 @@ import sys import codecs import json import csv -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et from datetime import datetime, date from beets.plugins import BeetsPlugin @@ -188,18 +188,18 @@ class XMLFormat(ExportFormat): def export(self, data, **kwargs): # Creates the XML file structure. - library = ET.Element(u'library') - tracks = ET.SubElement(library, u'tracks') + library = et.Element(u'library') + tracks = et.SubElement(library, u'tracks') if data and isinstance(data[0], dict): for index, item in enumerate(data): - track = ET.SubElement(tracks, u'track') + track = et.SubElement(tracks, u'track') for key, value in item.items(): - track_details = ET.SubElement(track, key) + track_details = et.SubElement(track, key) track_details.text = value # Depending on the version of python the encoding needs to change try: - data = ET.tostring(library, encoding='unicode', **kwargs) + data = et.tostring(library, encoding='unicode', **kwargs) except LookupError: - data = ET.tostring(library, encoding='utf-8', **kwargs) + data = et.tostring(library, encoding='utf-8', **kwargs) self.out_stream.write(data) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 71e1d4ef3..f73d72dfd 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -907,7 +907,7 @@ class LyricsPlugin(plugins.BeetsPlugin): return _scrape_strip_cruft(lyrics, True) def append_translation(self, text, to_lang): - import xml.etree.ElementTree as ET + import xml.etree.ElementTree as et if not self.bing_auth_token: self.bing_auth_token = self.get_bing_access_token() @@ -925,7 +925,7 @@ class LyricsPlugin(plugins.BeetsPlugin): self.bing_auth_token = None return self.append_translation(text, to_lang) return text - lines_translated = ET.fromstring(r.text.encode('utf-8')).text + lines_translated = et.fromstring(r.text.encode('utf-8')).text # Use a translation mapping dict to build resulting lyrics translations = dict(zip(text_lines, lines_translated.split('|'))) result = '' diff --git a/beetsplug/metasync/amarok.py b/beetsplug/metasync/amarok.py index 0622fc17a..1d28eb3c9 100644 --- a/beetsplug/metasync/amarok.py +++ b/beetsplug/metasync/amarok.py @@ -49,7 +49,7 @@ class Amarok(MetaSource): 'amarok_lastplayed': DateType(), } - queryXML = u' \ + query_xml = u' \ \ \ \ @@ -72,7 +72,7 @@ class Amarok(MetaSource): # of the result set. So query for the filename and then try to match # the correct item from the results we get back results = self.collection.Query( - self.queryXML % quoteattr(basename(path)) + self.query_xml % quoteattr(basename(path)) ) for result in results: if result['xesam:url'] != path: diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index 17fd8208d..d3d7caf5c 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -12,7 +12,7 @@ Put something like the following in your config.yaml to configure: from __future__ import division, absolute_import, print_function import requests -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et from six.moves.urllib.parse import urljoin, urlencode from beets import config from beets.plugins import BeetsPlugin @@ -28,7 +28,7 @@ def get_music_section(host, port, token, library_name): r = requests.get(url) # Parse xml tree and extract music section key. - tree = ET.fromstring(r.content) + tree = et.fromstring(r.content) for child in tree.findall('Directory'): if child.get('title') == library_name: return child.get('key') diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index 4ffdd21e7..f3157c96f 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -105,17 +105,17 @@ class SmartPlaylistPlugin(BeetsPlugin): playlist_data = (playlist['name'],) try: - for key, Model in (('query', Item), ('album_query', Album)): + for key, model in (('query', Item), ('album_query', Album)): qs = playlist.get(key) if qs is None: query_and_sort = None, None elif isinstance(qs, six.string_types): - query_and_sort = parse_query_string(qs, Model) + query_and_sort = parse_query_string(qs, model) elif len(qs) == 1: - query_and_sort = parse_query_string(qs[0], Model) + query_and_sort = parse_query_string(qs[0], model) else: # multiple queries and sorts - queries, sorts = zip(*(parse_query_string(q, Model) + queries, sorts = zip(*(parse_query_string(q, model) for q in qs)) query = OrQuery(queries) final_sorts = [] diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index c537c4093..a4577278a 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function import random import string -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et from hashlib import md5 from urllib.parse import urlencode @@ -85,7 +85,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def get_playlist(self, playlist_id): xml = self.send('getPlaylist', {'id': playlist_id}).text - playlist = ET.fromstring(xml)[0] + playlist = et.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' self._log.warn(playlist.attrib.get('message', alt_error)) @@ -101,8 +101,9 @@ class SubsonicPlaylistPlugin(BeetsPlugin): self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() if self.config['playlist_names'].as_str_seq(): - playlists = ET.fromstring(self.send('getPlaylists').text)[ - 0] + playlists = et.fromstring( + self.send('getPlaylists').text + )[0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ ' but no error message found' diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index fe36fbd13..01da5afff 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -25,7 +25,7 @@ from hashlib import md5 import os import shutil from itertools import chain -from pathlib import PurePosixPath +from pathlib2 import PurePosixPath import ctypes import ctypes.util diff --git a/setup.py b/setup.py index c50e65bf5..0be671f5d 100755 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ setup( 'requests_oauthlib' ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. - ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] + ['pathlib2'] if (sys.version_info < (3, 4, 0)) else [] ), # Plugin (optional) dependencies: @@ -144,7 +144,7 @@ setup( 'web': ['flask', 'flask-cors'], 'import': ['rarfile'], 'thumbnails': ['pyxdg', 'Pillow'] + - (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), + (['pathlib2'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], 'scrub': ['mutagen>=1.33'], diff --git a/test/test_export.py b/test/test_export.py index 757212a38..18b2b17be 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -23,7 +23,7 @@ from test.helper import TestHelper import re # used to test csv format import json from xml.etree.ElementTree import Element -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et class ExportPluginTest(unittest.TestCase, TestHelper): @@ -85,7 +85,7 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='xml', artist=item1.artist ) - library = ET.fromstring(out) + library = et.fromstring(out) self.assertIsInstance(library, Element) for track in library[0]: for details in track: diff --git a/tox.ini b/tox.ini index 19cb3069b..06f0bda8c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,54 +4,63 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py27-flake8, docs +envlist = + py27-{test} + py38-{test,cov,flake8} + docs -# The exhaustive list of environments is: -# envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs +# exhaustive list +# py{27,35,36,37,38}-{test,cov,flake8}, docs -[_test] -deps = - beautifulsoup4 - flask - mock - nose - nose-show-skipped - pylast - rarfile - responses>=0.3.0 - pyxdg - python-mpd2 - coverage - discogs-client - requests_oauthlib - -[_flake8] -deps = - flake8 - flake8-coding - flake8-future-import - flake8-blind-except - pep8-naming~=0.7.0 -files = beets beetsplug beet test setup.py docs +[gh-actions:env] +PLATFORM = + ubuntu-latest: linux + windows-latest: windows [testenv] passenv = NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. deps = {test,cov}: {[_test]deps} - py27: pathlib - py{27,34,35,36,37,38}-flake8: {[_flake8]deps} + flake8: {[_flake8]deps} + py27: pathlib2 commands = - py27-cov: python -m nose --with-coverage {posargs} py27-test: python -m nose {posargs} - py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py38-cov: python -bb -m nose --with-coverage {posargs} + py38-flake8: flake8 {posargs} {[_flake8]files} py3{4,5,6,7,8}-test: python -bb -m nose {posargs} - py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} - py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} - py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} - py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} - py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} - py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} + +[_test] +deps = + beautifulsoup4 + coverage + flask + discogs-client + mock + nose + nose-show-skipped + pylast + python-mpd2 + pyxdg + rarfile + requests_oauthlib + responses + sphinx + +[_flake8] +deps = + flake8 + flake8-blind-except + flake8-coding + flake8-future-import + pep8-naming +files = + beet + beets + beetsplug + docs + setup.py + test [testenv:docs] basepython = python2.7 From 9f8bd4cd3fb1a9d92a270be39d76e77f1a168647 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 25 May 2020 19:39:40 -0400 Subject: [PATCH 186/293] style: flake8 linting --- beetsplug/export.py | 14 +++++++------- beetsplug/lyrics.py | 5 +++-- beetsplug/plexupdate.py | 4 ++-- beetsplug/subsonicplaylist.py | 9 ++++----- beetsplug/thumbnails.py | 2 +- setup.py | 4 ++-- test/test_export.py | 4 ++-- tox.ini | 2 +- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 0b4876bbf..8d98d0ba2 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -21,7 +21,7 @@ import sys import codecs import json import csv -import xml.etree.ElementTree as et +from xml.etree import ElementTree from datetime import datetime, date from beets.plugins import BeetsPlugin @@ -188,18 +188,18 @@ class XMLFormat(ExportFormat): def export(self, data, **kwargs): # Creates the XML file structure. - library = et.Element(u'library') - tracks = et.SubElement(library, u'tracks') + library = ElementTree.Element(u'library') + tracks = ElementTree.SubElement(library, u'tracks') if data and isinstance(data[0], dict): for index, item in enumerate(data): - track = et.SubElement(tracks, u'track') + track = ElementTree.SubElement(tracks, u'track') for key, value in item.items(): - track_details = et.SubElement(track, key) + track_details = ElementTree.SubElement(track, key) track_details.text = value # Depending on the version of python the encoding needs to change try: - data = et.tostring(library, encoding='unicode', **kwargs) + data = ElementTree.tostring(library, encoding='unicode', **kwargs) except LookupError: - data = et.tostring(library, encoding='utf-8', **kwargs) + data = ElementTree.tostring(library, encoding='utf-8', **kwargs) self.out_stream.write(data) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index f73d72dfd..8807225a9 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -907,7 +907,7 @@ class LyricsPlugin(plugins.BeetsPlugin): return _scrape_strip_cruft(lyrics, True) def append_translation(self, text, to_lang): - import xml.etree.ElementTree as et + from xml.etree import ElementTree if not self.bing_auth_token: self.bing_auth_token = self.get_bing_access_token() @@ -925,7 +925,8 @@ class LyricsPlugin(plugins.BeetsPlugin): self.bing_auth_token = None return self.append_translation(text, to_lang) return text - lines_translated = et.fromstring(r.text.encode('utf-8')).text + lines_translated = ElementTree.fromstring( + r.text.encode('utf-8')).text # Use a translation mapping dict to build resulting lyrics translations = dict(zip(text_lines, lines_translated.split('|'))) result = '' diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index d3d7caf5c..ab3396191 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -12,7 +12,7 @@ Put something like the following in your config.yaml to configure: from __future__ import division, absolute_import, print_function import requests -import xml.etree.ElementTree as et +from xml.etree import ElementTree from six.moves.urllib.parse import urljoin, urlencode from beets import config from beets.plugins import BeetsPlugin @@ -28,7 +28,7 @@ def get_music_section(host, port, token, library_name): r = requests.get(url) # Parse xml tree and extract music section key. - tree = et.fromstring(r.content) + tree = ElementTree.fromstring(r.content) for child in tree.findall('Directory'): if child.get('title') == library_name: return child.get('key') diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index a4577278a..732b51c2f 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function import random import string -import xml.etree.ElementTree as et +from xml.etree import ElementTree from hashlib import md5 from urllib.parse import urlencode @@ -85,7 +85,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def get_playlist(self, playlist_id): xml = self.send('getPlaylist', {'id': playlist_id}).text - playlist = et.fromstring(xml)[0] + playlist = ElementTree.fromstring(xml)[0] if playlist.attrib.get('code', '200') != '200': alt_error = 'error getting playlist, but no error message found' self._log.warn(playlist.attrib.get('message', alt_error)) @@ -101,9 +101,8 @@ class SubsonicPlaylistPlugin(BeetsPlugin): self.config.set_args(opts) ids = self.config['playlist_ids'].as_str_seq() if self.config['playlist_names'].as_str_seq(): - playlists = et.fromstring( - self.send('getPlaylists').text - )[0] + playlists = ElementTree.fromstring( + self.send('getPlaylists').text)[0] if playlists.attrib.get('code', '200') != '200': alt_error = 'error getting playlists,' \ ' but no error message found' diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index 01da5afff..fe36fbd13 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -25,7 +25,7 @@ from hashlib import md5 import os import shutil from itertools import chain -from pathlib2 import PurePosixPath +from pathlib import PurePosixPath import ctypes import ctypes.util diff --git a/setup.py b/setup.py index 0be671f5d..c50e65bf5 100755 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ setup( 'requests_oauthlib' ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. - ['pathlib2'] if (sys.version_info < (3, 4, 0)) else [] + ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] ), # Plugin (optional) dependencies: @@ -144,7 +144,7 @@ setup( 'web': ['flask', 'flask-cors'], 'import': ['rarfile'], 'thumbnails': ['pyxdg', 'Pillow'] + - (['pathlib2'] if (sys.version_info < (3, 4, 0)) else []), + (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], 'scrub': ['mutagen>=1.33'], diff --git a/test/test_export.py b/test/test_export.py index 18b2b17be..779e74423 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -23,7 +23,7 @@ from test.helper import TestHelper import re # used to test csv format import json from xml.etree.ElementTree import Element -import xml.etree.ElementTree as et +from xml.etree import ElementTree class ExportPluginTest(unittest.TestCase, TestHelper): @@ -85,7 +85,7 @@ class ExportPluginTest(unittest.TestCase, TestHelper): format_type='xml', artist=item1.artist ) - library = et.fromstring(out) + library = ElementTree.fromstring(out) self.assertIsInstance(library, Element) for track in library[0]: for details in track: diff --git a/tox.ini b/tox.ini index 06f0bda8c..92086b820 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ passenv = deps = {test,cov}: {[_test]deps} flake8: {[_flake8]deps} - py27: pathlib2 + py27: pathlib commands = py27-test: python -m nose {posargs} py38-cov: python -bb -m nose --with-coverage {posargs} From 9a67cee95f442cd831163a6b262929f811ac93a7 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 25 May 2020 21:53:22 -0400 Subject: [PATCH 187/293] ci: added windows --- .github/workflows/ci.yaml | 42 +++++++++++++++++++++++++++------------ tox.ini | 1 + 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cbd6381e1..0f9ebbeba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,8 +5,9 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] env: + PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 @@ -15,26 +16,33 @@ jobs: with: python-version: 2.7 - uses: actions/cache@v1 + if: startsWith(runner.os, 'Linux') with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- + - uses: actions/cache@v1 + if: startsWith(runner.os, 'Windows') + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install base dependencies run: | python -m pip install --upgrade pip pip install tox sphinx - name: Test with tox - run: | - tox -e py27-test - tox -e docs + run: tox -e py27-test test-3x: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: + PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 @@ -43,24 +51,32 @@ jobs: with: python-version: 3.${{ matrix.python-version }} - uses: actions/cache@v1 + if: startsWith(runner.os, 'Linux') with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- + - uses: actions/cache@v1 + if: startsWith(runner.os, 'Windows') + with: + path: ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install base dependencies run: | python -m pip install --upgrade pip pip install tox sphinx - - name: Test with tox + - name: Test, coverage, and flake8 with tox + if: matrix.python-version == '8' run: | - if [[ ${{ matrix.python-version }} == '8' ]]; then - tox -e py3${{ matrix.python-version }}-test - tox -e py3${{ matrix.python-version }}-cov - tox -e py3${{ matrix.python-version }}-flake8 - else - tox -e py3${{ matrix.python-version }}-test - fi + tox -e py3${{ matrix.python-version }}-test + tox -e py3${{ matrix.python-version }}-cov + tox -e py3${{ matrix.python-version }}-flake8 + - name: Test with tox + if: matrix.python-version != '8' + run: tox -e py3${{ matrix.python-version }}-test docs: runs-on: ubuntu-latest steps: diff --git a/tox.ini b/tox.ini index 92086b820..b02e491fb 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ PLATFORM = [testenv] passenv = + LANG # avoids output errors NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. deps = {test,cov}: {[_test]deps} From b0f456b3c3410a9af3a78169801c5f55e39e839a Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 25 May 2020 23:18:59 -0400 Subject: [PATCH 188/293] ci: removed windows for now has to do with UnicodeDecodeError --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0f9ebbeba..f38ce898e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,7 +5,7 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] env: PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 @@ -39,7 +39,7 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: PY_COLOR: 1 From e8693acb6ae41c1824f0258b6b1e02e9d04402e1 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Tue, 26 May 2020 10:40:10 -0400 Subject: [PATCH 189/293] docs: elaborated exhaustive list Co-authored-by: Adrian Sampson --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b02e491fb..055f1e572 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ envlist = py38-{test,cov,flake8} docs -# exhaustive list +# The exhaustive list of environments is: # py{27,35,36,37,38}-{test,cov,flake8}, docs [gh-actions:env] From 0068afcb991a4c911142975f1a33dd3b5ab95c3b Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Tue, 26 May 2020 12:40:23 -0400 Subject: [PATCH 190/293] feat: add more options to tox --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 055f1e572..7c32fc460 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,11 @@ deps = flake8: {[_flake8]deps} py27: pathlib commands = + py27-cov: python -m nose --with-coverage {posargs} + py27-flake8: flake8 {posargs} {[_flake8]files} py27-test: python -m nose {posargs} - py38-cov: python -bb -m nose --with-coverage {posargs} - py38-flake8: flake8 {posargs} {[_flake8]files} + py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py3{4,5,6,7,8}-flake8: flake8 {posargs} {[_flake8]files} py3{4,5,6,7,8}-test: python -bb -m nose {posargs} [_test] From 9c93b4d35652c2b0aa90f1d5b1cc67bfeebb88ef Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Wed, 27 May 2020 16:36:27 -0400 Subject: [PATCH 191/293] docs: update punctuation Co-authored-by: Adrian Sampson --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f38ce898e..5c736e1ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ jobs: NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 - - name: Set up python 2.7 + - name: Set up Python 2.7 uses: actions/setup-python@v2 with: python-version: 2.7 @@ -46,7 +46,7 @@ jobs: NOSE_SHOW_SKIPPED: 1 steps: - uses: actions/checkout@v2 - - name: Set up python 3.${{ matrix.python-version }} + - name: Set up Python 3.${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: 3.${{ matrix.python-version }} @@ -81,7 +81,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python to build docs with sphinx + - name: Set up Python to build docs with Sphinx uses: actions/setup-python@v2 with: python-version: 2.7 @@ -90,4 +90,4 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx - name: Build and check docs using tox - run: tox -e docs \ No newline at end of file + run: tox -e docs From d3b759d7ca874695a04e13d4b49e1957515acd26 Mon Sep 17 00:00:00 2001 From: ybnd Date: Fri, 29 May 2020 13:29:31 +0200 Subject: [PATCH 192/293] Fix `list -af` example in documentation Didn't work since $tracktotal is not an Album field --- docs/reference/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index b66920d96..724afc80a 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -212,7 +212,7 @@ The ``-p`` option makes beets print out filenames of matched items, which might be useful for piping into other Unix commands (such as `xargs`_). Similarly, the ``-f`` option lets you specify a specific format with which to print every album or track. This uses the same template syntax as beets' :doc:`path formats -`. For example, the command ``beet ls -af '$album: $tracktotal' +`. For example, the command ``beet ls -af '$album: $albumtotal' beatles`` prints out the number of tracks on each Beatles album. In Unix shells, remember to enclose the template argument in single quotes to avoid environment variable expansion. From c69e96432a4bf9a82d683f7d8885aabb54fe39df Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Sun, 31 May 2020 00:10:32 -0400 Subject: [PATCH 193/293] importer: use tarfile.open to handle compressed archives Call tarfile.open instead of tarfile.TarFile from the importer so that we can import compressed tar archives. Note that tarfile.TarFile does not handle compressed archives: $ python3 Python 3.8.2 (default, Apr 27 2020, 15:53:34) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import tarfile >>> tf = tarfile.TarFile("Lagrimas.tar.bz2") Traceback (most recent call last): [...] tarfile.ReadError: invalid header >>> But tarfile.open does deal with them: $ python3 Python 3.8.2 (default, Apr 27 2020, 15:53:34) [GCC 9.3.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> import tarfile >>> tf = tarfile.open("Lagrimas.tar.bz2") >>> Tested: $ ls Lagrimas/*.mp3 | wc -l 11 $ tar cjf Lagrimas.tar.bz2 Lagrimas/ - Before: $ beet import Lagrimas.tar.bz2 extraction failed: invalid header No files imported from /tmp/Lagrimas.tar.bz2 - After: $ beet import Lagrimas.tar.bz2 [works] Fixes #3606. --- beets/importer.py | 4 ++-- docs/changelog.rst | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index e97b0a75c..68d5f3d5d 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -1034,8 +1034,8 @@ class ArchiveImportTask(SentinelImportTask): cls._handlers = [] from zipfile import is_zipfile, ZipFile cls._handlers.append((is_zipfile, ZipFile)) - from tarfile import is_tarfile, TarFile - cls._handlers.append((is_tarfile, TarFile)) + import tarfile + cls._handlers.append((tarfile.is_tarfile, tarfile.open)) try: from rarfile import is_rarfile, RarFile except ImportError: diff --git a/docs/changelog.rst b/docs/changelog.rst index 3f2890913..21cbd1217 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -124,6 +124,8 @@ New features: separators to support path queries. Thanks to :user:`nmeum`. :bug:`3567` +* ``beet import`` now handles tar archives with bzip2 or gzip compression. + :bug:`3606` Fixes: From 18645ffbe4b406fd9d7d21d465a5b21c78672bd7 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 00:26:33 -0400 Subject: [PATCH 194/293] chore(tox): revert changes --- tox.ini | 72 ++++++++++++++++++++++++--------------------------------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/tox.ini b/tox.ini index 7c32fc460..2e56f6107 100644 --- a/tox.ini +++ b/tox.ini @@ -4,68 +4,56 @@ # and then run "tox" from this directory. [tox] -envlist = - py27-{test} - py38-{test,cov,flake8} - docs +envlist = py27-test, py37-test, py27-flake8, docs # The exhaustive list of environments is: -# py{27,35,36,37,38}-{test,cov,flake8}, docs - -[gh-actions:env] -PLATFORM = - ubuntu-latest: linux - windows-latest: windows - -[testenv] -passenv = - LANG # avoids output errors - NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. -deps = - {test,cov}: {[_test]deps} - flake8: {[_flake8]deps} - py27: pathlib -commands = - py27-cov: python -m nose --with-coverage {posargs} - py27-flake8: flake8 {posargs} {[_flake8]files} - py27-test: python -m nose {posargs} - py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} - py3{4,5,6,7,8}-flake8: flake8 {posargs} {[_flake8]files} - py3{4,5,6,7,8}-test: python -bb -m nose {posargs} +# envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs [_test] deps = beautifulsoup4 - coverage flask - discogs-client mock nose nose-show-skipped pylast - python-mpd2 - pyxdg rarfile + responses>=0.3.0 + pyxdg + python-mpd2 + coverage + discogs-client requests_oauthlib - responses - sphinx [_flake8] deps = flake8 - flake8-blind-except flake8-coding flake8-future-import - pep8-naming -files = - beet - beets - beetsplug - docs - setup.py - test + flake8-blind-except + pep8-naming~=0.7.0 +files = beets beetsplug beet test setup.py docs + +[testenv] +passenv = + NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. +deps = + {test,cov}: {[_test]deps} + py27: pathlib + py{27,34,35,36,37,38}-flake8: {[_flake8]deps} +commands = + py27-cov: python -m nose --with-coverage {posargs} + py27-test: python -m nose {posargs} + py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} + py3{4,5,6,7,8}-test: python -bb -m nose {posargs} + py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} + py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} + py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} + py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} + py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} + py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} [testenv:docs] basepython = python2.7 deps = sphinx -commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} +commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} \ No newline at end of file From e66cfd58ad679f02a37480717b19de9c1077e7e4 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:09:35 -0400 Subject: [PATCH 195/293] feat: add windows into ci --- .github/workflows/ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5c736e1ba..50ab7462a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,10 +5,11 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] env: - PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 + PY_COLOR: 1 + PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 2.7 @@ -39,11 +40,12 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ windows-latest, ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: - PY_COLOR: 1 NOSE_SHOW_SKIPPED: 1 + PY_COLOR: 1 + PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 3.${{ matrix.python-version }} From 26caef544f5e2f5996dedaed3e6e20e99f9a5d1f Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:21:46 -0400 Subject: [PATCH 196/293] fix: remove `--min-version` --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 19cb3069b..9c1bc4106 100644 --- a/tox.ini +++ b/tox.ini @@ -46,12 +46,12 @@ commands = py27-test: python -m nose {posargs} py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} py3{4,5,6,7,8}-test: python -bb -m nose {posargs} - py27-flake8: flake8 --min-version 2.7 {posargs} {[_flake8]files} - py34-flake8: flake8 --min-version 3.4 {posargs} {[_flake8]files} - py35-flake8: flake8 --min-version 3.5 {posargs} {[_flake8]files} - py36-flake8: flake8 --min-version 3.6 {posargs} {[_flake8]files} - py37-flake8: flake8 --min-version 3.7 {posargs} {[_flake8]files} - py38-flake8: flake8 --min-version 3.8 {posargs} {[_flake8]files} + py27-flake8: flake8 {posargs} {[_flake8]files} + py34-flake8: flake8 {posargs} {[_flake8]files} + py35-flake8: flake8 {posargs} {[_flake8]files} + py36-flake8: flake8 {posargs} {[_flake8]files} + py37-flake8: flake8 {posargs} {[_flake8]files} + py38-flake8: flake8 {posargs} {[_flake8]files} [testenv:docs] basepython = python2.7 From f0534090c32ec386b8dfc21248b5426a70a271be Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:26:09 -0400 Subject: [PATCH 197/293] fix: revert add windows to ci --- .github/workflows/ci.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 50ab7462a..bec58f700 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,11 +5,10 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] env: NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 - PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 2.7 @@ -40,12 +39,11 @@ jobs: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ windows-latest, ubuntu-latest ] + platform: [ ubuntu-latest ] python-version: [ 5, 6, 7, 8 ] env: NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 - PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@v2 - name: Set up Python 3.${{ matrix.python-version }} From b9068e3fbd22409509da61a5a693bfbe174b6b16 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:40:05 -0400 Subject: [PATCH 198/293] fix: flake8 problem --- beets/autotag/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9bdf6b001..9841360ea 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -332,7 +332,7 @@ class Distance(object): self._penalties = {} @LazyClassProperty - def _weights(cls): # noqa + def _weights(self, cls): # noqa """A dictionary from keys to floating-point weights. """ weights_view = config['match']['distance_weights'] From 42e5844564740fd5fc5ba3393e03d96522352ff9 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 01:48:48 -0400 Subject: [PATCH 199/293] fix: revert flake8 change --- beets/autotag/hooks.py | 2 +- beetsplug/smartplaylist.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9841360ea..9bdf6b001 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -332,7 +332,7 @@ class Distance(object): self._penalties = {} @LazyClassProperty - def _weights(self, cls): # noqa + def _weights(cls): # noqa """A dictionary from keys to floating-point weights. """ weights_view = config['match']['distance_weights'] diff --git a/beetsplug/smartplaylist.py b/beetsplug/smartplaylist.py index f3157c96f..700b0c76a 100644 --- a/beetsplug/smartplaylist.py +++ b/beetsplug/smartplaylist.py @@ -105,17 +105,18 @@ class SmartPlaylistPlugin(BeetsPlugin): playlist_data = (playlist['name'],) try: - for key, model in (('query', Item), ('album_query', Album)): + for key, model_cls in (('query', Item), + ('album_query', Album)): qs = playlist.get(key) if qs is None: query_and_sort = None, None elif isinstance(qs, six.string_types): - query_and_sort = parse_query_string(qs, model) + query_and_sort = parse_query_string(qs, model_cls) elif len(qs) == 1: - query_and_sort = parse_query_string(qs[0], model) + query_and_sort = parse_query_string(qs[0], model_cls) else: # multiple queries and sorts - queries, sorts = zip(*(parse_query_string(q, model) + queries, sorts = zip(*(parse_query_string(q, model_cls) for q in qs)) query = OrQuery(queries) final_sorts = [] From b239a0b3d2dcbde0495750903dc2e69067d48ebf Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 4 Jun 2020 11:15:34 +0200 Subject: [PATCH 200/293] Fix item == None issues when writing lyrics ReST * Skip ReST writing & sphinx info messages if query doesn't yield anything * `writerest` into `appendrest` and `writerest`, don't call `writerest(item=None)` to flush state at the end. --- beetsplug/lyrics.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 71e1d4ef3..3e03bd224 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -770,7 +770,8 @@ class LyricsPlugin(plugins.BeetsPlugin): write = ui.should_write() if opts.writerest: self.writerest_indexes(opts.writerest) - for item in lib.items(ui.decargs(args)): + items = lib.items(ui.decargs(args)) + for item in items: if not opts.local_only and not self.config['local']: self.fetch_item_lyrics( lib, item, write, @@ -780,10 +781,10 @@ class LyricsPlugin(plugins.BeetsPlugin): if opts.printlyr: ui.print_(item.lyrics) if opts.writerest: - self.writerest(opts.writerest, item) - if opts.writerest: - # flush last artist - self.writerest(opts.writerest, None) + self.appendrest(opts.writerest, item) + if opts.writerest and len(items) > 0: + # flush last artist & write to ReST + self.writerest(opts.writerest) ui.print_(u'ReST files generated. to build, use one of:') ui.print_(u' sphinx-build -b html %s _build/html' % opts.writerest) @@ -795,26 +796,21 @@ class LyricsPlugin(plugins.BeetsPlugin): cmd.func = func return [cmd] - def writerest(self, directory, item): - """Write the item to an ReST file + def appendrest(self, directory, item): + """Append the item to an ReST file This will keep state (in the `rest` variable) in order to avoid writing continuously to the same files. """ - if item is None or slug(self.artist) != slug(item.albumartist): - if self.rest is not None: - path = os.path.join(directory, 'artists', - slug(self.artist) + u'.rst') - with open(path, 'wb') as output: - output.write(self.rest.encode('utf-8')) - self.rest = None - if item is None: - return + if slug(self.artist) != slug(item.albumartist): + # Write current file and start a new one ~ item.albumartist + self.writerest(directory) self.artist = item.albumartist.strip() self.rest = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ % (self.artist, u'=' * len(self.artist)) + if self.album != item.album: tmpalbum = self.album = item.album.strip() if self.album == '': @@ -826,6 +822,16 @@ class LyricsPlugin(plugins.BeetsPlugin): u'~' * len(title_str), block) + def writerest(self, directory): + """Write self.rest to a ReST file + """ + if self.rest is not None and self.artist is not None: + path = os.path.join(directory, 'artists', + slug(self.artist) + u'.rst') + with open(path, 'wb') as output: + output.write(self.rest.encode('utf-8')) + + def writerest_indexes(self, directory): """Write conf.py and index.rst files necessary for Sphinx From d1f3d664d75edb1f681264a2d578ddec21ea6d4f Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 4 Jun 2020 16:43:42 +0200 Subject: [PATCH 201/293] Implement comments & add to changelog --- beetsplug/lyrics.py | 3 +-- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 3e03bd224..05d3ea53f 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -782,7 +782,7 @@ class LyricsPlugin(plugins.BeetsPlugin): ui.print_(item.lyrics) if opts.writerest: self.appendrest(opts.writerest, item) - if opts.writerest and len(items) > 0: + if opts.writerest and items: # flush last artist & write to ReST self.writerest(opts.writerest) ui.print_(u'ReST files generated. to build, use one of:') @@ -831,7 +831,6 @@ class LyricsPlugin(plugins.BeetsPlugin): with open(path, 'wb') as output: output.write(self.rest.encode('utf-8')) - def writerest_indexes(self, directory): """Write conf.py and index.rst files necessary for Sphinx diff --git a/docs/changelog.rst b/docs/changelog.rst index 21cbd1217..691a42de5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -207,6 +207,9 @@ Fixes: * :doc:`/plugins/lyrics`: Adapt the Genius backend to changes in markup to reduce the scraping failure rate. :bug:`3535` :bug:`3594` +* :doc:`/plugins/lyrics`: Fix crash when writing ReST files for a query without + results or fetched lyrics + :bug:`2805` For plugin developers: From 14dc285b4f6348729e6cb3c7173fa3f786ef75b8 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 21:55:47 -0400 Subject: [PATCH 202/293] chore(flake8): update comment and version --- beets/autotag/hooks.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9bdf6b001..065d88170 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -332,7 +332,7 @@ class Distance(object): self._penalties = {} @LazyClassProperty - def _weights(cls): # noqa + def _weights(cls): # noqa: N805 """A dictionary from keys to floating-point weights. """ weights_view = config['match']['distance_weights'] diff --git a/tox.ini b/tox.ini index 9c1bc4106..047784f8b 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ deps = flake8-coding flake8-future-import flake8-blind-except - pep8-naming~=0.7.0 + pep8-naming files = beets beetsplug beet test setup.py docs [testenv] From c095429123c319fb355d22667a434dd1e64f42e5 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 22:03:16 -0400 Subject: [PATCH 203/293] feat: remove Travis CI in favor of GitHub Actions --- .travis.yml | 87 ----------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b889f698d..000000000 --- a/.travis.yml +++ /dev/null @@ -1,87 +0,0 @@ -dist: trusty -sudo: required -language: python - -env: - global: - # Undocumented feature of nose-show-skipped. - NOSE_SHOW_SKIPPED: 1 - -matrix: - include: - - python: 2.7.13 - env: {TOX_ENV: py27-cov, COVERAGE: 1} - - python: 2.7.13 - env: {TOX_ENV: py27-test} - # - python: 3.4 - # env: {TOX_ENV: py34-test} - # - python: 3.4_with_system_site_packages - # env: {TOX_ENV: py34-test} - - python: 3.5 - env: {TOX_ENV: py35-test} - - python: 3.6 - env: {TOX_ENV: py36-test} - - python: 3.7 - env: {TOX_ENV: py37-test} - dist: xenial - - python: 3.8 - env: {TOX_ENV: py38-test} - dist: xenial - # - python: pypy - # - env: {TOX_ENV: pypy-test} - - python: 3.6 - env: {TOX_ENV: py36-flake8} - - python: 2.7.13 - env: {TOX_ENV: docs} -# Non-Python dependencies. -addons: - apt: - sources: - - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty multiverse" - - sourceline: "deb https://archive.ubuntu.com/ubuntu/ trusty-updates multiverse" - packages: - - bash-completion - - gir1.2-gst-plugins-base-1.0 - - gir1.2-gstreamer-1.0 - - gstreamer1.0-plugins-good - - gstreamer1.0-plugins-bad - - imagemagick - - python-gi - - python-gst-1.0 - - python3-gi - - python3-gst-1.0 - - unrar - -# To install dependencies, tell tox to do everything but actually running the -# test. -install: - - travis_retry pip install tox sphinx - # upgrade requests to satisfy sphinx linkcheck (for building man pages) - - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then pip install -U requests; fi - - travis_retry tox -e $TOX_ENV --notest - -script: - # prevents "libdc1394 error: Failed to initialize libdc1394" errors - - sudo ln -s /dev/null /dev/raw1394 - - if [[ $TRAVIS_PYTHON_VERSION == *_site_packages ]]; then SITE_PACKAGES=--sitepackages; fi - # pip in trusty breaks on packages prefixed with "_". See https://github.com/pypa/pip/issues/3681 - - if [[ $TRAVIS_PYTHON_VERSION == 3.4_with_system_site_packages ]]; then sudo rm -rf /usr/lib/python3/dist-packages/_lxc-0.1.egg-info; fi - - tox -e $TOX_ENV $SITE_PACKAGES - -# Report coverage to codecov.io. -before_install: - - "[ ! -z $COVERAGE ] && travis_retry pip install codecov || true" -after_success: - - "[ ! -z $COVERAGE ] && codecov || true" - -cache: - pip: true - -notifications: - irc: - channels: - - "irc.freenode.org#beets" - use_notice: true - skip_join: true - on_success: change - on_failure: always From a96059634ca80a907d2a5f9d7d4302da0445ed9d Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 4 Jun 2020 22:09:18 -0400 Subject: [PATCH 204/293] feat: add `codecov` to GitHub Actions --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bec58f700..0f2c625a3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,6 +74,8 @@ jobs: tox -e py3${{ matrix.python-version }}-test tox -e py3${{ matrix.python-version }}-cov tox -e py3${{ matrix.python-version }}-flake8 + pip install codecov || true + codecov || true - name: Test with tox if: matrix.python-version != '8' run: tox -e py3${{ matrix.python-version }}-test From 1aa35edd3d56b3985284d70b2f0836f334ca52db Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 5 Jun 2020 10:03:37 -0400 Subject: [PATCH 205/293] Switch Travis badge to GitHub Actions --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0f653ac02..418ff0f56 100644 --- a/README.rst +++ b/README.rst @@ -4,8 +4,8 @@ .. image:: https://img.shields.io/codecov/c/github/beetbox/beets.svg :target: https://codecov.io/github/beetbox/beets -.. image:: https://travis-ci.org/beetbox/beets.svg?branch=master - :target: https://travis-ci.org/beetbox/beets +.. image:: https://github.com/beetbox/beets/workflows/ci/badge.svg?branch=master + :target: https://github.com/beetbox/beets/actions .. image:: https://repology.org/badge/tiny-repos/beets.svg :target: https://repology.org/project/beets/versions From 74ad96652883427e5d2d4d8363b21e7705bb6bd6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 5 Jun 2020 10:04:13 -0400 Subject: [PATCH 206/293] Run GitHub Actions CI on pull requests too --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0f2c625a3..5045a1b9c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,5 @@ name: ci -on: push +on: [push, pull_request] jobs: test-27: runs-on: ${{ matrix.platform }} From 2ae0dbbd7adbdf6f7ffe8a7f4ff343192df20721 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Jun 2020 16:29:33 -0400 Subject: [PATCH 207/293] docs: clarify `mbcollection` options --- docs/plugins/mbcollection.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/mbcollection.rst b/docs/plugins/mbcollection.rst index 113855bce..00acd4604 100644 --- a/docs/plugins/mbcollection.rst +++ b/docs/plugins/mbcollection.rst @@ -35,8 +35,8 @@ configuration file. There is one option available: - **auto**: Automatically amend your MusicBrainz collection whenever you import a new album. Default: ``no``. -- **collection**: Which MusicBrainz collection to update. +- **collection**: The MBID of which MusicBrainz collection to update. Default: ``None``. - **remove**: Remove albums from collections which are no longer present in the beets database. - Default: ``None``. + Default: ``no``. From 22d74a3b8a6978c213df922b1ef27bee60527d08 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Jun 2020 16:19:45 -0400 Subject: [PATCH 208/293] feat: support secure Plex connections --- beetsplug/plexupdate.py | 39 ++++++++++++++++++++++++++++--------- docs/plugins/plexupdate.rst | 4 ++++ test/test_plexupdate.py | 12 +++++++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index ab3396191..156d8835d 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -18,14 +18,16 @@ from beets import config from beets.plugins import BeetsPlugin -def get_music_section(host, port, token, library_name): +def get_music_section(host, port, token, library_name, use_secure, + ignore_cert_errors): """Getting the section key for the music library in Plex. """ api_endpoint = append_token('library/sections', token) - url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) + url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + port), api_endpoint) # Sends request. - r = requests.get(url) + r = requests.get(url, verify=not ignore_cert_errors) # Parse xml tree and extract music section key. tree = ElementTree.fromstring(r.content) @@ -34,17 +36,25 @@ def get_music_section(host, port, token, library_name): return child.get('key') -def update_plex(host, port, token, library_name): +def update_plex(host, port, token, library_name, use_secure, + ignore_cert_errors): + """Ignore certificate errors if configured to. + """ + if ignore_cert_errors: + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) """Sends request to the Plex api to start a library refresh. """ # Getting section key and build url. - section_key = get_music_section(host, port, token, library_name) + section_key = get_music_section(host, port, token, library_name, + use_secure, ignore_cert_errors) api_endpoint = 'library/sections/{0}/refresh'.format(section_key) api_endpoint = append_token(api_endpoint, token) - url = urljoin('http://{0}:{1}'.format(host, port), api_endpoint) + url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + port), api_endpoint) # Sends request and returns requests object. - r = requests.get(url) + r = requests.get(url, verify=not ignore_cert_errors) return r @@ -56,6 +66,13 @@ def append_token(url, token): return url +def get_protocol(use_secure): + if use_secure: + return 'https' + else: + return 'http' + + class PlexUpdate(BeetsPlugin): def __init__(self): super(PlexUpdate, self).__init__() @@ -65,7 +82,9 @@ class PlexUpdate(BeetsPlugin): u'host': u'localhost', u'port': 32400, u'token': u'', - u'library_name': u'Music'}) + u'library_name': u'Music', + u'use_secure': False, + u'ignore_cert_errors': False}) config['plex']['token'].redact = True self.register_listener('database_change', self.listen_for_db_change) @@ -85,7 +104,9 @@ class PlexUpdate(BeetsPlugin): config['plex']['host'].get(), config['plex']['port'].get(), config['plex']['token'].get(), - config['plex']['library_name'].get()) + config['plex']['library_name'].get(), + config['plex']['use_secure'].get(bool), + config['plex']['ignore_cert_errors'].get(bool)) self._log.info(u'... started.') except requests.exceptions.RequestException: diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index f9312280a..2c876c6fd 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -41,3 +41,7 @@ The available options under the ``plex:`` section are: Default: Empty. - **library_name**: The name of the Plex library to update. Default: ``Music`` +- **use_secure**: Use secure connections to the Plex server. + Default: ``False`` +- **ignore_cert_errors**: Ignore TLS certificate errors when using secure connections. + Default: ``False`` diff --git a/test/test_plexupdate.py b/test/test_plexupdate.py index f8b0bdcd9..862721837 100644 --- a/test/test_plexupdate.py +++ b/test/test_plexupdate.py @@ -93,7 +93,9 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['host'], self.config['plex']['port'], self.config['plex']['token'], - self.config['plex']['library_name'].get()), '2') + self.config['plex']['library_name'].get(), + self.config['plex']['use_secure'], + self.config['plex']['ignore_cert_errors']), '2') @responses.activate def test_get_named_music_section(self): @@ -104,7 +106,9 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['host'], self.config['plex']['port'], self.config['plex']['token'], - 'My Music Library'), '2') + 'My Music Library', + self.config['plex']['use_secure'], + self.config['plex']['ignore_cert_errors']), '2') @responses.activate def test_update_plex(self): @@ -117,7 +121,9 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['host'], self.config['plex']['port'], self.config['plex']['token'], - self.config['plex']['library_name'].get()).status_code, 200) + self.config['plex']['library_name'].get(), + self.config['plex']['use_secure'], + self.config['plex']['ignore_cert_errors']).status_code, 200) def suite(): From 6d41f31309503ff97cd2762b1e86d2abc3e4ee42 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Jun 2020 20:19:19 -0400 Subject: [PATCH 209/293] Rename use_secure to secure and add to changelog --- beetsplug/plexupdate.py | 18 +++++++++--------- docs/changelog.rst | 3 +++ docs/plugins/plexupdate.rst | 2 +- test/test_plexupdate.py | 6 +++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/beetsplug/plexupdate.py b/beetsplug/plexupdate.py index 156d8835d..860757cfb 100644 --- a/beetsplug/plexupdate.py +++ b/beetsplug/plexupdate.py @@ -18,12 +18,12 @@ from beets import config from beets.plugins import BeetsPlugin -def get_music_section(host, port, token, library_name, use_secure, +def get_music_section(host, port, token, library_name, secure, ignore_cert_errors): """Getting the section key for the music library in Plex. """ api_endpoint = append_token('library/sections', token) - url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + url = urljoin('{0}://{1}:{2}'.format(get_protocol(secure), host, port), api_endpoint) # Sends request. @@ -36,7 +36,7 @@ def get_music_section(host, port, token, library_name, use_secure, return child.get('key') -def update_plex(host, port, token, library_name, use_secure, +def update_plex(host, port, token, library_name, secure, ignore_cert_errors): """Ignore certificate errors if configured to. """ @@ -47,10 +47,10 @@ def update_plex(host, port, token, library_name, use_secure, """ # Getting section key and build url. section_key = get_music_section(host, port, token, library_name, - use_secure, ignore_cert_errors) + secure, ignore_cert_errors) api_endpoint = 'library/sections/{0}/refresh'.format(section_key) api_endpoint = append_token(api_endpoint, token) - url = urljoin('{0}://{1}:{2}'.format(get_protocol(use_secure), host, + url = urljoin('{0}://{1}:{2}'.format(get_protocol(secure), host, port), api_endpoint) # Sends request and returns requests object. @@ -66,8 +66,8 @@ def append_token(url, token): return url -def get_protocol(use_secure): - if use_secure: +def get_protocol(secure): + if secure: return 'https' else: return 'http' @@ -83,7 +83,7 @@ class PlexUpdate(BeetsPlugin): u'port': 32400, u'token': u'', u'library_name': u'Music', - u'use_secure': False, + u'secure': False, u'ignore_cert_errors': False}) config['plex']['token'].redact = True @@ -105,7 +105,7 @@ class PlexUpdate(BeetsPlugin): config['plex']['port'].get(), config['plex']['token'].get(), config['plex']['library_name'].get(), - config['plex']['use_secure'].get(bool), + config['plex']['secure'].get(bool), config['plex']['ignore_cert_errors'].get(bool)) self._log.info(u'... started.') diff --git a/docs/changelog.rst b/docs/changelog.rst index 691a42de5..5ecffb8e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -126,6 +126,9 @@ New features: :bug:`3567` * ``beet import`` now handles tar archives with bzip2 or gzip compression. :bug:`3606` +* :doc:`/plugins/plexupdate`: Add option to use secure connection to Plex + server, and to ignore certificate validation errors if necessary. + :bug:`2871` Fixes: diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index 2c876c6fd..92fc949d2 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -41,7 +41,7 @@ The available options under the ``plex:`` section are: Default: Empty. - **library_name**: The name of the Plex library to update. Default: ``Music`` -- **use_secure**: Use secure connections to the Plex server. +- **secure**: Use secure connections to the Plex server. Default: ``False`` - **ignore_cert_errors**: Ignore TLS certificate errors when using secure connections. Default: ``False`` diff --git a/test/test_plexupdate.py b/test/test_plexupdate.py index 862721837..ef89892a4 100644 --- a/test/test_plexupdate.py +++ b/test/test_plexupdate.py @@ -94,7 +94,7 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['port'], self.config['plex']['token'], self.config['plex']['library_name'].get(), - self.config['plex']['use_secure'], + self.config['plex']['secure'], self.config['plex']['ignore_cert_errors']), '2') @responses.activate @@ -107,7 +107,7 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['port'], self.config['plex']['token'], 'My Music Library', - self.config['plex']['use_secure'], + self.config['plex']['secure'], self.config['plex']['ignore_cert_errors']), '2') @responses.activate @@ -122,7 +122,7 @@ class PlexUpdateTest(unittest.TestCase, TestHelper): self.config['plex']['port'], self.config['plex']['token'], self.config['plex']['library_name'].get(), - self.config['plex']['use_secure'], + self.config['plex']['secure'], self.config['plex']['ignore_cert_errors']).status_code, 200) From dab0c1f9abda5b17cc7488f89a6fe08be7bc56a0 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 9 Jun 2020 19:34:31 +0200 Subject: [PATCH 210/293] compatibility with breaking changes to the ast module new in 3.10, also backported to 3.8 and 3.9: https://github.com/python/cpython/pull/20649 In fact, our generation of some Literals has been invalid since Python 3.4, fix that too. --- beets/util/functemplate.py | 29 ++++++++++++++++++++--------- docs/changelog.rst | 1 + 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index af22b7908..266534a9b 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -73,15 +73,26 @@ def ex_literal(val): """An int, float, long, bool, string, or None literal with the given value. """ - if val is None: - return ast.Name('None', ast.Load()) - elif isinstance(val, six.integer_types): - return ast.Num(val) - elif isinstance(val, bool): - return ast.Name(bytes(val), ast.Load()) - elif isinstance(val, six.string_types): - return ast.Str(val) - raise TypeError(u'no literal for {0}'.format(type(val))) + if sys.version_info[:2] < (3, 4): + if val is None: + return ast.Name('None', ast.Load()) + elif isinstance(val, six.integer_types): + return ast.Num(val) + elif isinstance(val, bool): + return ast.Name(bytes(val), ast.Load()) + elif isinstance(val, six.string_types): + return ast.Str(val) + raise TypeError(u'no literal for {0}'.format(type(val))) + elif sys.version_info[:2] < (3, 6): + if val in [None, True, False]: + return ast.NameConstant(val) + elif isinstance(val, six.integer_types): + return ast.Num(val) + elif isinstance(val, six.string_types): + return ast.Str(val) + raise TypeError(u'no literal for {0}'.format(type(val))) + else: + return ast.Constant(val) def ex_varassign(name, expr): diff --git a/docs/changelog.rst b/docs/changelog.rst index 5ecffb8e3..453d22305 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -213,6 +213,7 @@ Fixes: * :doc:`/plugins/lyrics`: Fix crash when writing ReST files for a query without results or fetched lyrics :bug:`2805` +* Adapt to breaking changes in Python's ``ast`` module in 3.8 For plugin developers: From ae175e156b92f77b593532f371f5a1c8317153d1 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 16:10:33 -0700 Subject: [PATCH 211/293] fix genius lyrics artist matching when artist contains a hyphen --- beetsplug/lyrics.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index f53191d8a..49d2be672 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -419,16 +419,23 @@ class Genius(Backend): return None for hit in json["response"]["hits"]: - # Genius uses zero-width characters to denote lowercase - # artist names. - hit_artist = hit["result"]["primary_artist"]["name"]. \ - strip(u'\u200b').lower() + hit_artist = self._clean(hit["result"]["primary_artist"]["name"]) - if hit_artist == artist.lower(): + if hit_artist.lower() == artist.lower(): return self.lyrics_from_song_page(hit["result"]["url"]) self._log.debug(u'genius: no matching artist') + def _clean(self, artist): + """Cleans genius-isms to help artist matching""" + # Genius uses zero-width characters to denote lowercase artist names + artist = artist.strip(u'\u200b') + + # Genius uses minus-hypen compared to beets hyphen + artist = artist.replace(u'\u002D', u'\u2010') + + return artist + class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" From bb5740d63caaaf230ceaab64e0ec9b52a1d19e1d Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 16:41:20 -0700 Subject: [PATCH 212/293] better debug message for no genius artist match --- beetsplug/lyrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 49d2be672..090b99278 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -424,7 +424,8 @@ class Genius(Backend): if hit_artist.lower() == artist.lower(): return self.lyrics_from_song_page(hit["result"]["url"]) - self._log.debug(u'genius: no matching artist') + self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', + artist) def _clean(self, artist): """Cleans genius-isms to help artist matching""" From 7220c91d2abc3c2bb565765ab512f20c11946c12 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 18:56:20 -0700 Subject: [PATCH 213/293] generalize genius artist matching to use slug() --- beetsplug/lyrics.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 090b99278..8e45d25b4 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -419,24 +419,14 @@ class Genius(Backend): return None for hit in json["response"]["hits"]: - hit_artist = self._clean(hit["result"]["primary_artist"]["name"]) + hit_artist = hit["result"]["primary_artist"]["name"] - if hit_artist.lower() == artist.lower(): + if slug(hit_artist) == slug(artist): return self.lyrics_from_song_page(hit["result"]["url"]) self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', artist) - def _clean(self, artist): - """Cleans genius-isms to help artist matching""" - # Genius uses zero-width characters to denote lowercase artist names - artist = artist.strip(u'\u200b') - - # Genius uses minus-hypen compared to beets hyphen - artist = artist.replace(u'\u002D', u'\u2010') - - return artist - class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" From 17d9477106afd319b3a861b09366e0a32953de74 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Fri, 26 Jun 2020 19:33:08 -0700 Subject: [PATCH 214/293] tests for applicable genius artist slugging --- test/test_lyrics.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index d31116284..22d5f2f0c 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function +import itertools import os import re import six @@ -485,18 +486,28 @@ class SlugTests(unittest.TestCase): # plain ascii passthrough text = u"test" self.assertEqual(lyrics.slug(text), 'test') + # german unicode and capitals text = u"Mørdag" self.assertEqual(lyrics.slug(text), 'mordag') + # more accents and quotes text = u"l'été c'est fait pour jouer" self.assertEqual(lyrics.slug(text), 'l-ete-c-est-fait-pour-jouer') + # accents, parens and spaces text = u"\xe7afe au lait (boisson)" self.assertEqual(lyrics.slug(text), 'cafe-au-lait-boisson') text = u"Multiple spaces -- and symbols! -- merged" self.assertEqual(lyrics.slug(text), 'multiple-spaces-and-symbols-merged') + text = u"\u200Bno-width-space" + self.assertEqual(lyrics.slug(text), 'no-width-space') + + # variations of dashes should get standardized + dashes = [u'\u200D', u'\u2010'] + for dash1, dash2 in itertools.combinations(dashes, 2): + self.assertEqual(lyrics.slug(dash1), lyrics.slug(dash2)) def suite(): From 2131f6f7d95a1c3bee17f878298ba217ea508c39 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 11:50:52 -0400 Subject: [PATCH 215/293] fetchart: Use Cover Art Archive thumbnails The Cover Art Archive API offers pre-resized thumbnails of cover art. If the `maxwidth` option of `fetchart` matches one of the supported Cover Art Archive thumbnail sizes fetch it directly instead of fetching the full size image then resizing it. --- beetsplug/fetchart.py | 20 +++++++++++++++++--- docs/changelog.rst | 3 +++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 70477a624..ad90b380c 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -317,12 +317,26 @@ class CoverArtArchive(RemoteArtSource): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ + + # Cover Art Archive API offers pre-resized thumbnails at several sizes. + # If the maxwidth config matches one of the already available sizes + # fetch it directly intead of fetching the full sized image and + # resizing it. + release_url = self.URL + group_url = self.GROUP_URL + valid_thumbnail_sizes = [250, 500, 1200] + if plugin.maxwidth in valid_thumbnail_sizes: + size_suffix = "-" + str(plugin.maxwidth) + release_url += size_suffix + group_url += size_suffix + if 'release' in self.match_by and album.mb_albumid: - yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT) + yield self._candidate( + url=release_url.format(mbid=album.mb_albumid), + match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( - url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), + url=group_url.format(mbid=album.mb_releasegroupid), match=Candidate.MATCH_FALLBACK) diff --git a/docs/changelog.rst b/docs/changelog.rst index 453d22305..5bc5c59cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -214,6 +214,9 @@ Fixes: results or fetched lyrics :bug:`2805` * Adapt to breaking changes in Python's ``ast`` module in 3.8 +* :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive + if the ``maxwidth`` option matches one of the sizes supported by the Cover + Art Archive API. For plugin developers: From 1e527ed4e52bad174f56cc2223bbb5ba4dc59ff1 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Jun 2020 12:49:32 -0400 Subject: [PATCH 216/293] Indentation fix --- beetsplug/fetchart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad90b380c..ebe343e82 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -333,7 +333,8 @@ class CoverArtArchive(RemoteArtSource): if 'release' in self.match_by and album.mb_albumid: yield self._candidate( url=release_url.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT) + match=Candidate.MATCH_EXACT, + ) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( url=group_url.format(mbid=album.mb_releasegroupid), From db181d35fb83339f1d24eeacba0a44f1edeb4740 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Jun 2020 12:50:13 -0400 Subject: [PATCH 217/293] Thanks & bug link for #3637 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bc5c59cc..112fdf23d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,6 +217,8 @@ Fixes: * :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive if the ``maxwidth`` option matches one of the sizes supported by the Cover Art Archive API. + Thanks to :user:`trolley`. + :bug:`3637` For plugin developers: From 71af5c04fc62a955712bc732b68c10c562d118c4 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 16:26:05 -0400 Subject: [PATCH 218/293] Revert "Thanks & bug link for #3637" This reverts commit db181d35fb83339f1d24eeacba0a44f1edeb4740. --- docs/changelog.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 112fdf23d..5bc5c59cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,8 +217,6 @@ Fixes: * :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive if the ``maxwidth`` option matches one of the sizes supported by the Cover Art Archive API. - Thanks to :user:`trolley`. - :bug:`3637` For plugin developers: From 3232a5b8b74134447f5dbaaefa7be5bd05147755 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 16:26:20 -0400 Subject: [PATCH 219/293] Revert "Indentation fix" This reverts commit 1e527ed4e52bad174f56cc2223bbb5ba4dc59ff1. --- beetsplug/fetchart.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ebe343e82..ad90b380c 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -333,8 +333,7 @@ class CoverArtArchive(RemoteArtSource): if 'release' in self.match_by and album.mb_albumid: yield self._candidate( url=release_url.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT, - ) + match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( url=group_url.format(mbid=album.mb_releasegroupid), From a4c647be461b56458a54fc1d7c74376a393cb60f Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sat, 27 Jun 2020 16:26:52 -0400 Subject: [PATCH 220/293] Revert "fetchart: Use Cover Art Archive thumbnails" This reverts commit 2131f6f7d95a1c3bee17f878298ba217ea508c39. --- beetsplug/fetchart.py | 20 +++----------------- docs/changelog.rst | 3 --- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index ad90b380c..70477a624 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -317,26 +317,12 @@ class CoverArtArchive(RemoteArtSource): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ - - # Cover Art Archive API offers pre-resized thumbnails at several sizes. - # If the maxwidth config matches one of the already available sizes - # fetch it directly intead of fetching the full sized image and - # resizing it. - release_url = self.URL - group_url = self.GROUP_URL - valid_thumbnail_sizes = [250, 500, 1200] - if plugin.maxwidth in valid_thumbnail_sizes: - size_suffix = "-" + str(plugin.maxwidth) - release_url += size_suffix - group_url += size_suffix - if 'release' in self.match_by and album.mb_albumid: - yield self._candidate( - url=release_url.format(mbid=album.mb_albumid), - match=Candidate.MATCH_EXACT) + yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), + match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: yield self._candidate( - url=group_url.format(mbid=album.mb_releasegroupid), + url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), match=Candidate.MATCH_FALLBACK) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bc5c59cc..453d22305 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -214,9 +214,6 @@ Fixes: results or fetched lyrics :bug:`2805` * Adapt to breaking changes in Python's ``ast`` module in 3.8 -* :doc:`/plugins/fetchart`: Fetch pre-resized thumbnails from Cover Art Archive - if the ``maxwidth`` option matches one of the sizes supported by the Cover - Art Archive API. For plugin developers: From 685d21a8fb907ff84559d8c98712ab196da579aa Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 27 Jun 2020 14:10:03 -0700 Subject: [PATCH 221/293] changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 453d22305..73cd89fb8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -129,6 +129,9 @@ New features: * :doc:`/plugins/plexupdate`: Add option to use secure connection to Plex server, and to ignore certificate validation errors if necessary. :bug:`2871` +* :doc:`/plugins/lyrics`: Improved searching Genius backend when artist + contained special characters. + :bug:`3634` Fixes: From 0f767ffa668bed98b5be9249f9b5e970b3245397 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Sun, 28 Jun 2020 16:04:29 -0400 Subject: [PATCH 222/293] fetchart: Use Cover Art Archive thumbnails The Cover Art Archive API offers pre-resized thumbnails of cover art. If the `maxwidth` option of `fetchart` matches one of the supported Cover Art Archive thumbnail sizes, and a thumbnail of that size exists in the Cover Art Archive, fetch it directly instead of fetching the full size image then resizing it. --- beetsplug/fetchart.py | 27 +++++++++++++++++++++++---- docs/changelog.rst | 3 +++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 70477a624..86c5b958f 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -305,6 +305,7 @@ class RemoteArtSource(ArtSource): class CoverArtArchive(RemoteArtSource): NAME = u"Cover Art Archive" VALID_MATCHING_CRITERIA = ['release', 'releasegroup'] + VALID_THUMBNAIL_SIZES = [250, 500, 1200] if util.SNI_SUPPORTED: URL = 'https://coverartarchive.org/release/{mbid}/front' @@ -317,13 +318,31 @@ class CoverArtArchive(RemoteArtSource): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ + release_url = self.URL.format(mbid=album.mb_albumid) + release_group_url = self.GROUP_URL.format(mbid=album.mb_releasegroupid) + + # Cover Art Archive API offers pre-resized thumbnails at several sizes. + # If the maxwidth config matches one of the already available sizes + # fetch it directly intead of fetching the full sized image and + # resizing it. + size_suffix = None + if plugin.maxwidth in self.VALID_THUMBNAIL_SIZES: + size_suffix = "-" + str(plugin.maxwidth) + if 'release' in self.match_by and album.mb_albumid: - yield self._candidate(url=self.URL.format(mbid=album.mb_albumid), + if size_suffix: + release_thumbnail_url = release_url + size_suffix + yield self._candidate(url=release_thumbnail_url, + match=Candidate.MATCH_EXACT) + yield self._candidate(url=release_url, match=Candidate.MATCH_EXACT) if 'releasegroup' in self.match_by and album.mb_releasegroupid: - yield self._candidate( - url=self.GROUP_URL.format(mbid=album.mb_releasegroupid), - match=Candidate.MATCH_FALLBACK) + if size_suffix: + release_group_thumbnail_url = release_group_url + size_suffix + yield self._candidate(url=release_group_thumbnail_url, + match=Candidate.MATCH_FALLBACK) + yield self._candidate(url=release_group_url, + match=Candidate.MATCH_FALLBACK) class Amazon(RemoteArtSource): diff --git a/docs/changelog.rst b/docs/changelog.rst index 73cd89fb8..b6904bb71 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,6 +217,9 @@ Fixes: results or fetched lyrics :bug:`2805` * Adapt to breaking changes in Python's ``ast`` module in 3.8 +* :doc:`/plugins/fetchart`: Attempt to fetch pre-resized thumbnails from Cover + Art Archive if the ``maxwidth`` option matches one of the sizes supported by + the Cover Art Archive API. For plugin developers: From c9519c1ef53b9ddeb725dd3e12a97383998060d8 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:32:37 -0500 Subject: [PATCH 223/293] Create pull_request_template.md --- .github/pull_request_template.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ef2679b50 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,9 @@ +## Description +(...) + +Fixes #(insert issue number here) + +## TODO: +- [ ] docs - If you've added a new command flag, for example, find the appropraite page under docs/ where it needs to be listed +- [ ] Add a **changelog entry** to docs/changelog.rst near the top of the document. +- [ ] Add tests (suggested, but not a hard requirement) From 3282f1565ea80733523ca5d9b323ddabfe1ce1df Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:33:42 -0500 Subject: [PATCH 224/293] Update pull_request_template.md --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ef2679b50..1d6217540 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,5 +5,5 @@ Fixes #(insert issue number here) ## TODO: - [ ] docs - If you've added a new command flag, for example, find the appropraite page under docs/ where it needs to be listed -- [ ] Add a **changelog entry** to docs/changelog.rst near the top of the document. +- [ ] Add a changelog entry to docs/changelog.rst near the top of the document. - [ ] Add tests (suggested, but not a hard requirement) From b458509ba18ba5da6191e2207b4f32d0486c8645 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:35:08 -0500 Subject: [PATCH 225/293] spelling --- .github/pull_request_template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1d6217540..49982230b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,9 @@ ## Description (...) -Fixes #(insert issue number here) +Fixes #(insert issue number here if applicable) ## TODO: -- [ ] docs - If you've added a new command flag, for example, find the appropraite page under docs/ where it needs to be listed +- [ ] docs - If you've added a new command flag, for example, find the appropriate page under docs/ where it needs to be listed - [ ] Add a changelog entry to docs/changelog.rst near the top of the document. - [ ] Add tests (suggested, but not a hard requirement) From def44e6f88fcec0d4bb6586feef3acd96050e9d2 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 28 Jun 2020 17:36:40 -0500 Subject: [PATCH 226/293] Update pull_request_template.md --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 49982230b..9abf2cdbd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -3,7 +3,7 @@ Fixes #(insert issue number here if applicable) -## TODO: +## TODO - [ ] docs - If you've added a new command flag, for example, find the appropriate page under docs/ where it needs to be listed - [ ] Add a changelog entry to docs/changelog.rst near the top of the document. - [ ] Add tests (suggested, but not a hard requirement) From aed81da2fada53a3921f0152a4c870ed5def0c6b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 28 Jun 2020 21:19:35 -0400 Subject: [PATCH 227/293] Thanks & bug link for #3637 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b6904bb71..1b80f5091 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -220,6 +220,8 @@ Fixes: * :doc:`/plugins/fetchart`: Attempt to fetch pre-resized thumbnails from Cover Art Archive if the ``maxwidth`` option matches one of the sizes supported by the Cover Art Archive API. + Thanks to :user:`trolley`. + :bug:`3637` For plugin developers: From 6411a58c80cfc755af744aae23637720bad632ef Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 29 Jun 2020 09:08:25 -0400 Subject: [PATCH 228/293] Slight formatting/wording improvements --- .github/pull_request_template.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9abf2cdbd..42cd8169b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,9 +1,11 @@ ## Description + +Fixes #X. + (...) -Fixes #(insert issue number here if applicable) +## To Do -## TODO -- [ ] docs - If you've added a new command flag, for example, find the appropriate page under docs/ where it needs to be listed -- [ ] Add a changelog entry to docs/changelog.rst near the top of the document. -- [ ] Add tests (suggested, but not a hard requirement) +- [ ] Documentation. (If you've add a new command-line flag, for example, find the appropriate page under `docs/` to describe it.) +- [ ] Changelog. (Add an entry to `docs/changelog.rst` near the top of the document.) +- [ ] Tests. (Encouraged but not strictly required.) From 27199abac8f2d82afc93993c1901a4edc6bb34b0 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Fri, 3 Jul 2020 09:27:55 -0700 Subject: [PATCH 229/293] Get the date of the parentwork, which may be different from the date of the child work. Also add a changelog entry and docs. --- beetsplug/parentwork.py | 4 +++- docs/changelog.rst | 4 ++++ docs/plugins/parentwork.rst | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/beetsplug/parentwork.py b/beetsplug/parentwork.py index dea7ba040..b3e464e60 100644 --- a/beetsplug/parentwork.py +++ b/beetsplug/parentwork.py @@ -131,6 +131,8 @@ class ParentWorkPlugin(BeetsPlugin): if artist['type'] == 'composer': parent_composer.append(artist['artist']['name']) parent_composer_sort.append(artist['artist']['sort-name']) + if 'end' in artist.keys(): + parentwork_info["parentwork_date"] = artist['end'] parentwork_info['parent_composer'] = u', '.join(parent_composer) parentwork_info['parent_composer_sort'] = u', '.join( @@ -207,4 +209,4 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid) item, fields=['parentwork', 'parentwork_disambig', 'mb_parentworkid', 'parent_composer', 'parent_composer_sort', 'work_date', - 'parentwork_workid_current']) + 'parentwork_workid_current', 'parentwork_date']) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1b80f5091..6ea5ae123 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -132,6 +132,10 @@ New features: * :doc:`/plugins/lyrics`: Improved searching Genius backend when artist contained special characters. :bug:`3634` +* :doc:`/plugins/parentwork`: Also get the composition date of the parent work, + instead of just the child work. + Thanks to :user:`aereaux`. + :bug:`3650` Fixes: diff --git a/docs/plugins/parentwork.rst b/docs/plugins/parentwork.rst index cc3f21c0c..fb15af9f1 100644 --- a/docs/plugins/parentwork.rst +++ b/docs/plugins/parentwork.rst @@ -26,7 +26,7 @@ At any later run of ``beet parentwork`` it will check if the tags not the case, it means the work has changed and all the tags need to be fetched again. -This plugin adds six tags: +This plugin adds seven tags: - **parentwork**: The title of the parent work. - **mb_parentworkid**: The MusicBrainz id of the parent work. @@ -38,6 +38,7 @@ This plugin adds six tags: - **parentwork_workid_current**: The MusicBrainz id of the work as it was when the parentwork was retrieved. This tag exists only for internal bookkeeping, to keep track of recordings whose works have changed. +- **parentwork_date**: The composition date of the parent work. To use the ``parentwork`` plugin, enable it in your configuration (see :ref:`using-plugins`). From c05ba5c814301d01f56099ce62d7e489d6f03478 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sun, 5 Jul 2020 21:41:22 -0700 Subject: [PATCH 230/293] add integration test github action --- .github/workflows/integration_test.yaml | 32 +++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 33 insertions(+) create mode 100644 .github/workflows/integration_test.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml new file mode 100644 index 000000000..11085223f --- /dev/null +++ b/.github/workflows/integration_test.yaml @@ -0,0 +1,32 @@ +name: integration tests +on: + schedule: + - cron: '0 0 * * SUN' # run every Sunday at midnight +jobs: + test integration: + runs-on: ubuntu-latest + strategy: + python-version: 3.8 + env: + NOSE_SHOW_SKIPPED: 1 + PY_COLOR: 1 + INTEGRATION_TEST: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + pip install tox + - name: Test with tox + run: | + tox -e py38-test \ No newline at end of file diff --git a/tox.ini b/tox.ini index 047784f8b..a62981ec8 100644 --- a/tox.ini +++ b/tox.ini @@ -37,6 +37,7 @@ files = beets beetsplug beet test setup.py docs [testenv] passenv = NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. + INTEGRATION_TEST # set to 1 for integration tests deps = {test,cov}: {[_test]deps} py27: pathlib From 06553cc37cf1113648070e3039c3cec878188e1d Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sun, 5 Jul 2020 21:41:31 -0700 Subject: [PATCH 231/293] adjust test_lyrics.py to work with integration tests --- test/test_lyrics.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 22d5f2f0c..ed221fbb1 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -298,8 +298,8 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): DEFAULT_SOURCES = [ dict(DEFAULT_SONG, backend=lyrics.LyricsWiki), - dict(artist=u'Santana', title=u'Black magic woman', - backend=lyrics.MusiXmatch), + # dict(artist=u'Santana', title=u'Black magic woman', + # backend=lyrics.MusiXmatch), dict(DEFAULT_SONG, backend=lyrics.Genius), ] @@ -313,9 +313,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(DEFAULT_SONG, url=u'http://www.chartlyrics.com', path=u'/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx'), - dict(DEFAULT_SONG, - url=u'http://www.elyricsworld.com', - path=u'/lady_madonna_lyrics_beatles.html'), + # dict(DEFAULT_SONG, + # url=u'http://www.elyricsworld.com', + # path=u'/lady_madonna_lyrics_beatles.html'), dict(url=u'http://www.lacoccinelle.net', artist=u'Jacques Brel', title=u"Amsterdam", path=u'/paroles-officielles/275679.html'), @@ -332,11 +332,11 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(url=u'http://www.lyricsontop.com', artist=u'Amy Winehouse', title=u"Jazz'n'blues", path=u'/amy-winehouse-songs/jazz-n-blues-lyrics.html'), - dict(DEFAULT_SONG, - url='http://www.metrolyrics.com/', - path='lady-madonna-lyrics-beatles.html'), - dict(url='http://www.musica.com/', path='letras.asp?letra=2738', - artist=u'Santana', title=u'Black magic woman'), + # dict(DEFAULT_SONG, + # url='http://www.metrolyrics.com/', + # path='lady-madonna-lyrics-beatles.html'), + # dict(url='http://www.musica.com/', path='letras.asp?letra=2738', + # artist=u'Santana', title=u'Black magic woman'), dict(url=u'http://www.paroles.net/', artist=u'Lilly Wood & the prick', title=u"Hey it's ok", path=u'lilly-wood-the-prick/paroles-hey-it-s-ok'), @@ -352,9 +352,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() - @unittest.skipUnless(os.environ.get( - 'BEETS_TEST_LYRICS_SOURCES', '0') == '1', - 'lyrics sources testing not enabled') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_backend_sources_ok(self): """Test default backends with songs known to exist in respective databases. """ @@ -366,9 +366,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): errors.append(s['backend'].__name__) self.assertFalse(errors) - @unittest.skipUnless(os.environ.get( - 'BEETS_TEST_LYRICS_SOURCES', '0') == '1', - 'lyrics sources testing not enabled') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_google_sources_ok(self): """Test if lyrics present on websites registered in beets google custom search engine are correctly scraped. From 45abc9ed7a4b75d2a723bf33ff6285729b0ec8f6 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 6 Jul 2020 08:41:27 -0500 Subject: [PATCH 232/293] Clean up code and tests for Genius Lyrics plugin backend (#3641) * clean-up code & add tests for genius lyrics backend * add genius fetch tests * organize imports: standard lib -> pip -> local * check in sample genius lyrics page * fix mock import * force utf-8 encoding for opened files * use io.open to force utf-8 encoding w/ python2.7 --- .gitignore | 1 - beetsplug/lyrics.py | 81 +- .../geniuscom/Wutangclancreamlyrics.txt | 2227 +++++++++++++++++ test/test_lyrics.py | 129 +- 4 files changed, 2356 insertions(+), 82 deletions(-) create mode 100644 test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt diff --git a/.gitignore b/.gitignore index 64f08abe5..241202b88 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ # Project Specific patterns man -test/rsrc/lyrics/* # The rest is from https://www.gitignore.io/api/python diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 8e45d25b4..81040af3a 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -359,16 +359,54 @@ class Genius(Backend): 'User-Agent': USER_AGENT, } - def lyrics_from_song_page(self, page_url): - # Gotta go regular html scraping... come on Genius. - self._log.debug(u'fetching lyrics from: {0}', page_url) - try: - page = requests.get(page_url) - except requests.RequestException as exc: - self._log.debug(u'Genius page request for {0} failed: {1}', - page_url, exc) + def fetch(self, artist, title): + """Fetch lyrics from genius.com + + Because genius doesn't allow accesssing lyrics via the api, + we first query the api for a url matching our artist & title, + then attempt to scrape that url for the lyrics. + """ + json = self._search(artist, title) + if not json: + self._log.debug(u'Genius API request returned invalid JSON') return None - html = BeautifulSoup(page.text, "html.parser") + + # find a matching artist in the json + for hit in json["response"]["hits"]: + hit_artist = hit["result"]["primary_artist"]["name"] + + if slug(hit_artist) == slug(artist): + return self._scrape_lyrics_from_html( + self.fetch_url(hit["result"]["url"])) + + self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', + artist) + + def _search(self, artist, title): + """Searches the genius api for a given artist and title + + https://docs.genius.com/#search-h2 + + :returns: json response + """ + search_url = self.base_url + "/search" + data = {'q': title + " " + artist.lower()} + try: + response = requests.get( + search_url, data=data, headers=self.headers) + except requests.RequestException as exc: + self._log.debug(u'Genius API request failed: {0}', exc) + return None + + try: + return response.json() + except ValueError: + return None + + def _scrape_lyrics_from_html(self, html): + """Scrape lyrics from a given genius.com html""" + + html = BeautifulSoup(html, "html.parser") # Remove script tags that they put in the middle of the lyrics. [h.extract() for h in html('script')] @@ -402,31 +440,6 @@ class Genius(Backend): return lyrics_div.get_text() - def fetch(self, artist, title): - search_url = self.base_url + "/search" - data = {'q': title + " " + artist.lower()} - try: - response = requests.get(search_url, data=data, - headers=self.headers) - except requests.RequestException as exc: - self._log.debug(u'Genius API request failed: {0}', exc) - return None - - try: - json = response.json() - except ValueError: - self._log.debug(u'Genius API request returned invalid JSON') - return None - - for hit in json["response"]["hits"]: - hit_artist = hit["result"]["primary_artist"]["name"] - - if slug(hit_artist) == slug(artist): - return self.lyrics_from_song_page(hit["result"]["url"]) - - self._log.debug(u'Genius failed to find a matching artist for \'{0}\'', - artist) - class LyricsWiki(SymbolsReplaced): """Fetch lyrics from LyricsWiki.""" diff --git a/test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt b/test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt new file mode 100644 index 000000000..08518f8ee --- /dev/null +++ b/test/rsrc/lyrics/geniuscom/Wutangclancreamlyrics.txt @@ -0,0 +1,2227 @@ + + + + + + + +Wu-Tang Clan – C.R.E.A.M. Lyrics | Genius Lyrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + {{:: 'cloud_flare_always_on_short_message' | i18n }} +
Check @genius for updates. We'll have things fixed soon. + +
+
+ + +
+ GENIUS +
+ + + +
+ + + + + + + + + + + + + +
+ + + + +
+ + + + + +
+ + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+
+
+ Https%3a%2f%2fimages +
+
+ +
+
+ +

C.R.E.A.M.

+

+ Wu-Tang Clan +

+ +

+ + +

+

+ +
+ Produced by + + + RZA + +
+ + +

+

+ +
+ Album + Enter the Wu-Tang (36 Chambers) +
+ + +

+ +
+
+
+
+
+
+
+ + +
+
+
+ +
+
+ Beets is awesome! +
+
+ + +
+
+ +
+
+ + + + +
+ +

+ About “C.R.E.A.M.” +

+ + +
+
+

Arguably one of the most iconic songs in hip-hop, the underlying idea of “C.R.E.A.M.” is found in its title—cash rules everything. The timeless piano riffs and background vocals come from a chopped up sample of The Charmels‘ 1967 record, “As Long As I’ve Got You,” that make up the entire track.

+ +

Although it was released as an official single in 1994, “C.R.E.A.M.” was first recorded in 1991, around the same time as RZA’s assault case, and featured himself and Ghostface Killah. The track went through several revisions and was later re-recorded by Raekwon and Inspectah Deck in 1993—an early title of the song was “Lifestyles of the Mega-Rich.”

+ +

In 2017, RZA explained to Power 106 how the final version of the track came together:

+ +

Once we got to the studio, I decided that this track had to be on the Wu-Tang album. I reminded Rae and Deck of their verses—their verses were long. […] Method Man, the master of hooks at the time, came in with this hook right here: ‘cash rules everything around me, cream, get the money.’ Once he added that element, I knew it was going to be a smash.

+ +

Since its release, the song and chorus have been referenced countless times by several artists. It has also been featured in movies such as Eminem’s 8 Mile and the N.W.A biopic, Straight Outta Compton.

+
+
+
+ +
+
+
    + +
  • +
    +

    What has RZA said about the song?

    +
    + +
    +

    + +
    +
    + + +
    • + + +
  • +
    +

    What has Raekwon said about the song?

    +
    + +
    +
    +

    ‘C.R.E.A.M.’ did a lot for my career personally. It gave me an opportunity to revisit the times where that cream meant that much to us. So, yeah, when I think of this record it just automatically puts me back into ‘87/’88 where we were standing in front of the building. It’s cold outside. We didn’t care. We’re out there, all black on trying to make dollars. Just trying to make some money and trying to eat. Survive.

    + +

    This song, I remember writing to the beat a long time ago before we actually came out. That beat is old. That was probably like a ‘89 beat. RZA had it that long because he had a bunch of breaks. He had all kind of things and he was making beats back then, but we was just picking and that beat happened to always sit around and I would be like, ‘I want that beat, so don’t give that beat to nobody.’ And he kept his word and let me have it.

    + +

    Meth came up with the hook but our dude named Raider Ruckus, this was like Meth’s homeboy back then, like they was real close, he came up with the phrase ‘cash rules everything around me.’ So when he showed Meth what it was and was like, ‘Cash rules everything around me,’ Meth was like, ‘Word, you right!’ And turned it into a movie, and I came in later that day and heard it and co-signed it.

    +
    + +

    via Complex

    +
    + + +
    • + + +
  • +
    +

    What has U-God said about the song?

    +
    + +
    +

    “C.R.E.A.M.” is a true song. Everything Inspectah Deck and Raekwon said is 100 percent true. Not one line in that entire song is a lie, or even a slight exaggeration. Deck did sell base, and he did go to jail at the age of fifteen. Rae was sticking up white boys on ball courts, rocking the same damn ’Lo sweater. And of “course, Meth on the hook was like butter on the popcorn. Meth knew the hard times, too, being out there smoking woolies and pumping crack, etc. That raspy shit he was kicking just echoed in everyone’s head long after the song was done playing.

    The realism on “C.R.E.A.M.” is what resonates with so many people all over the world. People everywhere know that sentiment of being slaves to the dollar. Cash is king, and we are its lowly subjects. That’s pretty much the case in every nation around the world, the desperation to put your life and your freedom on the line to make a couple dollars. Whether you’re working, stripping, hustling, or slinging, whether you’re a business owner or homeless, cash rules everything around us.

    + +

    Source: Raw:My Journey into Wu-Tang

    +
    + + +
    • + + +
  • +
    +

    What songs were sampled on the beat for “C.R.E.A.M.?”

    +
    + +
    +

    The vocals and background sample that can be heard on the song’s intro were taken taken from The Charmels’ 1967 song “As Long as I’ve Got You”:

    + +

    + +
    + +

    The classic keys sample that can be heard throughout the beat was also taken from the previously mentioned song:

    + +

    + +
    +
    + + +
    • + + +
  • +
    +

    What has Method Man said about the song?

    +
    + +
    +

    Meth told Complex,

    + +
    +

    ‘C.R.E.A.M.’ was the one that really put us on the map if you wanna be technical. I wasn’t there when they recorded ‘C.R.E.A.M.’ I came in after the fact. RZA was like, ‘Put a hook on this song’ and I put a hook on it. That’s how it always went. I liked doing hooks.

    + +

    The hook for that was done by my man Raider Ruckus. We used to work at the Statue of Liberty and when we were coming home we used to come up with all these made-up words that were acronyms.

    + +

    We had words like ‘BIBWAM’ which meant, ‘Bitches Is Busted Without A Man’ and all this other crazy shit. Raider Ruckus was so ill with the way he put the words together. We would call money ‘cream’ so he took each letter and made a word out of it and killed it the way he did it.

    + +

    Something like that had never been done before as far as a hook or even a way of speaking. This is just showing and proving that we paid attention in class when we was kids. You can’t do shit like that unless you got a brain in your fucking head! You got to have some level of intelligence to do something like that.

    + +

    The best acronym for a word that I heard was ‘P.R.O.J.E.C.T.S.’ by Killah Priest. He said ‘People Relying On Just Enough Cash To Survive.’ And he’s the one that came up with ‘Basic Instructions Before Leaving Earth,’ the acronym for B.I.B.L.E. This ain’t no fluke shit man.

    + +

    There’s a reason you got millions upon millions of fucking kids running around with Wu-Tang tattoos. You don’t just put something on your body permanently unless it’s official. At that time, when you’re coming out brand new and representing where you come from, everybody from that area wants you to win because they win. That’s what it was like for us.

    + +

    We were the only dudes from Staten Island doing it so everybody from Staten Island wanted us to win. Not just dudes from Staten Island, but dudes from Brooklyn too because they had peoples in the group too. Then it was just grimy niggas who loved to see real shit, saying, ‘We riding with them Wu-Tang niggas. Fuck all that shiny suit shit!’ That ain’t no take on Puff, a lot of niggas was wearing suits and shit man, but that ain’t us.

    +
    +
    + + +
    • + + + +
+
+ + +
+
+

"C.R.E.A.M." Track Info

+ + +
+ Written By + + + Method Man, Inspectah Deck & Raekwon + +
+ + + + + +
+ Programmer + + + RZA + +
+ + + + +
+ Mixing Engineer + + + RZA + +
+ + + + +
+ Label + + + RCA Records, Loud Records & Bertlesmann Music Group + +
+ + + + +
+ Mastering Engineer + + + Chris Gehringer + +
+ + + + +
+ Engineer + + + Ethan Ryman + +
+ + + + + +
+ Recorded At + Firehouse Studios (Brooklyn, New York City) +
+ + + +
+ Release Date + January 31, 1994 +
+ + + + + + +
+ Samples + + +
+ As Long As I've Got You by The Charmels +
+ + +
+ + + + + +
+ Sampled In + + +
+ Inspector Deck by A-Wax (Ft. Bandana Tha Rag) +
+ +
+ Cash Rules by ​iyla (Ft. Method Man) +
+ +
+ C.R.E.A.M by Fler +
+ +
+ Brain Cell by CunninLynguists +
+ +
+ Watch Your Mouth by Ghostface Killah (Ft. GZA, Inspectah Deck, Masta Killa, Method Man, Raekwon, RZA, U-God & Wu-Tang Clan) +
+ +
+ Misère by IAM +
+ +
+ Petit Frère by IAM +
+ +
+ Lift-Off by Luizão 911 +
+ +
+ It's Me Bitches (Remix) by Swizz Beatz (Ft. Jadakiss, Lil Wayne & R. Kelly) +
+ +
+ Rules by Wu-Tang Clan +
+ +
+ Cold World by Lupe Fiasco +
+ +
+ The Rep Growz Bigga by Gang Starr +
+ +
+ IAM - Petit Frère (English Translation) by Genius English Translations +
+ +
+ I Grew Up on Wu-Tang by The Game (Ft. Lorine Chia) +
+ +
+ Terrorist's Cell by 7L & Esoteric +
+ +
+ Flaschenpost by Eins Zwo +
+ +
+ Facts of Life by L.I.F.E. Long +
+ +
+ Respect Mine by PMD +
+ +
+ Heartbeat by Jon Murdock +
+ +
+ Anything (Old Skool Radio Version) by SWV (Ft. Wu-Tang Clan) +
+ +
+ Ghost Dog by Kyle Kallgren (Ft. Rap Critic) +
+ +
+ Deathbed Doctrine by Jedi Mind Tricks +
+ +
+ Mathematics by Yasiin Bey +
+ +
+ Cash Rule$ by King Lil G +
+ +
+ What It's All About by Girl Talk +
+ +
+ Il Mondo Che Non C'è by Chief & Soci +
+ +
+ Parasiet by Mertens +
+ +
+ Freestyle #13 by Funkmaster Flex (Ft. The LOX & The Notorious B.I.G.) +
+ +
+ T.O.N.Y. C.R.E.A.M. by DJ Unknown & DJ Mekalek (Ft. Percee P & Rok One) +
+ +
+ The Polorican by Thirstin Howl III +
+ +
+ 8 Mile: Cheddar Bob Freestyle by Eminem +
+ +
+ Ποτέ δε θα Φύγω (Wu-Tang Special) by Rappas +
+ +
+ Welcome to Killmington by Triigga +
+ +
+ Queens Get The Money by Oscar O'Neill +
+ +
+ My Crew by Saigon +
+ +
+ Ends by Everlast +
+ +
+ K.R.E.A.M. by U-N-I (Rap Group) +
+ +
+ C'est mort by Big Budha Cheez (Ft. Cenza) +
+ +
+ Look Up / Save The Day by Michael Christmas +
+ +
+ I Hustle Everyday by Deez Nuts +
+ +
+ Leave It All Behind by Hex One +
+ +
+ Papoose Freestyles on Flex | Freestyle #076 by Papoose +
+ +
+ One Twenty by Eyedea & Abilities (Ft. Eyedea) +
+ +
+ Fuego by Cypress Hill (Ft. La Bruja) +
+ +
+ History of Rap 6 by Jimmy Fallon (Ft. Justin Timberlake) +
+ +
+ Life as a Shorty by Shyheim +
+ +
+ Brass Knuckles by Money Making Jam Boys (Ft. DJ Omega) +
+ +
+ Põe na Balança by Rap Plus Size +
+ +
+ Everywhere We Go by French Montana (Ft. Wale) +
+ +
+ Still Strugglin' by Raekwon (Ft. The Notorious B.I.G.) +
+ +
+ 4 Tha OG'z by MC Eiht (Ft. Freddie Foxxx) +
+ +
+ Yessaï (Sur la carte) by Caballero & JeanJass +
+ +
+ Respect Mine by Pete Rock (Ft. O.C.) +
+ +
+ Stop The Breaks by DJ Ron G (Ft. Killa Sin, KRS-One, The Notorious B.I.G., O.C. & Raekwon) +
+ +
+ Half Man Half Amazin by Pete Rock (Ft. Method Man) +
+ +
+ The Wake Up Call by Smiley the Ghetto Child +
+ +
+ Tras este Escenario by Nach (Ft. Juaninacka) +
+ +
+ Reality by Da Youngsta's +
+ +
+ Pascal by Hocus Pocus +
+ +
+ Good Mourning by Reflection Eternal +
+ +
+ Cream by Lil Wayne (Ft. Euro) +
+ +
+ Ride On by Snowgoons (Ft. Defari, Maylay Sparks & Sondro Castro) +
+ +
+ Dollar Bill by Count Bass D +
+ +
+ Elevate by Jigmastas +
+ +
+ Carta Viola by Duke Montana (Ft. Club Dogo) +
+ +
+ Birds by Killah Priest (Ft. Jay Notes & Redd Mudd) +
+ +
+ Foul Child / City Soldier (performed by Tee Productions by Warlocks +
+ +
+ Bomber by Oxon +
+ +
+ Comin from the Lower Level by CHOPS (Ft. Phil Da Agony, Ras Kass & Talib Kweli) +
+ +
+ Get the money by PANTyRAiD +
+ +
+ Grown Ass Man by Outerspace +
+ +
+ Escuchame by Braintax +
+ +
+ Front Steps Pt II (Tough Love) by Akrobatik +
+ +
+ Amor veneno by Dealema +
+ +
+ Release Me by Datsik +
+ +
+ Cash Rules by Snow Tha Product +
+ +
+ Homage to Premier by Focus… (Ft. Phonte, Royce Da 5'9'’ & Stat Quo) +
+ +
+ Survival Skills by KRS-ONE & Buckshot +
+ +
+ The Truth by Smif N Wessun +
+ +
+ Days Of My Youth by Rashad & Confidence +
+ +
+ Je suis en vie by Akhenaton (Ft. Tyler Woods) +
+ +
+ Cream Freestyle [Wu Tang Killa Deez] by Mista Deezy +
+ +
+ Tricher by ATK +
+ +
+ Lettre aux hirondelles by Akhenaton +
+ +
+ Intro by The X-Ecutioners (Ft. DJ Apollo, DJ Shortkut & DJ Vinroc) +
+ +
+ 40 Years of Hip-Hop by The Hood Internet +
+ +
+ Live From Shaolin by Jae Millz +
+ +
+ Spare Change by Ea$y Money +
+ +
+ Blutgrueneaugen by Die Beleidiger +
+ +
+ Intro by Inoki +
+ +
+ Supreme by Young Fla$h (Ft. Kwame Badu & Mike Holmes (Rapper)) +
+ +
+ Firing Squad by Heath McNease +
+ +
+ Kream by Iggy Azalea (Ft. Tyga) +
+ +
+ Dinheiro by Mãolee (Ft. BK' & Filipe Ret) +
+ +
+ C.R.E.A.M. by Heath McNease (Ft. Krum) +
+ +
+ Cream Remix by Djflipmobeatz +
+ +
+ If You Don't Know, Now You Know by Rap Genius +
+ +
+ Lucid by Internet Strangers +
+ +
+ You by Nicole Bus +
+ +
+ Zωή Mικρή by RNS (GR) +
+ +
+ Dinheiro (Remix) by Mãolee (Ft. Akira Presidente) +
+ +
+ Torcida by Big Fish (Ft. Chadia Rodriguez, Emis Killa, Fabri Fibra & Jake La Furia) +
+ +
+ Livin' Proof by Group Home +
+ +
+ Pound Cake Freestyle by JAG +
+ +
+ Barrionalistas by Los Chikos del Maíz +
+ +
+ No hay Empate by Cidtronyck (Ft. Freeman) +
+ +
+ D.R.E.A.M. by Miley Cyrus (Ft. Ghostface Killah) +
+ +
+ Wu by ​nafla +
+ +
+ INTRO by Nolan The Ninja +
+ +
+ Quake by BROCKBEATS +
+ +
+ Gospel by Rodrigo Zin (Ft. Bruno Nix & Nanda Oliveira N/O) +
+ +
+ DENTI D'ORO by Ketama126 +
+ +
+ (Unreleased) Intro by Grown Ass Child +
+ +
+ C.R.E.A.M. by Martin Matys +
+ +
+ Drei Tage wach by Plusmacher +
+ +
+ C.R.E.A.M. Team by Alemán (Ft. Muelas De Gallo) +
+ +
+ Ends [Woodstock 99] by Everlast +
+ +
+ Full Immersion by DJ Fede (Ft. Ghali) +
+ +
+ OASE by RAMO +
+ +
+ C.R.E.A.M. (Remix) by Serious P +
+ +
+ Verlaine by Rockin' Squat +
+ +
+ ​chain on / hold me by BROCKHAMPTON (Ft. JPEGMAFIA) +
+ +
+ Coppola by Cartel MCs (Ft. Ajota) +
+ + +
+ + + + + +
+ Interpolates + + +
+ Money (Dollar Bill, Y'all) by Jimmy Spicer +
+ + +
+ + + + + +
+ Interpolated By + + +
+ Denaro by CHS (Ft. BK') +
+ +
+ YOUTH* by GOTBARS +
+ +
+ MONEY* by GOTBARS +
+ +
+ Finesse (C.R.E.A.M.) by BMB Entertainment (Ft. K-$krilla & TrapKamo) +
+ +
+ Notorious Thugs by The Notorious B.I.G. (Ft. Bone Thugs-N-Harmony) +
+ +
+ Pound Cake / Paris Morton Music 2 by Drake (Ft. JAY-Z) +
+ +
+ We Takin' Over by DJ Khaled (Ft. Akon, Birdman, Fat Joe, Lil Wayne, Rick Ross & T.I.) +
+ +
+ Paper Trail$ by Joey Bada$$ +
+ +
+ Ballin' by Fat Joe (Ft. Teyana Taylor & Wiz Khalifa) +
+ +
+ New World Water by Yasiin Bey +
+ +
+ Game's Pain by The Game (Ft. Keyshia Cole) +
+ +
+ Hip Hop by Yasiin Bey +
+ +
+ Money Man by A$AP Mob (Ft. A$AP Bari, A$AP NAST & A$AP Rocky) +
+ +
+ Pound Cake (Remix) by Chamillionaire +
+ +
+ Childish Gambino Sway In The Morning Freestyle by Sway in the Morning (Ft. Childish Gambino) +
+ +
+ Pound Cake Freestyle by Meek Mill (Ft. Spade) +
+ +
+ Pound Cake (Freestyle) by King Los +
+ +
+ Pound Cake (Freestyle) by Migos +
+ +
+ Pound Cake by OTF Nunu +
+ +
+ Pound Cake (Remix) by Chip +
+ +
+ Pound Cake by Papoose +
+ +
+ Runaway Love (Remix) by Justin Bieber (Ft. Kanye West & Raekwon) +
+ +
+ Money Make the World Go Round by G-Unit +
+ +
+ Wild Thots by A Boogie wit da Hoodie (Ft. Fabolous) +
+ +
+ Ain't The Same by MIST +
+ +
+ Rise by Lecrae +
+ +
+ N64 by Denzel Curry +
+ +
+ I Am The Narrator by Plan B +
+ +
+ Return of the Hustle by Fabolous (Ft. Swizz Beatz) +
+ +
+ R.R.E.A.M. by Kitty +
+ +
+ D.R.E.A.M. by Transplants (Ft. Danny Diablo) +
+ +
+ Why Why Why by RZA (Ft. Swnkah) +
+ +
+ Life as a Shorty by Fashawn +
+ +
+ Sweetest Girl (Dollar Bill) by Wyclef Jean (Ft. Akon, Lil Wayne & Niia) +
+ +
+ The Ass by Pharoahe Monch (Ft. Apani B Fly) +
+ +
+ Pro Cakes 2 by Dirty Sanchez (Ft. Dyemond Lewis & Nyck Caution) +
+ +
+ Ice Cream by Audrey Rose (Ft. Fetty Wap & Remy Ma) +
+ +
+ The Sidewalks by J-Live +
+ +
+ Krølla 50-lapp y'all by Karpe +
+ +
+ The Last Huzzah (Remix) by Mr. Muthafuckin eXquire (Ft. Danny Brown, Das Racist, Despot & El-P) +
+ +
+ Respect Mine by Fat Joe (Ft. Raekwon) +
+ +
+ M.O.N.E.Y. by Lords of the Underground +
+ +
+ Sausage by Lil' Mama +
+ +
+ Sho Nuff by Juelz Santana +
+ +
+ Show You by Lil Debbie +
+ +
+ This Can't Be Real by Freeway (Ft. Marsha Ambrosius) +
+ +
+ Perfect by Princess Superstar +
+ +
+ Double Cross by Ghostface Killah (Ft. AZ) +
+ +
+ Living My Dream by King Lil G (Ft. Carolyn Rodriguez, Isuppose & Self-Provoked) +
+ +
+ I Know Why The Caged Bird Sings (Part 2) by Binary Star +
+ +
+ Loyalty by Ghostface Killah (Ft. Kool G Rap & Nems) +
+ +
+ Scars & Memories by MF Grimm +
+ +
+ D.R.E.A.M. by Pharoahe Monch (Ft. Talib Kweli) +
+ +
+ Ay Yah by DeJ Loaf +
+ +
+ I Hustle Everyday by Deez Nuts +
+ +
+ High Society by Kottonmouth Kings +
+ +
+ Live from the Underground by Big K.R.I.T. +
+ +
+ Scorn by HS87 (Ft. Audio Push, Hit-Boy, Kent M$ney & Kid Cudi) +
+ +
+ Get Money by Redlight (Ft. Raekwon) +
+ +
+ Young Nigga by Young Dolph (Ft. Fiend) +
+ +
+ Longevity by Inspectah Deck (Ft. U-God) +
+ +
+ Life Is Hell by Apathy (Ft. Hayze) +
+ +
+ DILIGENCE by Inga Copeland +
+ +
+ La Petite Leçon by Oxmo Puccino +
+ +
+ When I Flow (It’s Gospel) by The Cross Movement +
+ +
+ Democracy by Killah Priest (Ft. Canibus) +
+ +
+ How Big Is Your World by Median +
+ +
+ Cash Rules by James Wade (US) (Ft. Kassidy Henderson) +
+ +
+ Welcome To The Darkside by Fat Joe (Ft. French Montana) +
+ +
+ Notes by Verbal Kent +
+ +
+ Tash Rules by Tash (Alkaholiks) (Ft. L.V. (South Central Cartel)) +
+ +
+ Do It All Over Again by Danny Swain (Ft. Joi & ?uestlove) +
+ +
+ Money, Power & Women by G-Spot-Geez +
+ +
+ Camouflage by C-Rayz Walz +
+ +
+ Catalina by Raekwon (Ft. Lyfe Jennings) +
+ +
+ MILK by BROCKHAMPTON +
+ +
+ Login by GIRIBOY (Ft. Swings & Young B (KR)) +
+ +
+ C.R.E.A.M. by Heath McNease (Ft. Krum) +
+ +
+ I'll Make U Famous by Da Youngsta's +
+ +
+ ​infinity (888) by XXXTENTACION (Ft. Joey Bada$$) +
+ +
+ FFM Freestyle: Big Baby Tape by Fast Food Music (Ft. Big Baby Tape) +
+ +
+ Gottaknow by Royce Da 5'9'’ +
+ +
+ MOOO! by Doja Cat +
+ +
+ Kream Get the Money by Maxo Kream +
+ +
+ Ciao Ciao by Colle Der Fomento (Ft. Kaos One & Piotta) +
+ +
+ Still by Loyle Carner +
+ +
+ Energy by Trizzy Trae +
+ +
+ Sorry by Future +
+ +
+ Killa Killa Hill by Cappadonna (Ft. Raekwon) +
+ +
+ Kream by KEZ +
+ +
+ Make My by The Roots (Ft. Big K.R.I.T. & Dice Raw) +
+ +
+ The Cave: Episode 12 by Kenny Beats (Ft. Denzel Curry) +
+ +
+ Second Round Knockout by Shaquille O'Neal +
+ +
+ Represent'96 by TVETH +
+ +
+ 7.O.D. by 2nd Generation Wu +
+ +
+ R.E.M.E.D.Y. by Remedy (NY) +
+ +
+ 5 Bucks by KIL +
+ +
+ Back In Time by E SENS +
+ +
+ Hey, Listen! by Kaisson +
+ +
+ EVIL TWIN by Denzel Curry (Ft. ZillaKami) +
+ +
+ C.R.E.A.M by Hester Shawty & BabySlugga +
+ +
+ Right Now by Malz Monday +
+ +
+ CREAM by Loccie Shmula +
+ +
+ Cream by VI Seconds +
+ +
+ Onun Adı V by ViO (Ft. DJ Artz) +
+ +
+ Coppola by Cartel MCs (Ft. Ajota) +
+ +
+ Dans la ville by Darryl Zeuja +
+ + +
+ + + + + + + + + +
+ Cover By + + +
+ C.R.E.A.M. by El Michels Affair +
+ +
+ C.R.E.A.M. by Ramin Djawadi +
+ + +
+ + + + + + + + + +
+ Remixed By + + +
+ C.R.E.A.M. by Remy Ma (Ft. Curren$y) +
+ +
+ Sleep Rules Everything Around Me by Wugazi (Ft. Fugazi, Ian MacKaye, Inspectah Deck, Raekwon & Wu-Tang Clan) +
+ +
+ D.R.E.A.M. by Reservoir Dogs +
+ + +
+ + + + + + + + + + + +
+
+ + +
+
+
+ + Https%3a%2f%2fimages + +
+ + Enter the Wu-Tang (36 Chambers) + + + Wu-Tang Clan +
+
+ +
+ +
+
+ + 1.   + + + + Bring Da Ruckus + + + + +
+
+ +
+
+ + 2.   + + + + Shame On A Nigga + + + + +
+
+ +
+
+ + 3.   + + + + Clan in Da Front + + + + +
+
+ +
+
+ + 4.   + + + + Wu-Tang: 7th Chamber + + + + +
+
+ + + +
+
+ + 6.   + + + + Da Mystery of Chessboxin’ + + + + +
+
+ + + +
+
+ + 8.   + + + + C.R.E.A.M. + + + + +
+
+ +
+
+ + 9.   + + + + Method Man + + + + +
+
+ +
+
+ + 10.   + + + + Protect Ya Neck + + + + +
+
+ +
+
+ + 11.   + + + + Tearz + + + + +
+
+ +
+
+ + 12.   + + + + Wu-Tang: 7th Chamber - Pt. II + + + + +
+
+ +
+
+ + 13.   + + + + Conclusion + + + + +
+
+ +
+ +
+
+ +
+
+
+ + + + + + + + + +
+
+ + + + +
+ © 2020 Genius Media Group Inc. +
+
+ +
+ Verified Artists + + All Artists: +
    + +
  • + A +
    • + +
  • + B +
    • + +
  • + C +
    • + +
  • + D +
    • + +
  • + E +
    • + +
  • + F +
    • + +
  • + G +
    • + +
  • + H +
    • + +
  • + I +
    • + +
  • + J +
    • + +
  • + K +
    • + +
  • + L +
    • + +
  • + M +
    • + +
  • + N +
    • + +
  • + O +
    • + +
  • + P +
    • + +
  • + Q +
    • + +
  • + R +
    • + +
  • + S +
    • + +
  • + T +
    • + +
  • + U +
    • + +
  • + V +
    • + +
  • + W +
    • + +
  • + X +
    • + +
  • + Y +
    • + +
  • + Z +
    • + +
  • + # +
    • + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 22d5f2f0c..1355ec779 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -18,23 +18,21 @@ from __future__ import absolute_import, division, print_function import itertools +from io import open import os import re import six import sys import unittest -from mock import patch -from test import _common +import confuse +from mock import MagicMock, patch from beets import logging from beets.library import Item from beets.util import bytestring_path -import confuse - from beetsplug import lyrics - -from mock import MagicMock +from test import _common log = logging.getLogger('beets.test_lyrics') @@ -232,38 +230,11 @@ class MockFetchUrl(object): def __call__(self, url, filename=None): self.fetched = url fn = url_to_filename(url) - with open(fn, 'r') as f: + with open(fn, 'r', encoding="utf8") as f: content = f.read() return content -class GeniusMockGet(object): - - def __init__(self, pathval='fetched_path'): - self.pathval = pathval - self.fetched = None - - def __call__(self, url, headers=False): - from requests.models import Response - # for the first requests.get() return a path - if headers: - response = Response() - response.status_code = 200 - response._content = b'{"meta":{"status":200},\ - "response":{"song":{"path":"/lyrics/sample"}}}' - return response - # for the second requests.get() return the genius page - else: - from mock import PropertyMock - self.fetched = url - fn = url_to_filename(url) - with open(fn, 'r') as f: - content = f.read() - response = Response() - type(response).text = PropertyMock(return_value=content) - return response - - def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" if not text: @@ -445,8 +416,9 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): google.is_page_candidate(url, url_title, s['title'], u'Sunn O)))') -class LyricsGeniusBaseTest(unittest.TestCase): +# test Genius backend +class GeniusBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: @@ -457,28 +429,91 @@ class LyricsGeniusBaseTest(unittest.TestCase): self.skipTest("Python's built-in HTML parser is not good enough") -class LyricsGeniusScrapeTest(LyricsGeniusBaseTest): - - """Checks that Genius backend works as intended. - """ - import requests +class GeniusScrapeLyricsFromHtmlTest(GeniusBaseTest): + """tests Genius._scrape_lyrics_from_html()""" def setUp(self): """Set up configuration""" - LyricsGeniusBaseTest.setUp(self) + GeniusBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() - @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): - """Ensure that `lyrics_from_song_page` doesn't crash when the html - for a Genius page doesn't contain
+ """Ensure we don't crash when the scraping the html for a genius page + doesn't contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None - song_url = 'https://genius.com/sample' - self.assertEqual(genius.lyrics_from_song_page(song_url), - None) + url = 'https://genius.com/sample' + mock = MockFetchUrl() + self.assertEqual(genius._scrape_lyrics_from_html(mock(url)), None) + def test_good_lyrics(self): + """Ensure we are able to scrape a page with lyrics""" + url = 'https://genius.com/Wu-tang-clan-cream-lyrics' + mock = MockFetchUrl() + self.assertIsNotNone(genius._scrape_lyrics_from_html(mock(url))) + + # TODO: find an example of a lyrics page with multiple divs and test it + + +class GeniusFetchTest(GeniusBaseTest): + """tests Genius.fetch()""" + + def setUp(self): + """Set up configuration""" + GeniusBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @patch.object(lyrics.Genius, '_scrape_lyrics_from_html') + @patch.object(lyrics.Backend, 'fetch_url', return_value=True) + def test_json(self, mock_fetch_url, mock_scrape): + """Ensure we're finding artist matches""" + with patch.object( + lyrics.Genius, '_search', return_value={ + "response": { + "hits": [ + { + "result": { + "primary_artist": { + "name": u"\u200Bblackbear", + }, + "url": "blackbear_url" + } + }, + { + "result": { + "primary_artist": { + "name": u"El\u002Dp" + }, + "url": "El-p_url" + } + } + ] + } + } + ) as mock_json: + # genius uses zero-width-spaces (\u200B) for lowercase + # artists so we make sure we can match those + self.assertIsNotNone(genius.fetch('blackbear', 'Idfc')) + mock_fetch_url.assert_called_once_with("blackbear_url") + mock_scrape.assert_called_once_with(True) + + # genius uses the hypen minus (\u002D) as their dash + self.assertIsNotNone(genius.fetch('El-p', 'Idfc')) + mock_fetch_url.assert_called_with('El-p_url') + mock_scrape.assert_called_with(True) + + # test no matching artist + self.assertIsNone(genius.fetch('doesntexist', 'none')) + + # test invalid json + mock_json.return_value = None + self.assertIsNone(genius.fetch('blackbear', 'Idfc')) + + # TODO: add integration test hitting real api + + +# test utilties class SlugTests(unittest.TestCase): From 723e04ebb3e2c6d5efbbc3b2d216eb68bb588e93 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 6 Jul 2020 17:02:23 -0700 Subject: [PATCH 233/293] deprecate `tests_require` in favor of extras --- setup.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index c50e65bf5..0e2cb332a 100755 --- a/setup.py +++ b/setup.py @@ -109,24 +109,33 @@ setup( ['colorama'] if (sys.platform == 'win32') else [] ), - tests_require=[ - 'beautifulsoup4', - 'flask', - 'mock', - 'pylast', - 'rarfile', - 'responses', - 'pyxdg', - 'python-mpd2', - 'discogs-client', - 'requests_oauthlib' - ] + ( - # Tests for the thumbnails plugin need pathlib on Python 2 too. - ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] - ), - - # Plugin (optional) dependencies: extras_require={ + 'test': [ + 'beautifulsoup4', + 'coverage', + 'discogs-client', + 'flask', + 'mock', + 'pylast', + 'pytest', + 'python-mpd2', + 'pyxdg', + 'rarfile', + 'responses>=0.3.0', + 'requests_oauthlib', + ] + ( + # Tests for the thumbnails plugin need pathlib on Python 2 too. + ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] + ), + 'lint': [ + 'flake8', + 'flake8-blind-except', + 'flake8-coding', + 'flake8-future-import', + 'pep8-naming', + ], + + # Plugin (optional) dependencies: 'absubmit': ['requests'], 'fetchart': ['requests', 'Pillow'], 'embedart': ['Pillow'], From dabde74683bbaf7a16520c19a908d2fbb600fae0 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 6 Jul 2020 17:03:11 -0700 Subject: [PATCH 234/293] nose -> pytest and clean-up tox --- .github/workflows/ci.yaml | 5 +-- .github/workflows/integration_test.yaml | 3 +- setup.cfg | 4 -- tox.ini | 54 +++++++------------------ 4 files changed, 18 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5045a1b9c..56596f83b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,9 +40,8 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - python-version: [ 5, 6, 7, 8 ] + python-version: [ 4, 5, 6, 7, 8 ] env: - NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 steps: - uses: actions/checkout@v2 @@ -73,7 +72,7 @@ jobs: run: | tox -e py3${{ matrix.python-version }}-test tox -e py3${{ matrix.python-version }}-cov - tox -e py3${{ matrix.python-version }}-flake8 + tox -e py3${{ matrix.python-version }}-lint pip install codecov || true codecov || true - name: Test with tox diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 11085223f..9a4469e14 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -10,7 +10,6 @@ jobs: env: NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 - INTEGRATION_TEST: 1 steps: - uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -29,4 +28,4 @@ jobs: pip install tox - name: Test with tox run: | - tox -e py38-test \ No newline at end of file + tox -e int \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 0660b2721..89649ed11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,3 @@ -[nosetests] -verbosity=1 -logging-clear-handlers=1 - [flake8] min-version=2.7 accept-encodings=utf-8 diff --git a/tox.ini b/tox.ini index a62981ec8..13531e556 100644 --- a/tox.ini +++ b/tox.ini @@ -4,57 +4,33 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py27-flake8, docs - -# The exhaustive list of environments is: -# envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs +envlist = py27-test, py38-{cov,lint}, docs [_test] -deps = - beautifulsoup4 - flask - mock - nose - nose-show-skipped - pylast - rarfile - responses>=0.3.0 - pyxdg - python-mpd2 - coverage - discogs-client - requests_oauthlib +deps = .[test] # defined in setup.py -[_flake8] -deps = - flake8 - flake8-coding - flake8-future-import - flake8-blind-except - pep8-naming +[_lint] +deps = .[lint] # defined in setup.py files = beets beetsplug beet test setup.py docs [testenv] -passenv = - NOSE_SHOW_SKIPPED # Undocumented feature of nose-show-skipped. - INTEGRATION_TEST # set to 1 for integration tests deps = {test,cov}: {[_test]deps} + lint: {[_lint]deps} py27: pathlib - py{27,34,35,36,37,38}-flake8: {[_flake8]deps} commands = - py27-cov: python -m nose --with-coverage {posargs} - py27-test: python -m nose {posargs} - py3{4,5,6,7,8}-cov: python -bb -m nose --with-coverage {posargs} - py3{4,5,6,7,8}-test: python -bb -m nose {posargs} - py27-flake8: flake8 {posargs} {[_flake8]files} - py34-flake8: flake8 {posargs} {[_flake8]files} - py35-flake8: flake8 {posargs} {[_flake8]files} - py36-flake8: flake8 {posargs} {[_flake8]files} - py37-flake8: flake8 {posargs} {[_flake8]files} - py38-flake8: flake8 {posargs} {[_flake8]files} + py27-test: python -m pytest {posargs} + py3{4,5,6,7,8}-test: python -bb -m pytest {posargs} + cov: coverage run -m pytest {posargs} + lint: python -m flake8 {posargs} {[_lint]files} [testenv:docs] basepython = python2.7 deps = sphinx commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} + +[testenv:int] +basepython = python3.8 +deps = {[_test]deps} +setenv = INTEGRATION_TEST = 1 +commands = python -m pytest {posargs} \ No newline at end of file From e4284cff55397c209a9f894771107f412547f784 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 6 Jul 2020 18:51:36 -0700 Subject: [PATCH 235/293] reverting change to ci for python3.4 seems github actions doesn't support python3.4 --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56596f83b..86a9fd45a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - python-version: [ 4, 5, 6, 7, 8 ] + python-version: [ 5, 6, 7, 8 ] env: PY_COLOR: 1 steps: From 917c7def4e2d2caf20263dcde0fdde60700db39b Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 09:37:22 -0700 Subject: [PATCH 236/293] test commit for windows ci windows seems to be having issues with tox installing depencies via `.[test]` this is a test to see if removing the inline comment changes anything --- .github/workflows/ci.yaml | 1 - tox.ini | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 86a9fd45a..9d2d8880f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,6 @@ jobs: matrix: platform: [ ubuntu-latest ] env: - NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 steps: - uses: actions/checkout@v2 diff --git a/tox.ini b/tox.ini index 13531e556..cb673b872 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,10 @@ envlist = py27-test, py38-{cov,lint}, docs [_test] -deps = .[test] # defined in setup.py +deps = .[test] [_lint] -deps = .[lint] # defined in setup.py +deps = .[lint] files = beets beetsplug beet test setup.py docs [testenv] From 25c6843c284ad7402b22467a18f743bbf58191e4 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 15:05:21 -0700 Subject: [PATCH 237/293] classify integration tests in `test_parentwork.py` --- test/test_parentwork.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_parentwork.py b/test/test_parentwork.py index aa25c7f51..df6a98d79 100644 --- a/test/test_parentwork.py +++ b/test/test_parentwork.py @@ -17,6 +17,7 @@ from __future__ import division, absolute_import, print_function +import os import unittest from test.helper import TestHelper @@ -34,6 +35,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.unload_plugins() self.teardown_beets() + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_normal_case(self): item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-b8ebc18e8c53') @@ -45,6 +49,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.assertEqual(item['mb_parentworkid'], u'32c8943f-1b27-3a23-8660-4567f4847c94') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_force(self): self.config['parentwork']['force'] = True item = Item(path='/file', @@ -58,6 +65,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): self.assertEqual(item['mb_parentworkid'], u'32c8943f-1b27-3a23-8660-4567f4847c94') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_no_force(self): self.config['parentwork']['force'] = True item = Item(path='/file', mb_workid=u'e27bda6e-531e-36d3-9cd7-\ @@ -72,6 +82,9 @@ class ParentWorkTest(unittest.TestCase, TestHelper): # test different cases, still with Matthew Passion Ouverture or Mozart # requiem + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') def test_direct_parent_work(self): mb_workid = u'2e4a3668-458d-3b2a-8be2-0b08e0d8243a' self.assertEqual(u'f04b42df-7251-4d86-a5ee-67cfa49580d1', From d5e5be5c8e640d7d2f113c959a226bbad32e42ea Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 15:05:40 -0700 Subject: [PATCH 238/293] simplify `tox.ini` --- tox.ini | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index cb673b872..e3476db1c 100644 --- a/tox.ini +++ b/tox.ini @@ -17,10 +17,8 @@ files = beets beetsplug beet test setup.py docs deps = {test,cov}: {[_test]deps} lint: {[_lint]deps} - py27: pathlib commands = - py27-test: python -m pytest {posargs} - py3{4,5,6,7,8}-test: python -bb -m pytest {posargs} + test: python -bb -m pytest {posargs} cov: coverage run -m pytest {posargs} lint: python -m flake8 {posargs} {[_lint]files} @@ -30,7 +28,6 @@ deps = sphinx commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} [testenv:int] -basepython = python3.8 deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 -commands = python -m pytest {posargs} \ No newline at end of file +commands = python -bb -m pytest {posargs} \ No newline at end of file From f2e7b3fdd7c103d8910020bc5b72279a80998b62 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 15:28:38 -0700 Subject: [PATCH 239/293] remove references to nose --- .coveragerc | 1 - .github/workflows/integration_test.yaml | 1 - appveyor.yml | 3 --- test/_common.py | 2 +- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7d111e74c..bcc99c18f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,6 @@ omit = */pyshared/* */python?.?/* - */site-packages/nose/* */test/* exclude_lines = assert False diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 9a4469e14..7d930bb72 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -8,7 +8,6 @@ jobs: strategy: python-version: 3.8 env: - NOSE_SHOW_SKIPPED: 1 PY_COLOR: 1 steps: - uses: actions/checkout@v2 diff --git a/appveyor.yml b/appveyor.yml index 00a3eb189..5a0f32135 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,9 +6,6 @@ skip_commits: message: /\[appveyor skip\]/ environment: - # Undocumented feature of nose-show-skipped. - NOSE_SHOW_SKIPPED: 1 - matrix: - PYTHON: C:\Python27 TOX_ENV: py27-test diff --git a/test/_common.py b/test/_common.py index 5412ab650..8e3b1dd18 100644 --- a/test/_common.py +++ b/test/_common.py @@ -44,7 +44,7 @@ beetsplug.__path__ = [os.path.abspath( RSRC = util.bytestring_path(os.path.join(os.path.dirname(__file__), 'rsrc')) PLUGINPATH = os.path.join(os.path.dirname(__file__), 'rsrc', 'beetsplug') -# Propagate to root logger so nosetest can capture it +# Propagate to root logger so the test runner can capture it log = logging.getLogger('beets') log.propagate = True log.setLevel(logging.DEBUG) From 7427bfbc582851e19b64dd4030c26cbe18b37953 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 19:54:01 -0700 Subject: [PATCH 240/293] clean github workflows --- .github/workflows/ci.yaml | 137 +++++++++++------------- .github/workflows/integration_test.yaml | 16 +-- 2 files changed, 66 insertions(+), 87 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9d2d8880f..c0b24d9c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,93 +1,78 @@ name: ci on: [push, pull_request] jobs: - test-27: + test: runs-on: ${{ matrix.platform }} strategy: matrix: - platform: [ ubuntu-latest ] + platform: [ubuntu-latest] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] env: - PY_COLOR: 1 + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 - - name: Set up Python 2.7 + + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 2.7 - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Linux') - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Windows') - with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install base dependencies - run: | - python -m pip install --upgrade pip - pip install tox sphinx - - name: Test with tox - run: tox -e py27-test - test-3x: - runs-on: ${{ matrix.platform }} - strategy: - matrix: - platform: [ ubuntu-latest ] - python-version: [ 5, 6, 7, 8 ] - env: - PY_COLOR: 1 - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: 3.${{ matrix.python-version }} - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Linux') - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - uses: actions/cache@v1 - if: startsWith(runner.os, 'Windows') - with: - path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install base dependencies - run: | - python -m pip install --upgrade pip - pip install tox sphinx - - name: Test, coverage, and flake8 with tox - if: matrix.python-version == '8' - run: | - tox -e py3${{ matrix.python-version }}-test - tox -e py3${{ matrix.python-version }}-cov - tox -e py3${{ matrix.python-version }}-lint - pip install codecov || true - codecov || true - - name: Test with tox - if: matrix.python-version != '8' - run: tox -e py3${{ matrix.python-version }}-test - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python to build docs with Sphinx - uses: actions/setup-python@v2 - with: - python-version: 2.7 + python-version: ${{ matrix.python-version }} + - name: Install base dependencies run: | python -m pip install --upgrade pip python -m pip install tox sphinx + + - name: Test with tox + if: matrix.python-version != '3.8' + run: | + tox -e py-test + + - name: Test with tox and get coverage + if: matrix.python-version == '3.8' + run: | + tox -vv -e py-cov + + - name: Upload code coverage + if: matrix.python-version == '3.8' + run: | + pip install codecov || true + codecov || true + + test-docs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 2.7 + uses: actions/setup-python@v2 + with: + python-version: 2.7 + + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox sphinx + - name: Build and check docs using tox run: tox -e docs + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install base dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox sphinx + + - name: Lint with flake8 + run: tox -e py-lint \ No newline at end of file diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 7d930bb72..6afda7e23 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -5,26 +5,20 @@ on: jobs: test integration: runs-on: ubuntu-latest - strategy: - python-version: 3.8 - env: - PY_COLOR: 1 + steps: - uses: actions/checkout@v2 + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: python-version: 3.8 - - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + - name: Install base dependencies run: | python -m pip install --upgrade pip - pip install tox + python -m pip install tox sphinx + - name: Test with tox run: | tox -e int \ No newline at end of file From 7280ab2be0c72e1dd47cf769df201beaa16d1cb7 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 19:54:12 -0700 Subject: [PATCH 241/293] remove nosetest.xml from .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 241202b88..3a0340a3f 100644 --- a/.gitignore +++ b/.gitignore @@ -53,7 +53,6 @@ htmlcov/ .coverage .coverage.* .cache -nosetests.xml coverage.xml *,cover .hypothesis/ From f6855cb28418e0c96600e9f5d803a6519f50a39e Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 7 Jul 2020 19:58:27 -0700 Subject: [PATCH 242/293] add py_colors to all github workflows --- .github/workflows/ci.yaml | 7 +++++++ .github/workflows/integration_test.yaml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0b24d9c5..f9cce8d23 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,6 +7,7 @@ jobs: matrix: platform: [ubuntu-latest] python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + env: PY_COLORS: 1 @@ -42,6 +43,9 @@ jobs: test-docs: runs-on: ubuntu-latest + env: + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 @@ -61,6 +65,9 @@ jobs: lint: runs-on: ubuntu-latest + env: + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 6afda7e23..fd67bb395 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -6,6 +6,9 @@ jobs: test integration: runs-on: ubuntu-latest + env: + PY_COLORS: 1 + steps: - uses: actions/checkout@v2 From d230b3b87024e230e28a879f20ec1b9c2270307e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 11 Jul 2020 10:04:00 -0400 Subject: [PATCH 243/293] Roll back logging changes --- beets/library.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 3ad875991..e22d4edc0 100644 --- a/beets/library.py +++ b/beets/library.py @@ -419,13 +419,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # This is helpful in path formats when the album artist is unset # on as-is imports. if key == 'artist' and not value: - new_value = self._get('albumartist') - log.debug('No artist, using album artist {0}'.format(new_value)) - return new_value + return self._get('albumartist') elif key == 'albumartist' and not value: - new_value = self._get('artist') - log.debug('No albumartist, using artist {0}'.format(new_value)) - return new_value + return self._get('artist') else: return value From cc9f4ab25123690c25ccd4f01b111fb1ea614ff0 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 11 Jul 2020 15:59:34 -0500 Subject: [PATCH 244/293] Add a stale-bot (#3659) will apply the `stale` label to any issues and pull requests with >60/120 days inactivity respectively --- .github/stale.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/stale.yml diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 000000000..850446d81 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,26 @@ +# Configuration for probot-stale - https://github.com/probot/stale + +daysUntilClose: 7 +staleLabel: stale + +issues: + daysUntilStale: 60 + onlyLabels: + - needinfo + markComment: > + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + +pulls: + daysUntilStale: 120 + markComment: > + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. From 16fddbaef6a1407898b160efbdafac55591c772b Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 11 Jul 2020 17:27:37 -0500 Subject: [PATCH 245/293] add line break to stale-bot comments --- .github/stale.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 850446d81..2f3dbe041 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -8,8 +8,9 @@ issues: onlyLabels: - needinfo markComment: > - Is this still relevant? If so, what is blocking it? - Is there anything you can do to help move it forward? + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you @@ -18,8 +19,9 @@ issues: pulls: daysUntilStale: 120 markComment: > - Is this still relevant? If so, what is blocking it? - Is there anything you can do to help move it forward? + Is this still relevant? If so, what is blocking it? + Is there anything you can do to help move it forward? + This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you From 1db46dfeb6607c164afb247d8da82443677795c1 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 18:15:41 -0700 Subject: [PATCH 246/293] reformat flake8 errors --- setup.cfg | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/setup.cfg b/setup.cfg index 89649ed11..ff9af3fd0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,19 +1,32 @@ [flake8] -min-version=2.7 -accept-encodings=utf-8 -# Errors we ignore: -# - E121,E123,E126,E24,E704,W503,W504 flake8 default ignores, excluding E226 (have to be listed here to not be overridden) -# - E221: multiple spaces before operator (used to align visually) -# - E731: do not assign a lambda expression, use a def -# - F405 object may be undefined, or defined from star imports -# - C901: function/method complexity -# - E305: spacing after a declaration (might be nice to change eventually) -# `flake8-future-import` errors we ignore: -# - FI50: `__future__` import "division" present -# - FI51: `__future__` import "absolute_import" present -# - FI12: `__future__` import "with_statement" missing -# - FI53: `__future__` import "print_function" present -# - FI14: `__future__` import "unicode_literals" missing -# - FI15: `__future__` import "generator_stop" missing -# - E741: ambiguous variable name -ignore=E121,E123,E126,E24,E704,W503,W504,E305,C901,E221,E731,F405,FI50,FI51,FI12,FI53,FI14,FI15,E741 +min-version = 2.7 +accept-encodings = utf-8 +docstring-convention = numpy +# flake8 errors we ignore. See https://www.flake8rules.com/ +ignore = + # pycodestyle errors + E121, # continuation line under-indented for hanging indent + E123, # closing bracket does not match indentation of opening bracket's line + E126, # continuation line over-indented for hangin indent + E221, # multiple spaces before operator (for vertical alignment) + E226, # missing whitespace around arithmetic operator + E241, # multiple spaces after non-arithmetic operators (for vertical alignment) + E242, # tabs after ',' + E305, # expected 2 blank lines after end of function or class + E704, # multiple statements on one line + E731, # do not assign a lamba expression, use a def + E741, # do not use variables name 'I', 'O', or 'l' + # pycodestyle warnings + W503, # line break occurred before a binary operator + W504, # line break occurred after a binary operator + # pyflakes errors + F405, # name be undefined, or defined from star imports: module + # mccabe error + C901, # function is too complex + # future-import errors + FI12, # `__future__` import "with_statement" missing + FI14, # `__future__` import "unicode_literals" missing + FI15, # `__future__` import "generator_stop" missing + FI50, # `__future__` import "division" present + FI51, # `__future__` import "absolute_import" present + FI53, # `__future__` import "print_function" present \ No newline at end of file From abad0f9c08edb97e8332ea091636ac2f9fd2affb Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 18:22:43 -0700 Subject: [PATCH 247/293] un-ignore flake8 E226, E242, E704 none of these required any code changes --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index ff9af3fd0..71da3bcc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,11 +9,8 @@ ignore = E123, # closing bracket does not match indentation of opening bracket's line E126, # continuation line over-indented for hangin indent E221, # multiple spaces before operator (for vertical alignment) - E226, # missing whitespace around arithmetic operator E241, # multiple spaces after non-arithmetic operators (for vertical alignment) - E242, # tabs after ',' E305, # expected 2 blank lines after end of function or class - E704, # multiple statements on one line E731, # do not assign a lamba expression, use a def E741, # do not use variables name 'I', 'O', or 'l' # pycodestyle warnings From 124c882b4d850b08ba3c52ff05bef219878165ad Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 18:33:08 -0700 Subject: [PATCH 248/293] un-ignore E221 --- beetsplug/discogs.py | 2 +- beetsplug/playlist.py | 2 +- setup.cfg | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 038fa809a..3b2595585 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -38,7 +38,7 @@ from string import ascii_lowercase USER_AGENT = u'beets/{0} +https://beets.io/'.format(beets.__version__) -API_KEY = 'rAzVUQYRaoFjeBjyWuWZ' +API_KEY = 'rAzVUQYRaoFjeBjyWuWZ' API_SECRET = 'plxtUTqoCzwxZpqdPysCwGuBSmZNdZVy' # Exceptions that discogs_client should really handle but does not. diff --git a/beetsplug/playlist.py b/beetsplug/playlist.py index 302ddb56d..3f8ba2b0e 100644 --- a/beetsplug/playlist.py +++ b/beetsplug/playlist.py @@ -71,7 +71,7 @@ class PlaylistQuery(beets.dbcore.Query): if not self.paths: # Playlist is empty return '0', () - clause = 'path IN ({0})'.format(', '.join('?' for path in self.paths)) + clause = 'path IN ({0})'.format(', '.join('?' for path in self.paths)) return clause, (beets.library.BLOB_TYPE(p) for p in self.paths) def match(self, item): diff --git a/setup.cfg b/setup.cfg index 71da3bcc6..1b61dfae8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,6 @@ ignore = E121, # continuation line under-indented for hanging indent E123, # closing bracket does not match indentation of opening bracket's line E126, # continuation line over-indented for hangin indent - E221, # multiple spaces before operator (for vertical alignment) E241, # multiple spaces after non-arithmetic operators (for vertical alignment) E305, # expected 2 blank lines after end of function or class E731, # do not assign a lamba expression, use a def From 52afd717aabc2955d10a0389f4b77e6f98245881 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 11 Jul 2020 19:10:04 -0700 Subject: [PATCH 249/293] remove docstring-convention and update comment --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1b61dfae8..3f5fb0c57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,7 @@ [flake8] min-version = 2.7 accept-encodings = utf-8 -docstring-convention = numpy -# flake8 errors we ignore. See https://www.flake8rules.com/ +# errors we ignore; see https://www.flake8rules.com/ for more info ignore = # pycodestyle errors E121, # continuation line under-indented for hanging indent From 44e4d558c97271244091e8541336454129b0622c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 12 Jul 2020 22:18:30 +0200 Subject: [PATCH 250/293] lyrics: fix bad_triggers in the google plugin regression from 117d16f --- beetsplug/lyrics.py | 2 +- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 81040af3a..16696d425 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -563,7 +563,7 @@ class Google(Backend): bad_triggers = ['lyrics', 'copyright', 'property', 'links'] if artist: - bad_triggers_occ += [artist] + bad_triggers += [artist] for item in bad_triggers: bad_triggers_occ += [item] * len(re.findall(r'\W%s\W' % item, diff --git a/docs/changelog.rst b/docs/changelog.rst index 6ea5ae123..0268aa2f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -136,6 +136,9 @@ New features: instead of just the child work. Thanks to :user:`aereaux`. :bug:`3650` +* :doc:`/plugins/lyrics`: Fix a bug in the heuristic for detecting valid + lyrics in the Google source of the lyrics plugin + :bug:`2969` Fixes: From d9272fcb88a617a36a1bc1f7c6405b6df8da8c4f Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 13 Jul 2020 14:20:20 -0500 Subject: [PATCH 251/293] fix integration test job name --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index fd67bb395..05623bf20 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -3,7 +3,7 @@ on: schedule: - cron: '0 0 * * SUN' # run every Sunday at midnight jobs: - test integration: + test_integration: runs-on: ubuntu-latest env: @@ -24,4 +24,4 @@ jobs: - name: Test with tox run: | - tox -e int \ No newline at end of file + tox -e int From 05516f9503bdb6c23015d844269ea809114f8db2 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 14 Jul 2020 06:59:48 -0400 Subject: [PATCH 252/293] Thanks & bug number for #2554 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 90e48de6b..167eba953 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -230,6 +230,8 @@ Fixes: Thanks to :user:`trolley`. :bug:`3637` * :doc:`/plugins/ipfs`: Fix Python 3 compatibility. + Thanks to :user:`musoke`. + :bug:`2554` For plugin developers: From dc7e4337680f7166167e5f08814fb2aee99ae155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Albornoz?= Date: Wed, 15 Jul 2020 22:00:50 -0400 Subject: [PATCH 253/293] Fix thumbnail plugin pathlib bug (#3360) PathlibURI.uri() would fail if given a path as bytes instead of as string. Co-authored-by: Jacob Pavlock --- beetsplug/thumbnails.py | 2 +- docs/changelog.rst | 3 +++ test/test_thumbnails.py | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index fe36fbd13..1b262eca5 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -224,7 +224,7 @@ class PathlibURI(URIGetter): name = "Python Pathlib" def uri(self, path): - return PurePosixPath(path).as_uri() + return PurePosixPath(util.py3_path(path)).as_uri() def copy_c_string(c_string): diff --git a/docs/changelog.rst b/docs/changelog.rst index 167eba953..0f41c38ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -139,6 +139,9 @@ New features: * :doc:`/plugins/lyrics`: Fix a bug in the heuristic for detecting valid lyrics in the Google source of the lyrics plugin :bug:`2969` +* :doc:`/plugins/thumbnails`: Fix a bug where pathlib expected a string instead + of bytes for a path. + :bug:`3360` Fixes: diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index dc03f06f7..dbbc032f7 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -284,6 +284,15 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): u'file:///music/%EC%8B%B8%EC%9D%B4') +class TestPathlibURI(): + """Test PathlibURI class""" + def test_uri(self): + test_uri = PathlibURI() + + # test it won't break if we pass it bytes for a path + test_uri.uri(b'/') + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 441cfebff19bdc08ffaab8590db94b32d1ae2c0a Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sun, 19 Jul 2020 13:25:55 -0500 Subject: [PATCH 254/293] add .venv/ to gitignore .venv/ is recommended via https://docs.python.org/3/library/venv.html --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3a0340a3f..370776197 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,7 @@ target/ # virtualenv venv/ +.venv/ ENV/ # Spyder project settings From 62ce5d15dbde671c9c64864d696cecb3546bfa19 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 08:12:34 -0700 Subject: [PATCH 255/293] add contributing.md & update readme (#3643) * add contributing.md & update readme * add for developers section of docs to contributing.md * remove relative wiki links * readme spelling fix * clarify discourse/docs idea for user 'recipes' to get featured * remove coveralls from testing section * fix api links & clarify docstring standard * Contributing -> Contribute * remove manual pip module installation instructions * add link to testing wiki page on "how to contribute" * add databse access examples * added voting clarification * Rename contributing.md to CONTRIBUTING.md * remove out of date info Includes references to gitter, the old 'good first issue' label and also some updates to testing. * No need for backticks around beets We usually just typeset it as an ordinary noun, not as code. :) * Whitespace * End-to-end CONTRIBUTING pass Co-authored-by: Adrian Sampson --- CONTRIBUTING.md | 165 ++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 15 +++-- 2 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..4e2c8964d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,165 @@ +# Thank you! + +First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. + +These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team. By following them, you indicate your respect for the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. + +# Types of Contributions + +We love to get contributions from our community—you! There are many ways to contribute, whether you're a programmer or not. + +## Non-Programming + +* Promote beets! Help get the word out by telling your friends, writing a blog + post, or discussing it on a forum you frequent. +* Improve the [documentation][docs]. It's incredibly easy to contribute here: + just find a page you want to modify and hit the "Edit on GitHub" button in + the upper-right. You can automatically send us a pull request for your + changes. +* GUI design. For the time being, beets is a command-line-only affair. But + that's mostly because we don't have any great ideas for what a good GUI + should look like. If you have those great ideas, please get in touch. +* Benchmarks. We'd like to have a consistent way of measuring speed + improvements in beets' tagger and other functionality as well as a way of + comparing beets' performance to other tools. You can help by compiling a + library of freely-licensed music files (preferably with incorrect metadata) + for testing and measurement. +* Think you have a nice config or cool use-case for beets? We'd love to hear about it! Submit a post to our [our forums][forum] under the "Show and Tell" category for a chance to get featured in [the docs][advanced]. +* Consider helping out in [our forums][forum] by responding to support requests or driving some new discussions. + +[docs]: http://beets.readthedocs.org/ +[forum]: https://discourse.beets.io/ +[advanced]: https://beets.readthedocs.io/en/stable/guides/advanced.html + +## Programming + +* As a programmer (even if you're just a beginner!), you have a ton of opportunities to get your feet wet with beets. +* For developing plugins, or hacking away at beets, there's some good information in the ["For Developers" section of the docs][dev-docs]. + +[dev-docs]: https://beets.readthedocs.io/en/stable/dev/ + +### Getting the Source + +The easiest way to get started with the latest beets source is to use [pip][] to install an "editable" package. This can be done with one command: + + $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets + +Or, equivalently: + + $ git clone https://github.com/beetbox/beets.git + $ cd beets + $ pip install -e . + +If you already have a released version of beets installed, you may need to +remove it first by typing `pip uninstall beets`. The pip command above will put +the beets source in a `src/beets` directory and install the `beet` CLI script to +a standard location on your system. You may want to use the `--src` option to specify +the parent directory where the source will be checked out and the `--user` option +such that the package will be installed to your home directory (compare with the output of +`pip install --help`). + +[pip]: https://pip.pypa.io/ + +### Code Contribution Ideas + +* We maintain a set of [issues marked as "bite-sized"](https://github.com/beetbox/beets/labels/bitesize). These are issues that would serve as a good introduction to the codebase. Claim one and start exploring! +* Like testing? Our [test coverage](https://codecov.io/github/beetbox/beets) is somewhat low. You can help out by finding low-coverage modules or checking out other [testing-related issues](https://github.com/beetbox/beets/labels/testing). +* There are several ways to improve the tests in general (see [Testing](https://github.com/beetbox/beets/wiki/Testing)) and some places to think about performance optimization (see [Optimization](https://github.com/beetbox/beets/wiki/Optimization)). +* Not all of our code is up to our coding conventions. In particular, the [API documentation](https://beets.readthedocs.io/en/stable/dev/api.html) are currently quite sparse. You can help by adding to the docstrings in the code and to the documentation pages themselves. beets follows [PEP-257](https://www.python.org/dev/peps/pep-0257/) for docstrings and in some places, we also sometimes use [ReST autodoc syntax for Sphinx](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to, for example, refer to a class name. + +## Your First Contribution + +If this is your first time contributing to an open source project, welcome! If you are confused at all about how to contribute or what to contribute, take a look at [this great tutorial](http://makeapullrequest.com/), or stop by our [forums](https://discourse.beets.io/) if you have any questions. + +We maintain a list of issues we reserved for those new to open source labeled ["first timers only"](https://github.com/beetbox/beets/issues?q=is%3Aopen+is%3Aissue+label%3A%22first+timers+only%22). Since the goal of these issues is to get users comfortable with contributing to an open source project, please do not hesitate to ask any questions. + +## How to Submit Your Work + +Do you have a great bug fix, new feature, or documentation expansion you'd like to contribute? Follow these steps to create a GitHub pull request and your code will ship in no time. + +1. Fork the beets repository and clone it (see above) to create a workspace. +2. Make your changes. +3. Add tests. If you've fixed a bug, write a test to ensure that you've actually fixed it. If there's a new feature or plugin, please contribute tests that show that your code does what it says. +4. Add documentation. If you've added a new command flag, for example, find the appropriate page under `docs/` where it needs to be listed. +5. Add a changelog entry to `docs/changelog.rst` near the top of the document. +6. Run the tests and style checker. The easiest way to run the tests is to use [tox](https://tox.readthedocs.org/en/latest/). For more information on running tests, see our [Testing wiki page](https://github.com/beetbox/beets/wiki/Testing). +7. Push to your fork and open a pull request! We'll be in touch shortly. +8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn't automatically notify us when commits are added. + +Remember, code contributions have four parts: the code, the tests, the documentation, and the changelog entry. Thank you for contributing! + + +# The Code + +The documentation has an [API section](https://beets.readthedocs.io/en/stable/dev/api.html) that serves as an introduction to beets' design. + +## Coding Conventions + +There are a few coding conventions we use in beets: + +* Whenever you access the library database, do so through the provided Library + methods or via a Transaction object. Never call `lib.conn.*` directly. + For example, do this: + ``` + with g.lib.transaction() as tx: + rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' + .format(field, model._table, sort_field)) + ``` + To fetch Item objects from the database, use lib.items(...) and supply a query as an argument. Resist the urge to write raw SQL for your query. If you must use lower-level queries into the database, do this: + ``` + with lib.transaction() as tx: + rows = tx.query('SELECT …') + ``` + Transaction objects help control concurrent access to the database and assist in debugging conflicting accesses. +* Always use the [future imports][] `print_function`, `division`, and + `absolute_import`, but *not* `unicode_literals`. These help keep your code + modern and will help in the eventual move to Python 3. +* `str.format()` should be used instead of the `%` operator +* Never `print` informational messages; use the [logging][] module instead. In + particular, we have our own logging shim, so you'll see `from beets import + logging` in most files. + * Always log Unicode strings (e.g., `log.debug(u"hello world")`). + * The loggers use [str.format][]-style logging instead of ``%``-style, so + you can type `log.debug(u"{0}", obj)` to do your formatting. +* Exception handlers must use `except A as B:` instead of `except A, B:`. + +[future imports]: http://docs.python.org/library/__future__.html +[logging]: http://docs.python.org/library/logging.html +[str.format]: http://docs.python.org/library/stdtypes.html#str.format +[modformat]: http://docs.python.org/library/stdtypes.html#string-formatting-operations + +We follow [PEP 8](http://www.python.org/dev/peps/pep-0008/) for style. You can use `tox -e lint` to check your code for any style errors. + +## Handling Paths + +A great deal of convention deals with the handling of **paths**. Paths are +stored internally—in the database, for instance—as byte strings (i.e., `bytes` instead of `str` in Python 3). This is because POSIX operating systems' path names are only +reliably usable as byte strings—operating systems typically recommend but do not require that filenames use a given encoding, so violations of any reported encoding are inevitable. +On Windows, the strings are always encoded with UTF-8; on +Unix, the encoding is controlled by the filesystem. Here are some guidelines to +follow: + +* If you have a Unicode path or you're not sure whether something is Unicode or + not, pass it through `bytestring_path` function in the `beets.util` module to + convert it to bytes. +* Pass every path name trough the `syspath` function (also in `beets.util`) + before sending it to any *operating system* file operation (`open`, for + example). This is necessary to use long filenames (which, maddeningly, must + be Unicode) on Windows. This allows us to consistently store bytes in the + database but use the native encoding rule on both POSIX and Windows. +* Similarly, the `displayable_path` utility function converts bytestring paths + to a Unicode string for displaying to the user. Every time you want to print + out a string to the terminal or log it with the `logging` module, feed it + through this function. + +## Editor Settings + +Personally, I work on beets with [vim](http://www.vim.org/). Here are some +`.vimrc` lines that might help with PEP 8-compliant Python coding: + + filetype indent on + autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent + +Consider installing [this alternative Python indentation +plugin](https://github.com/mitsuhiko/vim-python-combined). I also like +[neomake](https://github.com/neomake/neomake) with its flake8 checker. diff --git a/README.rst b/README.rst index 418ff0f56..7f509efdc 100644 --- a/README.rst +++ b/README.rst @@ -90,11 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Check out the `Hacking`_ page on the wiki for tips on how to help out. -You might also be interested in the `For Developers`_ section in the docs. +Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `Contributing.md`_. -.. _Hacking: https://github.com/beetbox/beets/wiki/Hacking -.. _For Developers: https://beets.readthedocs.io/en/stable/dev/ +.. _Contributing.md: https://github.com/beetbox/beets/blob/contributing.md/contributing.md Read More --------- @@ -105,6 +103,15 @@ news and updates. .. _its Web site: https://beets.io/ .. _@b33ts: https://twitter.com/b33ts/ +Contact +------- +* Encountered a bug you'd like to report or have an idea for a new feature? Check out our `issue tracker`_! If your issue or feature hasn't already been reported, please `open a new ticket`_ and we'll be in touch with you shortly. If you'd like to vote on a feature/bug, simply give a :+1: on issues you'd like to see prioritized over others. +* Need help/support, would like to start a discussion, or would just like to introduce yourself to the team? Check out our `forums`_! + +.. _issue tracker: https://github.com/beetbox/beets/issues +.. _open a new ticket: https://github.com/beetbox/beets/issues/new/choose +.. _forums: https://discourse.beets.io/ + Authors ------- From 426cb0800305f4b88854c17aeed9d742e3b57551 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:02:38 -0500 Subject: [PATCH 256/293] fix CONTRIBUTING.md link and spelling --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7f509efdc..4cf66a29a 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `Contributing.md`_. +Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.md`_. -.. _Contributing.md: https://github.com/beetbox/beets/blob/contributing.md/contributing.md +.. _CONTRIBUTING.md: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.md Read More --------- From 5544c7a8813fe68ce89e2e65aecfc8c9b6f70f73 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:04:44 -0500 Subject: [PATCH 257/293] Fix wording --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e2c8964d..4ed28746e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. -These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team. By following them, you indicate your respect for the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. +These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team as it indicates you respect the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. # Types of Contributions From c1b9e99722291cd267ffc5ca985302d80c99d445 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:07:32 -0500 Subject: [PATCH 258/293] maintainers *will* reciprocate respect :) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ed28746e..e16e10cb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. -These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team as it indicates you respect the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. +These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team as it indicates you respect the maintainers' time; in return, the maintainers will reciprocate by helping to address your issue, review changes, and finalize pull requests. # Types of Contributions From b309ed95c2fdf0613b522db0fa2048a75e3ee3ab Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Mon, 20 Jul 2020 19:17:11 -0500 Subject: [PATCH 259/293] Remove duplicate forum mention Forums are already mentioned in the 'Contact' section --- README.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4cf66a29a..d57afa933 100644 --- a/README.rst +++ b/README.rst @@ -115,8 +115,6 @@ Contact Authors ------- -Beets is by `Adrian Sampson`_ with a supporting cast of thousands. For help, -please visit our `forum`_. +Beets is by `Adrian Sampson`_ with a supporting cast of thousands. -.. _forum: https://discourse.beets.io .. _Adrian Sampson: https://www.cs.cornell.edu/~asampson/ From faa55534680f816fa81e71ea886b54dfee5bfa0f Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 18:55:27 -0700 Subject: [PATCH 260/293] convert CONTRIBUTING.md to rst --- CONTRIBUTING.md | 165 ------------------------------ CONTRIBUTING.rst | 255 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+), 165 deletions(-) delete mode 100644 CONTRIBUTING.md create mode 100644 CONTRIBUTING.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 4e2c8964d..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,165 +0,0 @@ -# Thank you! - -First off, thank you for considering contributing to beets! It's people like you that make beets continue to succeed. - -These guidelines describe how you can help most effectively. By following these guidelines, you can make life easier for the development team. By following them, you indicate your respect for the maintainers' time; in return, the maintainers can reciprocate by helping to address your issue, review changes, and finalize pull requests. - -# Types of Contributions - -We love to get contributions from our community—you! There are many ways to contribute, whether you're a programmer or not. - -## Non-Programming - -* Promote beets! Help get the word out by telling your friends, writing a blog - post, or discussing it on a forum you frequent. -* Improve the [documentation][docs]. It's incredibly easy to contribute here: - just find a page you want to modify and hit the "Edit on GitHub" button in - the upper-right. You can automatically send us a pull request for your - changes. -* GUI design. For the time being, beets is a command-line-only affair. But - that's mostly because we don't have any great ideas for what a good GUI - should look like. If you have those great ideas, please get in touch. -* Benchmarks. We'd like to have a consistent way of measuring speed - improvements in beets' tagger and other functionality as well as a way of - comparing beets' performance to other tools. You can help by compiling a - library of freely-licensed music files (preferably with incorrect metadata) - for testing and measurement. -* Think you have a nice config or cool use-case for beets? We'd love to hear about it! Submit a post to our [our forums][forum] under the "Show and Tell" category for a chance to get featured in [the docs][advanced]. -* Consider helping out in [our forums][forum] by responding to support requests or driving some new discussions. - -[docs]: http://beets.readthedocs.org/ -[forum]: https://discourse.beets.io/ -[advanced]: https://beets.readthedocs.io/en/stable/guides/advanced.html - -## Programming - -* As a programmer (even if you're just a beginner!), you have a ton of opportunities to get your feet wet with beets. -* For developing plugins, or hacking away at beets, there's some good information in the ["For Developers" section of the docs][dev-docs]. - -[dev-docs]: https://beets.readthedocs.io/en/stable/dev/ - -### Getting the Source - -The easiest way to get started with the latest beets source is to use [pip][] to install an "editable" package. This can be done with one command: - - $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets - -Or, equivalently: - - $ git clone https://github.com/beetbox/beets.git - $ cd beets - $ pip install -e . - -If you already have a released version of beets installed, you may need to -remove it first by typing `pip uninstall beets`. The pip command above will put -the beets source in a `src/beets` directory and install the `beet` CLI script to -a standard location on your system. You may want to use the `--src` option to specify -the parent directory where the source will be checked out and the `--user` option -such that the package will be installed to your home directory (compare with the output of -`pip install --help`). - -[pip]: https://pip.pypa.io/ - -### Code Contribution Ideas - -* We maintain a set of [issues marked as "bite-sized"](https://github.com/beetbox/beets/labels/bitesize). These are issues that would serve as a good introduction to the codebase. Claim one and start exploring! -* Like testing? Our [test coverage](https://codecov.io/github/beetbox/beets) is somewhat low. You can help out by finding low-coverage modules or checking out other [testing-related issues](https://github.com/beetbox/beets/labels/testing). -* There are several ways to improve the tests in general (see [Testing](https://github.com/beetbox/beets/wiki/Testing)) and some places to think about performance optimization (see [Optimization](https://github.com/beetbox/beets/wiki/Optimization)). -* Not all of our code is up to our coding conventions. In particular, the [API documentation](https://beets.readthedocs.io/en/stable/dev/api.html) are currently quite sparse. You can help by adding to the docstrings in the code and to the documentation pages themselves. beets follows [PEP-257](https://www.python.org/dev/peps/pep-0257/) for docstrings and in some places, we also sometimes use [ReST autodoc syntax for Sphinx](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html) to, for example, refer to a class name. - -## Your First Contribution - -If this is your first time contributing to an open source project, welcome! If you are confused at all about how to contribute or what to contribute, take a look at [this great tutorial](http://makeapullrequest.com/), or stop by our [forums](https://discourse.beets.io/) if you have any questions. - -We maintain a list of issues we reserved for those new to open source labeled ["first timers only"](https://github.com/beetbox/beets/issues?q=is%3Aopen+is%3Aissue+label%3A%22first+timers+only%22). Since the goal of these issues is to get users comfortable with contributing to an open source project, please do not hesitate to ask any questions. - -## How to Submit Your Work - -Do you have a great bug fix, new feature, or documentation expansion you'd like to contribute? Follow these steps to create a GitHub pull request and your code will ship in no time. - -1. Fork the beets repository and clone it (see above) to create a workspace. -2. Make your changes. -3. Add tests. If you've fixed a bug, write a test to ensure that you've actually fixed it. If there's a new feature or plugin, please contribute tests that show that your code does what it says. -4. Add documentation. If you've added a new command flag, for example, find the appropriate page under `docs/` where it needs to be listed. -5. Add a changelog entry to `docs/changelog.rst` near the top of the document. -6. Run the tests and style checker. The easiest way to run the tests is to use [tox](https://tox.readthedocs.org/en/latest/). For more information on running tests, see our [Testing wiki page](https://github.com/beetbox/beets/wiki/Testing). -7. Push to your fork and open a pull request! We'll be in touch shortly. -8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn't automatically notify us when commits are added. - -Remember, code contributions have four parts: the code, the tests, the documentation, and the changelog entry. Thank you for contributing! - - -# The Code - -The documentation has an [API section](https://beets.readthedocs.io/en/stable/dev/api.html) that serves as an introduction to beets' design. - -## Coding Conventions - -There are a few coding conventions we use in beets: - -* Whenever you access the library database, do so through the provided Library - methods or via a Transaction object. Never call `lib.conn.*` directly. - For example, do this: - ``` - with g.lib.transaction() as tx: - rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' - .format(field, model._table, sort_field)) - ``` - To fetch Item objects from the database, use lib.items(...) and supply a query as an argument. Resist the urge to write raw SQL for your query. If you must use lower-level queries into the database, do this: - ``` - with lib.transaction() as tx: - rows = tx.query('SELECT …') - ``` - Transaction objects help control concurrent access to the database and assist in debugging conflicting accesses. -* Always use the [future imports][] `print_function`, `division`, and - `absolute_import`, but *not* `unicode_literals`. These help keep your code - modern and will help in the eventual move to Python 3. -* `str.format()` should be used instead of the `%` operator -* Never `print` informational messages; use the [logging][] module instead. In - particular, we have our own logging shim, so you'll see `from beets import - logging` in most files. - * Always log Unicode strings (e.g., `log.debug(u"hello world")`). - * The loggers use [str.format][]-style logging instead of ``%``-style, so - you can type `log.debug(u"{0}", obj)` to do your formatting. -* Exception handlers must use `except A as B:` instead of `except A, B:`. - -[future imports]: http://docs.python.org/library/__future__.html -[logging]: http://docs.python.org/library/logging.html -[str.format]: http://docs.python.org/library/stdtypes.html#str.format -[modformat]: http://docs.python.org/library/stdtypes.html#string-formatting-operations - -We follow [PEP 8](http://www.python.org/dev/peps/pep-0008/) for style. You can use `tox -e lint` to check your code for any style errors. - -## Handling Paths - -A great deal of convention deals with the handling of **paths**. Paths are -stored internally—in the database, for instance—as byte strings (i.e., `bytes` instead of `str` in Python 3). This is because POSIX operating systems' path names are only -reliably usable as byte strings—operating systems typically recommend but do not require that filenames use a given encoding, so violations of any reported encoding are inevitable. -On Windows, the strings are always encoded with UTF-8; on -Unix, the encoding is controlled by the filesystem. Here are some guidelines to -follow: - -* If you have a Unicode path or you're not sure whether something is Unicode or - not, pass it through `bytestring_path` function in the `beets.util` module to - convert it to bytes. -* Pass every path name trough the `syspath` function (also in `beets.util`) - before sending it to any *operating system* file operation (`open`, for - example). This is necessary to use long filenames (which, maddeningly, must - be Unicode) on Windows. This allows us to consistently store bytes in the - database but use the native encoding rule on both POSIX and Windows. -* Similarly, the `displayable_path` utility function converts bytestring paths - to a Unicode string for displaying to the user. Every time you want to print - out a string to the terminal or log it with the `logging` module, feed it - through this function. - -## Editor Settings - -Personally, I work on beets with [vim](http://www.vim.org/). Here are some -`.vimrc` lines that might help with PEP 8-compliant Python coding: - - filetype indent on - autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent - -Consider installing [this alternative Python indentation -plugin](https://github.com/mitsuhiko/vim-python-combined). I also like -[neomake](https://github.com/neomake/neomake) with its flake8 checker. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 000000000..2596fd135 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,255 @@ +Thank you! +========== + +First off, thank you for considering contributing to beets! It’s people +like you that make beets continue to succeed. + +These guidelines describe how you can help most effectively. By +following these guidelines, you can make life easier for the development +team as it indicates you respect the maintainers’ time; in return, the +maintainers will reciprocate by helping to address your issue, review +changes, and finalize pull requests. + +Types of Contributions +====================== + +We love to get contributions from our community—you! There are many ways +to contribute, whether you’re a programmer or not. + +Non-Programming +--------------- + +- Promote beets! Help get the word out by telling your friends, writing + a blog post, or discussing it on a forum you frequent. +- Improve the `documentation `__. It’s + incredibly easy to contribute here: just find a page you want to + modify and hit the “Edit on GitHub” button in the upper-right. You + can automatically send us a pull request for your changes. +- GUI design. For the time being, beets is a command-line-only affair. + But that’s mostly because we don’t have any great ideas for what a + good GUI should look like. If you have those great ideas, please get + in touch. +- Benchmarks. We’d like to have a consistent way of measuring speed + improvements in beets’ tagger and other functionality as well as a + way of comparing beets’ performance to other tools. You can help by + compiling a library of freely-licensed music files (preferably with + incorrect metadata) for testing and measurement. +- Think you have a nice config or cool use-case for beets? We’d love to + hear about it! Submit a post to our `our + forums `__ under the “Show and Tell” + category for a chance to get featured in `the + docs `__. +- Consider helping out in `our forums `__ + by responding to support requests or driving some new discussions. + +Programming +----------- + +- As a programmer (even if you’re just a beginner!), you have a ton of + opportunities to get your feet wet with beets. +- For developing plugins, or hacking away at beets, there’s some good + information in the `“For Developers” section of the + docs `__. + +Getting the Source +~~~~~~~~~~~~~~~~~~ + +The easiest way to get started with the latest beets source is to use +`pip `__ to install an “editable” package. This +can be done with one command: + +:: + + $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets + +Or, equivalently: + +:: + + $ git clone https://github.com/beetbox/beets.git + $ cd beets + $ pip install -e . + +If you already have a released version of beets installed, you may need +to remove it first by typing ``pip uninstall beets``. The pip command +above will put the beets source in a ``src/beets`` directory and install +the ``beet`` CLI script to a standard location on your system. You may +want to use the ``--src`` option to specify the parent directory where +the source will be checked out and the ``--user`` option such that the +package will be installed to your home directory (compare with the +output of ``pip install --help``). + +Code Contribution Ideas +~~~~~~~~~~~~~~~~~~~~~~~ + +- We maintain a set of `issues marked as + “bite-sized” `__. + These are issues that would serve as a good introduction to the + codebase. Claim one and start exploring! +- Like testing? Our `test + coverage `__ is somewhat + low. You can help out by finding low-coverage modules or checking out + other `testing-related + issues `__. +- There are several ways to improve the tests in general (see + `Testing `__) and some + places to think about performance optimization (see + `Optimization `__). +- Not all of our code is up to our coding conventions. In particular, + the `API + documentation `__ + are currently quite sparse. You can help by adding to the docstrings + in the code and to the documentation pages themselves. beets follows + `PEP-257 `__ for + docstrings and in some places, we also sometimes use `ReST autodoc + syntax for + Sphinx `__ + to, for example, refer to a class name. + +Your First Contribution +----------------------- + +If this is your first time contributing to an open source project, +welcome! If you are confused at all about how to contribute or what to +contribute, take a look at `this great +tutorial `__, or stop by our +`forums `__ if you have any questions. + +We maintain a list of issues we reserved for those new to open source +labeled `“first timers +only” `__. +Since the goal of these issues is to get users comfortable with +contributing to an open source project, please do not hesitate to ask +any questions. + +How to Submit Your Work +----------------------- + +Do you have a great bug fix, new feature, or documentation expansion +you’d like to contribute? Follow these steps to create a GitHub pull +request and your code will ship in no time. + +1. Fork the beets repository and clone it (see above) to create a + workspace. +2. Make your changes. +3. Add tests. If you’ve fixed a bug, write a test to ensure that you’ve + actually fixed it. If there’s a new feature or plugin, please + contribute tests that show that your code does what it says. +4. Add documentation. If you’ve added a new command flag, for example, + find the appropriate page under ``docs/`` where it needs to be + listed. +5. Add a changelog entry to ``docs/changelog.rst`` near the top of the + document. +6. Run the tests and style checker. The easiest way to run the tests is + to use `tox `__. For more + information on running tests, see our `Testing wiki + page `__. +7. Push to your fork and open a pull request! We’ll be in touch shortly. +8. If you add commits to a pull request, please add a comment or + re-request a review after you push them since GitHub doesn’t + automatically notify us when commits are added. + +Remember, code contributions have four parts: the code, the tests, the +documentation, and the changelog entry. Thank you for contributing! + +The Code +======== + +The documentation has an `API +section `__ that +serves as an introduction to beets’ design. + +Coding Conventions +------------------ + +There are a few coding conventions we use in beets: + +- Whenever you access the library database, do so through the provided + Library methods or via a Transaction object. Never call + ``lib.conn.*`` directly. For example, do this: + + :: + + with g.lib.transaction() as tx: + rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' + .format(field, model._table, sort_field)) + + To fetch Item objects from the database, use lib.items(…) and supply + a query as an argument. Resist the urge to write raw SQL for your + query. If you must use lower-level queries into the database, do + this: + + :: + + with lib.transaction() as tx: + rows = tx.query('SELECT …') + + Transaction objects help control concurrent access to the database + and assist in debugging conflicting accesses. +- Always use the `future + imports `__ + ``print_function``, ``division``, and ``absolute_import``, but *not* + ``unicode_literals``. These help keep your code modern and will help + in the eventual move to Python 3. +- ``str.format()`` should be used instead of the ``%`` operator +- Never ``print`` informational messages; use the + `logging `__ module + instead. In particular, we have our own logging shim, so you’ll see + ``from beets import logging`` in most files. + + - Always log Unicode strings (e.g., ``log.debug(u"hello world")``). + - The loggers use + `str.format `__-style + logging instead of ``%``-style, so you can type + ``log.debug(u"{0}", obj)`` to do your formatting. + +- Exception handlers must use ``except A as B:`` instead of + ``except A, B:``. + +We follow `PEP 8 `__ for +style. You can use ``tox -e lint`` to check your code for any style +errors. + +Handling Paths +-------------- + +A great deal of convention deals with the handling of **paths**. Paths +are stored internally—in the database, for instance—as byte strings +(i.e., ``bytes`` instead of ``str`` in Python 3). This is because POSIX +operating systems’ path names are only reliably usable as byte +strings—operating systems typically recommend but do not require that +filenames use a given encoding, so violations of any reported encoding +are inevitable. On Windows, the strings are always encoded with UTF-8; +on Unix, the encoding is controlled by the filesystem. Here are some +guidelines to follow: + +- If you have a Unicode path or you’re not sure whether something is + Unicode or not, pass it through ``bytestring_path`` function in the + ``beets.util`` module to convert it to bytes. +- Pass every path name trough the ``syspath`` function (also in + ``beets.util``) before sending it to any *operating system* file + operation (``open``, for example). This is necessary to use long + filenames (which, maddeningly, must be Unicode) on Windows. This + allows us to consistently store bytes in the database but use the + native encoding rule on both POSIX and Windows. +- Similarly, the ``displayable_path`` utility function converts + bytestring paths to a Unicode string for displaying to the user. + Every time you want to print out a string to the terminal or log it + with the ``logging`` module, feed it through this function. + +Editor Settings +--------------- + +Personally, I work on beets with `vim `__. Here are +some ``.vimrc`` lines that might help with PEP 8-compliant Python +coding: + +:: + + filetype indent on + autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent + +Consider installing `this alternative Python indentation +plugin `__. I also +like `neomake `__ with its flake8 +checker. From 2982f245e1a8a41758bfb8038aa9229e3330facd Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 18:55:46 -0700 Subject: [PATCH 261/293] add CONTRIBUTING.rst to the docs --- docs/contributing.rst | 3 +++ docs/index.rst | 1 + 2 files changed, 4 insertions(+) create mode 100644 docs/contributing.rst diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 000000000..6af7deaef --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,3 @@ +.. contributing: + +.. include:: ../CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst index 62c87461b..1d03a0f27 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents reference/index plugins/index faq + contributing dev/index .. toctree:: From 66e6af48be5b09bba5b71cbfbfa4e53521a9a887 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 19:38:55 -0700 Subject: [PATCH 262/293] add table of contents to contributing.rst --- CONTRIBUTING.rst | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2596fd135..66298c199 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,3 +1,10 @@ +############ +Contributing +############ + +.. contents:: + :depth: 3 + Thank you! ========== @@ -52,19 +59,19 @@ Programming docs `__. Getting the Source -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ The easiest way to get started with the latest beets source is to use `pip `__ to install an “editable” package. This can be done with one command: -:: +.. code-block:: bash $ pip install -e git+https://github.com/beetbox/beets.git#egg=beets Or, equivalently: -:: +.. code-block:: bash $ git clone https://github.com/beetbox/beets.git $ cd beets @@ -80,7 +87,7 @@ package will be installed to your home directory (compare with the output of ``pip install --help``). Code Contribution Ideas -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ - We maintain a set of `issues marked as “bite-sized” `__. @@ -107,7 +114,7 @@ Code Contribution Ideas to, for example, refer to a class name. Your First Contribution ------------------------ +======================= If this is your first time contributing to an open source project, welcome! If you are confused at all about how to contribute or what to @@ -123,7 +130,7 @@ contributing to an open source project, please do not hesitate to ask any questions. How to Submit Your Work ------------------------ +======================= Do you have a great bug fix, new feature, or documentation expansion you’d like to contribute? Follow these steps to create a GitHub pull @@ -160,15 +167,17 @@ section `__ that serves as an introduction to beets’ design. Coding Conventions ------------------- +================== +General +------- There are a few coding conventions we use in beets: - Whenever you access the library database, do so through the provided Library methods or via a Transaction object. Never call ``lib.conn.*`` directly. For example, do this: - :: + .. code-block:: python with g.lib.transaction() as tx: rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"' @@ -179,7 +188,7 @@ There are a few coding conventions we use in beets: query. If you must use lower-level queries into the database, do this: - :: + .. code-block:: python with lib.transaction() as tx: rows = tx.query('SELECT …') @@ -206,6 +215,8 @@ There are a few coding conventions we use in beets: - Exception handlers must use ``except A as B:`` instead of ``except A, B:``. +Style +----- We follow `PEP 8 `__ for style. You can use ``tox -e lint`` to check your code for any style errors. @@ -238,7 +249,7 @@ guidelines to follow: with the ``logging`` module, feed it through this function. Editor Settings ---------------- +^^^^^^^^^^^^^^^ Personally, I work on beets with `vim `__. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python From f677fd24078a634271c6b20e6c3bdca48142cdc8 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 19:39:06 -0700 Subject: [PATCH 263/293] add testing wiki to contributing.rst --- CONTRIBUTING.rst | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 66298c199..8f6ab54b2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -264,3 +264,95 @@ Consider installing `this alternative Python indentation plugin `__. I also like `neomake `__ with its flake8 checker. + +Testing +======= + +Running the Tests +----------------- + +To run the tests for multiple Python versions, compile the docs, and +check style, use `tox`_. Just type ``tox`` or use something like +``tox -e py27`` to test a specific configuration. `detox`_ makes this go +faster. + +You can disable a hand-selected set of "slow" tests by setting the +environment variable SKIP_SLOW_TESTS before running them. + +Other ways to run the tests: + +- ``python testall.py`` (ditto) +- ``python -m unittest discover -p 'test_*'`` (ditto) +- `pytest`_ + +You can also see the latest test results on `Linux`_ and on `Windows`_. + +Test Dependencies +^^^^^^^^^^^^^^^^^ + +The tests have a few more dependencies than beets itself. (The +additional dependencies consist of testing utilities and dependencies of +non-default plugins exercised by the test suite.) The dependencies are +listed under 'test' in ``extras_require`` in `setup.py`_. +To install the test dependencies, run ``python -m pip install .[test]``. +Or, just run a test suite with ``tox`` which will install them +automatically. + +.. _setup.py: https://github.com/beetbox/beets/blob/master/setup.py#L99` + +Writing Tests +------------- + +Writing tests is done by adding or modifying files in folder `test`_. +Take a look at +`https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`_ +to get a basic view on how tests are written. Despite using ``pytest`` +as a test runner, we prefer to write tests using the standard +`unittest`_ testing framework. + +Any tests that involve sending out network traffic e.g. an external API +call, should be skipped normally and run under our weekly `integration +test`_ suite. These tests can be useful in detecting external changes +that would affect ``beets``. In order to do this, simply add the +following snippet before the applicable test case: + +:: + + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') + +If you do this, it is also advised to create a similar test that 'mocks' +the network call and can be run under normal circumstances by our CI and +others. See `unittest.mock`_ for more info. + +Basics +^^^^^^ + +- Your file should contain a class derived from unittest.TestCase +- Each method in this class which name starts with the letters *test* + will be executed to test functionality +- Errors are raised with these methods: + + - ``self.assertEqual`` + - ``self.assertTrue`` + - ``self.assertFalse`` + - ``self.assertRaises`` + +- For detailed information see `Python unittest`_ +- **AVOID** using the ``start()`` and ``stop()`` methods of + ``mock.patch``, as they require manual cleanup. Use the annotation or + context mana + +.. _tox: http://tox.readthedocs.org +.. _detox: https://pypi.python.org/pypi/detox/ +.. _pytest: http://pytest.org +.. _Linux: https://github.com/beetbox/beets/actions +.. _Windows: https://ci.appveyor.com/project/beetbox/beets/ +.. _`https://github.com/beetbox/beets/blob/master/setup.py#L99`: https://github.com/beetbox/beets/blob/master/setup.py#L99 +.. _test: https://github.com/beetbox/beets/tree/master/test +.. _`https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`: https://github.com/beetbox/beets/blob/master/test/test_template.py#L224 +.. _unittest: https://docs.python.org/3.8/library/unittest.html +.. _integration test: https://github.com/beetbox/beets/actions?query=workflow%3A%22integration+tests%22 +.. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html +.. _Python unittest: https://docs.python.org/2/library/unittest.html From cb4bab175a4b5348a058654a527c04b3a2f9c233 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Mon, 20 Jul 2020 19:51:48 -0700 Subject: [PATCH 264/293] fix link to contributing page in readme --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d57afa933..4a2d8cb10 100644 --- a/README.rst +++ b/README.rst @@ -90,9 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.md`_. +Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.rst`_. -.. _CONTRIBUTING.md: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.md +.. _CONTRIBUTING.rst: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.rst Read More --------- From bf868be10ea19955dd5dd9b28d8206665374e9ec Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 21 Jul 2020 14:41:01 -0700 Subject: [PATCH 265/293] fix-up adding testing wiki to contributing.rst --- CONTRIBUTING.rst | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8f6ab54b2..5e3a155cf 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -98,8 +98,7 @@ Code Contribution Ideas low. You can help out by finding low-coverage modules or checking out other `testing-related issues `__. -- There are several ways to improve the tests in general (see - `Testing `__) and some +- There are several ways to improve the tests in general (see :ref:`testing` and some places to think about performance optimization (see `Optimization `__). - Not all of our code is up to our coding conventions. In particular, @@ -149,8 +148,7 @@ request and your code will ship in no time. document. 6. Run the tests and style checker. The easiest way to run the tests is to use `tox `__. For more - information on running tests, see our `Testing wiki - page `__. + information on running tests, see :ref:`testing`. 7. Push to your fork and open a pull request! We’ll be in touch shortly. 8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn’t @@ -265,6 +263,8 @@ plugin `__. I also like `neomake `__ with its flake8 checker. +.. _testing: + Testing ======= @@ -287,6 +287,19 @@ Other ways to run the tests: You can also see the latest test results on `Linux`_ and on `Windows`_. +Coverage +^^^^^^^^ + +``tox -e cov`` will add coverage info for tests: Coverage is pretty low +still -- see the current status on `Codecov`_. + +Red Flags +^^^^^^^^^ + +The `pytest-random`_ plugin makes it easy to randomize the order of +tests. ``py.test test --random`` will occasionally turn up failing tests +that reveal ordering dependencies—which are bad news! + Test Dependencies ^^^^^^^^^^^^^^^^^ @@ -342,8 +355,11 @@ Basics - For detailed information see `Python unittest`_ - **AVOID** using the ``start()`` and ``stop()`` methods of ``mock.patch``, as they require manual cleanup. Use the annotation or - context mana + context manager forms instead. +.. _Python unittest: https://docs.python.org/2/library/unittest.html +.. _Codecov: https://codecov.io/github/beetbox/beets +.. _pytest-random: https://github.com/klrmn/pytest-random .. _tox: http://tox.readthedocs.org .. _detox: https://pypi.python.org/pypi/detox/ .. _pytest: http://pytest.org From 9690e4d51fe91081921fff2dc0e370d7c61aeec5 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Tue, 21 Jul 2020 14:49:51 -0700 Subject: [PATCH 266/293] improve code blocks --- CONTRIBUTING.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5e3a155cf..6c024e3d9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -251,9 +251,7 @@ Editor Settings Personally, I work on beets with `vim `__. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python -coding: - -:: +coding:: filetype indent on autocmd FileType python setlocal shiftwidth=4 tabstop=4 softtabstop=4 expandtab shiftround autoindent @@ -329,11 +327,11 @@ test`_ suite. These tests can be useful in detecting external changes that would affect ``beets``. In order to do this, simply add the following snippet before the applicable test case: -:: +.. code-block:: python - @unittest.skipUnless( - os.environ.get('INTEGRATION_TEST', '0') == '1', - 'integration testing not enabled') + @unittest.skipUnless( + os.environ.get('INTEGRATION_TEST', '0') == '1', + 'integration testing not enabled') If you do this, it is also advised to create a similar test that 'mocks' the network call and can be run under normal circumstances by our CI and From 6febcf11903c31b35b644ca82584ea187d32a155 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:14:43 +0200 Subject: [PATCH 267/293] fix #3685 --- beets/config_default.yaml | 1 + beets/dbcore/db.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 0fd6eb592..c75778b80 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -44,6 +44,7 @@ replace: '^\s+': '' '^-': _ path_sep_replace: _ +drive_sep_replace: _ asciify_paths: false art_filename: cover max_filename_length: 0 diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index b13f2638a..abcbdd029 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -19,6 +19,7 @@ from __future__ import division, absolute_import, print_function import time import os +import re from collections import defaultdict import threading import sqlite3 @@ -83,6 +84,8 @@ class FormattedMapping(Mapping): value = value.decode('utf-8', 'ignore') if self.for_path: + if re.match(r'^\w:', value): + value = value.replace(':', beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() for sep in (os.path.sep, os.path.altsep): if sep: From dfdb068b51288b77ccb238edb48d1d082d8d705e Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:24:39 +0200 Subject: [PATCH 268/293] fix linting error --- beets/dbcore/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index abcbdd029..595078c42 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -85,7 +85,8 @@ class FormattedMapping(Mapping): if self.for_path: if re.match(r'^\w:', value): - value = value.replace(':', beets.config['drive_sep_replace'].as_str()) + value = value.replace(':', + beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() for sep in (os.path.sep, os.path.altsep): if sep: From 8dad7fa67fc19dae6e33101fcafa9f4bb3458cd3 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:26:21 +0200 Subject: [PATCH 269/293] fix linting error again --- beets/dbcore/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 595078c42..38fbae3bc 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -85,8 +85,8 @@ class FormattedMapping(Mapping): if self.for_path: if re.match(r'^\w:', value): - value = value.replace(':', - beets.config['drive_sep_replace'].as_str()) + value = value.replace(':', + beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() for sep in (os.path.sep, os.path.altsep): if sep: From b91efb88f484dad47b4b4abe333447cabfabf66e Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:30:13 +0200 Subject: [PATCH 270/293] extract config to variable --- beets/dbcore/db.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 38fbae3bc..9dbe9316b 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -84,10 +84,11 @@ class FormattedMapping(Mapping): value = value.decode('utf-8', 'ignore') if self.for_path: - if re.match(r'^\w:', value): - value = value.replace(':', - beets.config['drive_sep_replace'].as_str()) sep_repl = beets.config['path_sep_replace'].as_str() + sep_drive = beets.config['drive_sep_replace'].as_str() + if re.match(r'^\w:', value): + value = value.replace(':', sep_drive) + for sep in (os.path.sep, os.path.altsep): if sep: value = value.replace(sep, sep_repl) From 1e6db940450a561d110fb3e716bd6f0c11149905 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 20:32:11 +0200 Subject: [PATCH 271/293] remove whitespace --- beets/dbcore/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 9dbe9316b..0f43f4bc5 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -86,9 +86,9 @@ class FormattedMapping(Mapping): if self.for_path: sep_repl = beets.config['path_sep_replace'].as_str() sep_drive = beets.config['drive_sep_replace'].as_str() - if re.match(r'^\w:', value): + if re.match(r'^\w:', value): value = value.replace(':', sep_drive) - + for sep in (os.path.sep, os.path.altsep): if sep: value = value.replace(sep, sep_repl) From 0a9a801b0538bff872cd18666988840ae1ab7b84 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 21:50:33 +0200 Subject: [PATCH 272/293] add unit tests --- test/test_files.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_files.py b/test/test_files.py index f31779672..ee50e583e 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -102,6 +102,19 @@ class MoveTest(_common.TestCase): self.i.move() self.assertEqual(self.i.path, old_path) + def test_move_file_with_colon(self): + self.i.artist = u'C:DOS' + self.i.move() + self.assertRegex(self.i.path.decode(), r'C_DOS') + + def test_move_file_with_colon_alt_separator(self): + old = beets.config['drive_sep_replace'] + beets.config["drive_sep_replace"] = '0' + self.i.artist = u'C:DOS' + self.i.move() + self.assertRegex(self.i.path.decode(), r"C0DOS") + beets.config["drive_sep_replace"] = old + def test_read_only_file_copied_writable(self): # Make the source file read-only. os.chmod(self.path, 0o444) From f73a438dce79cdf48d6d14c730a1cb8d7b166408 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 25 Jul 2020 13:02:19 -0700 Subject: [PATCH 273/293] Add python 3.9-dev to CI tests (#3687) * Add python 3.9-dev to ci tests * test integration tests on latest versions of python --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/integration_test.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f9cce8d23..08e7548f2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,7 +6,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: [2.7, 3.5, 3.6, 3.7, 3.8] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9-dev] env: PY_COLORS: 1 @@ -82,4 +82,4 @@ jobs: python -m pip install tox sphinx - name: Lint with flake8 - run: tox -e py-lint \ No newline at end of file + run: tox -e py-lint diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 05623bf20..1e8b3a773 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up latest Python version uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9-dev - name: Install base dependencies run: | From 043fa10f5566c2cab0b7d6c15590cd8f60e6be4e Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 25 Jul 2020 13:58:57 -0500 Subject: [PATCH 274/293] Add tox windows issue to CONTRIBUTING.rst --- CONTRIBUTING.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6c024e3d9..b1bd22169 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -285,6 +285,11 @@ Other ways to run the tests: You can also see the latest test results on `Linux`_ and on `Windows`_. +Note, if you are on Windows and are seeing errors running tox, it may be related to `this issue`_, +in which case you may have to install tox v3.8.3 e.g. ``python -m pip install tox=3.8.3`` + +.. _this issue: https://github.com/tox-dev/tox/issues/1550 + Coverage ^^^^^^^^ From 1eda0d39e6dcb8f75741e50e9379ef7da0ccdd28 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sat, 25 Jul 2020 22:11:07 +0200 Subject: [PATCH 275/293] better regex for drive letter; hopefully fixed tests for 2.7; added test to show only drive-letter like colons are affected --- beets/dbcore/db.py | 3 ++- test/test_files.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 0f43f4bc5..46b47a2e1 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -86,8 +86,9 @@ class FormattedMapping(Mapping): if self.for_path: sep_repl = beets.config['path_sep_replace'].as_str() sep_drive = beets.config['drive_sep_replace'].as_str() + if re.match(r'^\w:', value): - value = value.replace(':', sep_drive) + value = re.sub(r'(?<=^\w):', sep_drive, value) for sep in (os.path.sep, os.path.altsep): if sep: diff --git a/test/test_files.py b/test/test_files.py index ee50e583e..13a8b4407 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -105,14 +105,20 @@ class MoveTest(_common.TestCase): def test_move_file_with_colon(self): self.i.artist = u'C:DOS' self.i.move() - self.assertRegex(self.i.path.decode(), r'C_DOS') + self.assertIn('C_DOS', self.i.path.decode()) + + def test_move_file_with_multiple_colons(self): + print(beets.config['replace']) + self.i.artist = u'COM:DOS' + self.i.move() + self.assertIn('COM_DOS', self.i.path.decode()) def test_move_file_with_colon_alt_separator(self): old = beets.config['drive_sep_replace'] beets.config["drive_sep_replace"] = '0' self.i.artist = u'C:DOS' self.i.move() - self.assertRegex(self.i.path.decode(), r"C0DOS") + self.assertIn('C0DOS', self.i.path.decode()) beets.config["drive_sep_replace"] = old def test_read_only_file_copied_writable(self): From 9ee341374af8e1f3a4011d98aa0a20aaaa34886b Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sat, 25 Jul 2020 15:56:00 -0700 Subject: [PATCH 276/293] fix subheading in contributing.rst --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b1bd22169..dc8617712 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -247,7 +247,7 @@ guidelines to follow: with the ``logging`` module, feed it through this function. Editor Settings -^^^^^^^^^^^^^^^ +--------------- Personally, I work on beets with `vim `__. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python From 08b51ee5bc009133140307d42fc80782a3905be1 Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Sun, 26 Jul 2020 20:53:30 +0200 Subject: [PATCH 277/293] add changelog entry --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0f41c38ec..64e6ab85c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -235,6 +235,9 @@ Fixes: * :doc:`/plugins/ipfs`: Fix Python 3 compatibility. Thanks to :user:`musoke`. :bug:`2554` +* Fix a bug that caused metadata starting with something resembling a drive + letter to be incorrectly split into an extra directory after the colon. + :bug:`3685` For plugin developers: From 30173b7331fb887f3b036865e4cad41ff4516a4c Mon Sep 17 00:00:00 2001 From: chipsnblip Date: Sun, 26 Jul 2020 13:50:05 -0700 Subject: [PATCH 278/293] update link to current homepage update link from non-existent page (https://2.python-requests.org/en/latest/) to the plugin's current homepage: https://requests.readthedocs.io/en/master/ --- docs/plugins/fetchart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 92ed5f32a..168ca0fa0 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -11,7 +11,7 @@ To use the ``fetchart`` plugin, first enable it in your configuration (see The plugin uses `requests`_ to fetch album art from the Web. -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ Fetching Album Art During Import -------------------------------- From 0cd102b0ddaa3244682cfaa6e20745b891b593ad Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Tue, 28 Jul 2020 21:26:12 -0500 Subject: [PATCH 279/293] Update pytest test writing restriction I think the basics sections is fairly self explanatory at this point especially with the copious amounts of examples we have. Also, if we kept it, we'd have to expand on pytest basics as well. I'd rather just point to the docs/getting started guides for each. --- CONTRIBUTING.rst | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dc8617712..e839599d4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -322,9 +322,8 @@ Writing Tests Writing tests is done by adding or modifying files in folder `test`_. Take a look at `https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`_ -to get a basic view on how tests are written. Despite using ``pytest`` -as a test runner, we prefer to write tests using the standard -`unittest`_ testing framework. +to get a basic view on how tests are written. We currently allow writing +tests with either `unittest`_ or `pytest`_. Any tests that involve sending out network traffic e.g. an external API call, should be skipped normally and run under our weekly `integration @@ -342,20 +341,6 @@ If you do this, it is also advised to create a similar test that 'mocks' the network call and can be run under normal circumstances by our CI and others. See `unittest.mock`_ for more info. -Basics -^^^^^^ - -- Your file should contain a class derived from unittest.TestCase -- Each method in this class which name starts with the letters *test* - will be executed to test functionality -- Errors are raised with these methods: - - - ``self.assertEqual`` - - ``self.assertTrue`` - - ``self.assertFalse`` - - ``self.assertRaises`` - -- For detailed information see `Python unittest`_ - **AVOID** using the ``start()`` and ``stop()`` methods of ``mock.patch``, as they require manual cleanup. Use the annotation or context manager forms instead. From 019055c156de1b9d740caa306f2da0376051afe5 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sun, 19 Jul 2020 12:17:11 -0700 Subject: [PATCH 280/293] add docstring checks to flake8 currently ignore all errors on a per-file basis --- setup.cfg | 178 +++++++++++++++++++++++++++++++++++++++++++++++++++++- setup.py | 1 + 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3f5fb0c57..c87113538 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,4 +24,180 @@ ignore = FI15, # `__future__` import "generator_stop" missing FI50, # `__future__` import "division" present FI51, # `__future__` import "absolute_import" present - FI53, # `__future__` import "print_function" present \ No newline at end of file + FI53, # `__future__` import "print_function" present +per-file-ignores = + ./beet:D + ./docs/conf.py:D + ./extra/release.py:D + ./beetsplug/duplicates.py:D + ./beetsplug/bpm.py:D + ./beetsplug/convert.py:D + ./beetsplug/info.py:D + ./beetsplug/parentwork.py:D + ./beetsplug/deezer.py:D + ./beetsplug/smartplaylist.py:D + ./beetsplug/absubmit.py:D + ./beetsplug/subsonicupdate.py:D + ./beetsplug/fromfilename.py:D + ./beetsplug/mpdstats.py:D + ./beetsplug/gmusic.py:D + ./beetsplug/subsonicplaylist.py:D + ./beetsplug/rewrite.py:D + ./beetsplug/hook.py:D + ./beetsplug/playlist.py:D + ./beetsplug/ftintitle.py:D + ./beetsplug/bpd/gstplayer.py:D + ./beetsplug/bpd/__init__.py:D + ./beetsplug/scrub.py:D + ./beetsplug/sonosupdate.py:D + ./beetsplug/embyupdate.py:D + ./beetsplug/plexupdate.py:D + ./beetsplug/mbsync.py:D + ./beetsplug/lyrics.py:D + ./beetsplug/inline.py:D + ./beetsplug/freedesktop.py:D + ./beetsplug/acousticbrainz.py:D + ./beetsplug/beatport.py:D + ./beetsplug/cue.py:D + ./beetsplug/thumbnails.py:D + ./beetsplug/random.py:D + ./beetsplug/loadext.py:D + ./beetsplug/replaygain.py:D + ./beetsplug/export.py:D + ./beetsplug/fuzzy.py:D + ./beetsplug/importadded.py:D + ./beetsplug/web/__init__.py:D + ./beetsplug/bucket.py:D + ./beetsplug/the.py:D + ./beetsplug/ihate.py:D + ./beetsplug/bench.py:D + ./beetsplug/permissions.py:D + ./beetsplug/spotify.py:D + ./beetsplug/lastgenre/__init__.py:D + ./beetsplug/mbcollection.py:D + ./beetsplug/metasync/amarok.py:D + ./beetsplug/metasync/itunes.py:D + ./beetsplug/metasync/__init__.py:D + ./beetsplug/importfeeds.py:D + ./beetsplug/kodiupdate.py:D + ./beetsplug/zero.py:D + ./beetsplug/bpsync.py:D + ./beetsplug/__init__.py:D + ./beetsplug/edit.py:D + ./beetsplug/types.py:D + ./beetsplug/embedart.py:D + ./beetsplug/mpdupdate.py:D + ./beetsplug/ipfs.py:D + ./beetsplug/discogs.py:D + ./beetsplug/chroma.py:D + ./beetsplug/fish.py:D + ./beetsplug/missing.py:D + ./beetsplug/fetchart.py:D + ./beetsplug/mbsubmit.py:D + ./beetsplug/filefilter.py:D + ./beetsplug/badfiles.py:D + ./beetsplug/play.py:D + ./beetsplug/keyfinder.py:D + ./beetsplug/unimported.py:D + ./beetsplug/lastimport.py:D + ./test/test_parentwork.py:D + ./test/test_hook.py:D + ./test/test_keyfinder.py:D + ./test/test_util.py:D + ./test/test_plexupdate.py:D + ./test/test_importfeeds.py:D + ./test/test_discogs.py:D + ./test/test_acousticbrainz.py:D + ./test/test_pipeline.py:D + ./test/test_mb.py:D + ./test/test_playlist.py:D + ./test/helper.py:D + ./test/test_player.py:D + ./test/test_template.py:D + ./test/test_web.py:D + ./test/test_replaygain.py:D + ./test/test_hidden.py:D + ./test/test_info.py:D + ./test/test_dbcore.py:D + ./test/test_vfs.py:D + ./test/test_subsonic.py:D + ./test/test_play.py:D + ./test/test_types_plugin.py:D + ./test/test_plugins.py:D + ./test/test_importer.py:D + ./test/test_smartplaylist.py:D + ./test/test_spotify.py:D + ./test/test_metasync.py:D + ./test/test_bucket.py:D + ./test/test_ftintitle.py:D + ./test/lyrics_download_samples.py:D + ./test/test_convert.py:D + ./test/test_mbsubmit.py:D + ./test/testall.py:D + ./test/test_fetchart.py:D + ./test/test_ui_importer.py:D + ./test/test_mbsync.py:D + ./test/test_art.py:D + ./test/test_permissions.py:D + ./test/test_embedart.py:D + ./test/test_the.py:D + ./test/test_export.py:D + ./test/rsrc/beetsplug/test.py:D + ./test/rsrc/convert_stub.py:D + ./test/test_ui_init.py:D + ./test/test_filefilter.py:D + ./test/test_logging.py:D + ./test/test_thumbnails.py:D + ./test/test_ipfs.py:D + ./test/test_autotag.py:D + ./test/__init__.py:D + ./test/test_plugin_mediafield.py:D + ./test/test_files.py:D + ./test/test_lastgenre.py:D + ./test/_common.py:D + ./test/test_zero.py:D + ./test/test_edit.py:D + ./test/test_ihate.py:D + ./test/test_ui.py:D + ./test/test_mpdstats.py:D + ./test/test_importadded.py:D + ./test/test_query.py:D + ./test/test_sort.py:D + ./test/test_library.py:D + ./test/test_ui_commands.py:D + ./test/test_lyrics.py:D + ./test/test_beatport.py:D + ./test/test_random.py:D + ./test/test_embyupdate.py:D + ./test/test_datequery.py:D + ./test/test_config_command.py:D + ./setup.py:D + ./beets/ui/__init__.py:D + ./beets/ui/commands.py:D + ./beets/autotag/mb.py:D + ./beets/autotag/hooks.py:D + ./beets/autotag/__init__.py:D + ./beets/autotag/match.py:D + ./beets/__main__.py:D + ./beets/importer.py:D + ./beets/plugins.py:D + ./beets/util/bluelet.py:D + ./beets/util/enumeration.py:D + ./beets/util/artresizer.py:D + ./beets/util/functemplate.py:D + ./beets/util/confit.py:D + ./beets/util/pipeline.py:D + ./beets/util/hidden.py:D + ./beets/util/__init__.py:D + ./beets/library.py:D + ./beets/random.py:D + ./beets/art.py:D + ./beets/logging.py:D + ./beets/vfs.py:D + ./beets/__init__.py:D + ./beets/dbcore/query.py:D + ./beets/dbcore/db.py:D + ./beets/dbcore/__init__.py:D + ./beets/dbcore/queryparse.py:D + ./beets/dbcore/types.py:D + ./beets/mediafile.py:D diff --git a/setup.py b/setup.py index 0e2cb332a..520fc567f 100755 --- a/setup.py +++ b/setup.py @@ -131,6 +131,7 @@ setup( 'flake8', 'flake8-blind-except', 'flake8-coding', + 'flake8-docstrings', 'flake8-future-import', 'pep8-naming', ], From 02fcbea6767fd4dfdd540f679dd4f0191fae3ef4 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Wed, 22 Jul 2020 17:04:35 -0700 Subject: [PATCH 281/293] add google docstring convention to CONTRIBUTING.rst --- CONTRIBUTING.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e839599d4..d86c490b9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -215,9 +215,13 @@ There are a few coding conventions we use in beets: Style ----- -We follow `PEP 8 `__ for -style. You can use ``tox -e lint`` to check your code for any style -errors. + +We follow `PEP 8`_ and `google's docstring format`_. + +You can use ``tox -e lint`` to check your code for any style errors. + +.. _PEP 8: https://www.python.org/dev/peps/pep-0008/ +.. _google's docstring format: https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings Handling Paths -------------- From 4e9346942136d86b2250b9299bcb6e616a57c41f Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Wed, 29 Jul 2020 20:32:53 -0700 Subject: [PATCH 282/293] fix missing docstring-convention declaration --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index c87113538..761ab1de3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] min-version = 2.7 accept-encodings = utf-8 +docstring-convention = google # errors we ignore; see https://www.flake8rules.com/ for more info ignore = # pycodestyle errors From 48d39ea11ecf17d13b035e9fb975d5ce526575cc Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Thu, 30 Jul 2020 22:42:31 +0200 Subject: [PATCH 283/293] fix regex, add test and changelog entry --- beetsplug/the.py | 2 +- docs/changelog.rst | 3 +++ test/test_the.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/beetsplug/the.py b/beetsplug/the.py index 238aec32f..dfc58817d 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -23,7 +23,7 @@ from beets.plugins import BeetsPlugin __author__ = 'baobab@heresiarch.info' __version__ = '1.1' -PATTERN_THE = u'^[the]{3}\\s' +PATTERN_THE = u'^the\\s' PATTERN_A = u'^[a][n]?\\s' FORMAT = u'{0}, {1}' diff --git a/docs/changelog.rst b/docs/changelog.rst index 64e6ab85c..4a87f7f07 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -145,6 +145,9 @@ New features: Fixes: +* :doc:`/plugins/the`: Fixed incorrect regex for 'the' that matched any + 3-letter combination of the letters t, h, e. + :bug:`3701` * :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take environment variables such as proxy servers into account when making requests :bug:`3450` diff --git a/test/test_the.py b/test/test_the.py index 263446b92..1fc488953 100644 --- a/test/test_the.py +++ b/test/test_the.py @@ -36,6 +36,8 @@ class ThePluginTest(_common.TestCase): u'A Thing, An') self.assertEqual(ThePlugin().unthe(u'the An Arse', PATTERN_A), u'the An Arse') + self.assertEqual(ThePlugin().unthe(u'TET - Travailleur', PATTERN_THE), + u'TET - Travailleur') def test_unthe_with_strip(self): config['the']['strip'] = True From 9291d9c30417d5072d217e0bc4894348de057c11 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 1 Aug 2020 14:35:53 -0700 Subject: [PATCH 284/293] Fix rarfile 4.0 dependency conflict (#3711) --- setup.py | 9 ++++++--- tox.ini | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 520fc567f..2c3cb2b55 100755 --- a/setup.py +++ b/setup.py @@ -120,13 +120,14 @@ setup( 'pytest', 'python-mpd2', 'pyxdg', - 'rarfile', 'responses>=0.3.0', 'requests_oauthlib', ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] - ), + ) + [ + 'rarfile<4' if sys.version_info < (3, 6, 0) else 'rarfile', + ], 'lint': [ 'flake8', 'flake8-blind-except', @@ -152,7 +153,9 @@ setup( 'mpdstats': ['python-mpd2>=0.4.2'], 'plexupdate': ['requests'], 'web': ['flask', 'flask-cors'], - 'import': ['rarfile'], + 'import': ( + ['rarfile<4' if (sys.version_info < (3, 6, 0)) else 'rarfile'] + ), 'thumbnails': ['pyxdg', 'Pillow'] + (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], diff --git a/tox.ini b/tox.ini index e3476db1c..bb7419dbc 100644 --- a/tox.ini +++ b/tox.ini @@ -30,4 +30,4 @@ commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} [testenv:int] deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 -commands = python -bb -m pytest {posargs} \ No newline at end of file +commands = python -bb -m pytest {posargs} From c9f9f9691d1ec7905049bbe76d93eb57e7f8e0a5 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 14:59:21 +0100 Subject: [PATCH 285/293] Notify Zulip chat on integration test failure --- .github/workflows/integration_test.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1e8b3a773..5c023e933 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -25,3 +25,20 @@ jobs: - name: Test with tox run: | tox -e int + + - name: Notify on failure + if: ${{ failure() }} + env: + ZULIP_BOT_CREDENTIALS: ${{ secrets.ZULIP_BOT_CREDENTIALS }} + run: | + if [ -z "${ZULIP_BOT_CREDENTIALS}" ]; then + echo "Skipping notify, ZULIP_BOT_CREDENTIALS is unset" + exit 0 + fi + + curl -X POST https://beets.zulipchat.com/api/v1/messages \ + -u "${ZULIP_BOT_CREDENTIALS}" \ + -d "type=stream" \ + -d "to=general" \ + -d "subject=${GITHUB_WORKFLOW} - $(date -u +%Y-%m-%d)" \ + -d "content=[${GITHUB_WORKFLOW}#${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) failed." From 5f44c3147dc3bcf67af0afc162156b7f3d3d6abd Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 15:37:01 +0100 Subject: [PATCH 286/293] Switch to #github for integration test messages --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 5c023e933..0e5efe793 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -39,6 +39,6 @@ jobs: curl -X POST https://beets.zulipchat.com/api/v1/messages \ -u "${ZULIP_BOT_CREDENTIALS}" \ -d "type=stream" \ - -d "to=general" \ + -d "to=github" \ -d "subject=${GITHUB_WORKFLOW} - $(date -u +%Y-%m-%d)" \ -d "content=[${GITHUB_WORKFLOW}#${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) failed." From c7859ca9c69bfd6fb75462dd512f3b7b4dc09380 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 15:51:07 +0100 Subject: [PATCH 287/293] Add worfklow_dispatch trigger This change allows us to trigger integration tests manually. --- .github/workflows/integration_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1e8b3a773..209be0b93 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -1,5 +1,6 @@ name: integration tests on: + workflow_dispatch: schedule: - cron: '0 0 * * SUN' # run every Sunday at midnight jobs: From 92f425628d4b8659bee42925dd7766b243538676 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 16:55:51 +0100 Subject: [PATCH 288/293] Skip Genius integration test on GitHub actions --- test/test_lyrics.py | 7 ++++++- tox.ini | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 11006348e..e0ec1e548 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -330,7 +330,12 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): """Test default backends with songs known to exist in respective databases. """ errors = [] - for s in self.DEFAULT_SOURCES: + # GitHub actions seems to be on a Cloudflare blacklist, so we can't + # contact genius. + sources = [s for s in self.DEFAULT_SOURCES if + s['backend'] != lyrics.Genius or + os.environ.get('GITHUB_ACTIONS') != 'true'] + for s in sources: res = s['backend'](self.plugin.config, self.plugin._log).fetch( s['artist'], s['title']) if not is_lyrics_content_ok(s['title'], res): diff --git a/tox.ini b/tox.ini index bb7419dbc..cbf953033 100644 --- a/tox.ini +++ b/tox.ini @@ -30,4 +30,5 @@ commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} [testenv:int] deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 +passenv = GITHUB_ACTIONS commands = python -bb -m pytest {posargs} From 20a228c21b7fdaa909343f69019c000f727507b0 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 21:54:20 +0100 Subject: [PATCH 289/293] Add flake8 problem matcher --- .github/flake8-problem-matcher.json | 17 +++++++++++++++++ .github/workflows/ci.yaml | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 .github/flake8-problem-matcher.json diff --git a/.github/flake8-problem-matcher.json b/.github/flake8-problem-matcher.json new file mode 100644 index 000000000..bb8db292c --- /dev/null +++ b/.github/flake8-problem-matcher.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "flake8", + "pattern": [ + { + "regexp": "^(.*?):(\\d+):(\\d+): ([^ ]+) (.*)$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ] + } + ] +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08e7548f2..48d43d059 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,9 +65,6 @@ jobs: lint: runs-on: ubuntu-latest - env: - PY_COLORS: 1 - steps: - uses: actions/checkout@v2 @@ -81,5 +78,8 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx + - name: Add problem matcher + run: echo "::add-matcher::.github/flake8-problem-matcher.json" + - name: Lint with flake8 run: tox -e py-lint From 5d24cb0e1dded59960ea59a3191f21b4f0597ecc Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 22:41:01 +0100 Subject: [PATCH 290/293] Include error code in message --- .github/flake8-problem-matcher.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/flake8-problem-matcher.json b/.github/flake8-problem-matcher.json index bb8db292c..52f94e05e 100644 --- a/.github/flake8-problem-matcher.json +++ b/.github/flake8-problem-matcher.json @@ -4,12 +4,11 @@ "owner": "flake8", "pattern": [ { - "regexp": "^(.*?):(\\d+):(\\d+): ([^ ]+) (.*)$", + "regexp": "^(.*?):(\\d+):(\\d+): (.*)$", "file": 1, "line": 2, "column": 3, - "code": 4, - "message": 5 + "message": 4 } ] } From 490fc0751600c02659dd2c11c2f174b8bdec0ad2 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 3 Aug 2020 13:10:21 +0100 Subject: [PATCH 291/293] Add sphinx problem matcher --- .github/sphinx-problem-matcher.json | 18 ++++++++++++++++++ .github/workflows/ci.yaml | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 .github/sphinx-problem-matcher.json diff --git a/.github/sphinx-problem-matcher.json b/.github/sphinx-problem-matcher.json new file mode 100644 index 000000000..d33eb09b6 --- /dev/null +++ b/.github/sphinx-problem-matcher.json @@ -0,0 +1,18 @@ +{ + "problemMatcher": [ + { + "owner": "sphinx", + "pattern": [ + { + "regexp": "^Warning, treated as error:$" + }, + { + "regexp": "^(.*?):(\\d+):(.*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48d43d059..ecb7e03dd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,6 +59,9 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx + - name: Add problem matcher + run: echo "::add-matcher::.github/sphinx-problem-matcher.json" + - name: Build and check docs using tox run: tox -e docs From b89a2650cc8728d21a8aaa48b82e9b24564e44b0 Mon Sep 17 00:00:00 2001 From: Samuel Cook Date: Mon, 3 Aug 2020 18:20:20 -0700 Subject: [PATCH 292/293] Delete after convert (#3700) * If import move is true, files will be deleted after converting. Fixes #2947 * Removed trailing whitespace to comply with W293, fixing build * Add period to the end of the comment Co-Authored-By: Adrian Sampson * Added changelog entry for this fix. * Added delete_originals option to remove source files after transcode * Added unit test, removed redundant syspath call Co-authored-by: Logan Arens Co-authored-by: Logan Arens Co-authored-by: Adrian Sampson Co-authored-by: Logan Arens --- beetsplug/convert.py | 6 ++++++ docs/changelog.rst | 4 ++++ docs/plugins/convert.rst | 2 ++ test/test_convert.py | 10 ++++++++++ 4 files changed, 22 insertions(+) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index e7ac4f3ac..70363f6eb 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -148,6 +148,7 @@ class ConvertPlugin(BeetsPlugin): u'never_convert_lossy_files': False, u'copy_album_art': False, u'album_art_maxwidth': 0, + u'delete_originals': False, }) self.early_import_stages = [self.auto_convert] @@ -532,11 +533,16 @@ class ConvertPlugin(BeetsPlugin): # Change the newly-imported database entry to point to the # converted file. + source_path = item.path item.path = dest item.write() item.read() # Load new audio information data. item.store() + if self.config['delete_originals']: + self._log.info(u'Removing original file {0}', source_path) + util.remove(source_path, False) + def _cleanup(self, task, session): for path in task.old_paths: if path in _temp_files: diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a87f7f07..0dfa2c5f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -142,6 +142,10 @@ New features: * :doc:`/plugins/thumbnails`: Fix a bug where pathlib expected a string instead of bytes for a path. :bug:`3360` +* :doc:`/plugins/convert`: If ``delete_originals`` is enabled, then the source files will + be deleted after importing. + Thanks to :user:`logan-arens`. + :bug:`2947` Fixes: diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 6e9d00a11..9581e24a4 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -111,6 +111,8 @@ file. The available options are: This option overrides ``link``. Only works when converting to a directory on the same filesystem as the library. Default: ``false``. +- **delete_originals**: Transcoded files will be copied or moved to their destination, depending on the import configuration. By default, the original files are not modified by the plugin. This option deletes the original files after the transcoding step has completed. + Default: ``false``. You can also configure the format to use for transcoding (see the next section): diff --git a/test/test_convert.py b/test/test_convert.py index 33bdb3b24..b8cd56741 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -15,6 +15,7 @@ from __future__ import division, absolute_import, print_function +import fnmatch import sys import re import os.path @@ -121,6 +122,15 @@ class ImportConvertTest(unittest.TestCase, TestHelper): self.assertIsNotNone(item) self.assertTrue(os.path.isfile(item.path)) + def test_delete_originals(self): + self.config['convert']['delete_originals'] = True + self.importer.run() + for path in self.importer.paths: + for root, dirnames, filenames in os.walk(path): + self.assertTrue(len(fnmatch.filter(filenames, '*.mp3')) == 0, + u'Non-empty import directory {0}' + .format(util.displayable_path(path))) + class ConvertCommand(object): """A mixin providing a utility method to run the `convert`command From fce27e6fa950d1846ccc9ba99640d61c98299f62 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Sun, 9 Aug 2020 19:12:58 -0700 Subject: [PATCH 293/293] mpdstats: Don't record a skip when stopping MPD. MPD keeps the current track in the queue when stopping, so it's not really like a skip, and I use it so that I can stop the music, and later start at the beginning of a track. I do this by keeping track of the current song id, and then comparing them when we receive a stop signal. --- beetsplug/mpdstats.py | 14 +++++++++----- docs/changelog.rst | 4 ++++ test/test_mpdstats.py | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index f232d87e9..39b045f9b 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -108,8 +108,9 @@ class MPDClientWrapper(object): return self.get(command, retries=retries - 1) def currentsong(self): - """Return the path to the currently playing song. Prefixes paths with the - music_directory, to get the absolute path. + """Return the path to the currently playing song, along with its + songid. Prefixes paths with the music_directory, to get the absolute + path. """ result = None entry = self.get('currentsong') @@ -118,7 +119,7 @@ class MPDClientWrapper(object): result = os.path.join(self.music_directory, entry['file']) else: result = entry['file'] - return result + return result, entry.get('id') def status(self): """Return the current status of the MPD. @@ -240,7 +241,9 @@ class MPDStats(object): def on_stop(self, status): self._log.info(u'stop') - if self.now_playing: + # if the current song stays the same it means that we stopped on the + # current track and should not record a skip. + if self.now_playing and self.now_playing['id'] != status.get('songid'): self.handle_song_change(self.now_playing) self.now_playing = None @@ -251,7 +254,7 @@ class MPDStats(object): def on_play(self, status): - path = self.mpd.currentsong() + path, songid = self.mpd.currentsong() if not path: return @@ -286,6 +289,7 @@ class MPDStats(object): 'started': time.time(), 'remaining': remaining, 'path': path, + 'id': songid, 'beets_item': self.get_item(path), } diff --git a/docs/changelog.rst b/docs/changelog.rst index 0dfa2c5f8..47159ddc2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -245,6 +245,10 @@ Fixes: * Fix a bug that caused metadata starting with something resembling a drive letter to be incorrectly split into an extra directory after the colon. :bug:`3685` +* :doc:`/plugins/mpdstats`: Don't record a skip when stopping MPD, as MPD keeps + the current track in the queue. + Thanks to :user:`aereaux`. + :bug:`3722` For plugin developers: diff --git a/test/test_mpdstats.py b/test/test_mpdstats.py index 0117e22aa..20226927f 100644 --- a/test/test_mpdstats.py +++ b/test/test_mpdstats.py @@ -62,10 +62,11 @@ class MPDStatsTest(unittest.TestCase, TestHelper): {'state': u'stop'}] EVENTS = [["player"]] * (len(STATUSES) - 1) + [KeyboardInterrupt] item_path = util.normpath('/foo/bar.flac') + songid = 1 @patch("beetsplug.mpdstats.MPDClientWrapper", return_value=Mock(**{ "events.side_effect": EVENTS, "status.side_effect": STATUSES, - "currentsong.return_value": item_path})) + "currentsong.return_value": (item_path, songid)})) def test_run_mpdstats(self, mpd_mock): item = Item(title=u'title', path=self.item_path, id=1) item.add(self.lib)