Merge branch 'master' into subsonicplaylist

.travis.yml

@@ -13,10 +13,10 @@ matrix:
           env: {TOX_ENV: py27-cov, COVERAGE: 1}
         - python: 2.7.13
           env: {TOX_ENV: py27-test}
-        - python: 3.4
-          env: {TOX_ENV: py34-test}
-        - python: 3.4_with_system_site_packages
-          env: {TOX_ENV: py34-test}
+        # - python: 3.4
+        #   env: {TOX_ENV: py34-test}
+        # - python: 3.4_with_system_site_packages
+        #   env: {TOX_ENV: py34-test}
         - python: 3.5
           env: {TOX_ENV: py35-test}
         - python: 3.6
@@ -24,9 +24,9 @@ matrix:
         - python: 3.7
           env: {TOX_ENV: py37-test}
           dist: xenial
-        # - python: 3.8-dev
-        #   env: {TOX_ENV: py38-test}
-        #   dist: xenial
+        - python: 3.8
+          env: {TOX_ENV: py38-test}
+          dist: xenial
         # - python: pypy
         #   - env: {TOX_ENV: pypy-test}
         - python: 3.6

beets/__init__.py

@@ -15,9 +15,8 @@
 
 from __future__ import division, absolute_import, print_function
 
-import os
-
 import confuse
+from sys import stderr
 
 __version__ = u'1.5.0'
 __author__ = u'Adrian Sampson <adrian@radbox.org>'
@@ -32,11 +31,12 @@ class IncludeLazyConfig(confuse.LazyConfig):
 
         try:
             for view in self['include']:
-                filename = view.as_filename()
-                if os.path.isfile(filename):
-                    self.set_file(filename)
+                self.set_file(view.as_filename())
         except confuse.NotFoundError:
             pass
+        except confuse.ConfigReadError as err:
+            stderr.write("configuration `import` failed: {}"
+                         .format(err.reason))
 
 
 config = IncludeLazyConfig('beets', __name__)

beets/art.py

@@ -51,8 +51,8 @@ def get_art(log, item):
 
 
 def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
-               compare_threshold=0, ifempty=False, as_album=False,
-               id3v23=None):
+               compare_threshold=0, ifempty=False, as_album=False, id3v23=None,
+               quality=0):
     """Embed an image into the item's media file.
     """
     # Conditions and filters.
@@ -64,7 +64,7 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
         log.info(u'media file already contained art')
         return
     if maxwidth and not as_album:
-        imagepath = resize_image(log, imagepath, maxwidth)
+        imagepath = resize_image(log, imagepath, maxwidth, quality)
 
     # Get the `Image` object from the file.
     try:
@@ -84,8 +84,8 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
     item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23)
 
 
-def embed_album(log, album, maxwidth=None, quiet=False,
-                compare_threshold=0, ifempty=False):
+def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0,
+                ifempty=False, quality=0):
     """Embed album art into all of the album's items.
     """
     imagepath = album.artpath
@@ -97,20 +97,23 @@ def embed_album(log, album, maxwidth=None, quiet=False,
                  displayable_path(imagepath), album)
         return
     if maxwidth:
-        imagepath = resize_image(log, imagepath, maxwidth)
+        imagepath = resize_image(log, imagepath, maxwidth, quality)
 
     log.info(u'Embedding album art into {0}', album)
 
     for item in album.items():
-        embed_item(log, item, imagepath, maxwidth, None,
-                   compare_threshold, ifempty, as_album=True)
+        embed_item(log, item, imagepath, maxwidth, None, compare_threshold,
+                   ifempty, as_album=True, quality=quality)
 
 
-def resize_image(log, imagepath, maxwidth):
-    """Returns path to an image resized to maxwidth.
+def resize_image(log, imagepath, maxwidth, quality):
+    """Returns path to an image resized to maxwidth and encoded with the
+    specified quality level.
     """
-    log.debug(u'Resizing album art to {0} pixels wide', maxwidth)
-    imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath))
+    log.debug(u'Resizing album art to {0} pixels wide and encoding at quality \
+              level {1}', maxwidth, quality)
+    imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath),
+                                         quality=quality)
     return imagepath
 
 

beets/autotag/hooks.py

@@ -614,17 +614,21 @@ def tracks_for_id(track_id):
 
 
 @plugins.notify_info_yielded(u'albuminfo_received')
-def album_candidates(items, artist, album, va_likely):
+def album_candidates(items, artist, album, va_likely, extra_tags):
     """Search for album matches. ``items`` is a list of Item objects
     that make up the album. ``artist`` and ``album`` are the respective
     names (strings), which may be derived from the item list or may be
     entered by the user. ``va_likely`` is a boolean indicating whether
-    the album is likely to be a "various artists" release.
+    the album is likely to be a "various artists" release. ``extra_tags``
+    is an optional dictionary of additional tags used to further
+    constrain the search.
     """
+
     # Base candidates if we have album and artist to match.
     if artist and album:
         try:
-            for candidate in mb.match_album(artist, album, len(items)):
+            for candidate in mb.match_album(artist, album, len(items),
+                                            extra_tags):
                 yield candidate
         except mb.MusicBrainzAPIError as exc:
             exc.log(log)
@@ -632,13 +636,15 @@ def album_candidates(items, artist, album, va_likely):
     # Also add VA matches from MusicBrainz where appropriate.
     if va_likely and album:
         try:
-            for candidate in mb.match_album(None, album, len(items)):
+            for candidate in mb.match_album(None, album, len(items),
+                                            extra_tags):
                 yield candidate
         except mb.MusicBrainzAPIError as exc:
             exc.log(log)
 
     # Candidates from plugins.
-    for candidate in plugins.candidates(items, artist, album, va_likely):
+    for candidate in plugins.candidates(items, artist, album, va_likely,
+                                        extra_tags):
         yield candidate
 
 

beets/autotag/match.py

@@ -447,6 +447,12 @@ def tag_album(items, search_artist=None, search_album=None,
             search_artist, search_album = cur_artist, cur_album
         log.debug(u'Search terms: {0} - {1}', search_artist, search_album)
 
+        extra_tags = None
+        if config['musicbrainz']['extra_tags']:
+            tag_list = config['musicbrainz']['extra_tags'].get()
+            extra_tags = {k: v for (k, v) in likelies.items() if k in tag_list}
+            log.debug(u'Additional search terms: {0}', extra_tags)
+
         # Is this album likely to be a "various artist" release?
         va_likely = ((not consensus['artist']) or
                      (search_artist.lower() in VA_ARTISTS) or
@@ -457,7 +463,8 @@ def tag_album(items, search_artist=None, search_album=None,
         for matched_candidate in hooks.album_candidates(items,
                                                         search_artist,
                                                         search_album,
-                                                        va_likely):
+                                                        va_likely,
+                                                        extra_tags):
             _add_candidate(items, candidates, matched_candidate)
 
     log.debug(u'Evaluating {0} candidates.', len(candidates))

beets/autotag/mb.py

@@ -38,6 +38,14 @@ else:
 
 SKIPPED_TRACKS = ['[data track]']
 
+FIELDS_TO_MB_KEYS = {
+    'catalognum': 'catno',
+    'country': 'country',
+    'label': 'label',
+    'media': 'format',
+    'year': 'date',
+}
+
 musicbrainzngs.set_useragent('beets', beets.__version__,
                              'https://beets.io/')
 
@@ -411,13 +419,13 @@ def album_info(release):
     return info
 
 
-def match_album(artist, album, tracks=None):
+def match_album(artist, album, tracks=None, extra_tags=None):
     """Searches for a single album ("release" in MusicBrainz parlance)
     and returns an iterator over AlbumInfo objects. May raise a
     MusicBrainzAPIError.
 
     The query consists of an artist name, an album name, and,
-    optionally, a number of tracks on the album.
+    optionally, a number of tracks on the album and any other extra tags.
     """
     # Build search criteria.
     criteria = {'release': album.lower().strip()}
@@ -429,6 +437,16 @@ def match_album(artist, album, tracks=None):
     if tracks is not None:
         criteria['tracks'] = six.text_type(tracks)
 
+    # Additional search cues from existing metadata.
+    if extra_tags:
+        for tag in extra_tags:
+            key = FIELDS_TO_MB_KEYS[tag]
+            value = six.text_type(extra_tags.get(tag, '')).lower().strip()
+            if key == 'catno':
+                value = value.replace(u' ', '')
+            if value:
+                criteria[key] = value
+
     # Abort if we have no search terms.
     if not any(criteria.values()):
         return

beets/config_default.yaml

@@ -103,6 +103,7 @@ musicbrainz:
     ratelimit: 1
     ratelimit_interval: 1.0
     searchlimit: 5
+    extra_tags: []
 
 match:
     strong_rec_thresh: 0.04

beets/dbcore/db.py

@@ -189,7 +189,7 @@ class LazyConvertDict(object):
 
 class Model(object):
     """An abstract object representing an object in the database. Model
-    objects act like dictionaries (i.e., the allow subscript access like
+    objects act like dictionaries (i.e., they allow subscript access like
     ``obj['field']``). The same field set is available via attribute
     access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are
     available:

beets/dbcore/query.py

@@ -156,12 +156,8 @@ class NoneQuery(FieldQuery):
     def col_clause(self):
         return self.field + " IS NULL", ()
 
-    @classmethod
-    def match(cls, item):
-        try:
-            return item[cls.field] is None
-        except KeyError:
-            return True
+    def match(self, item):
+        return item.get(self.field) is None
 
     def __repr__(self):
         return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self)

beets/dbcore/types.py

@@ -131,6 +131,14 @@ class Integer(Type):
     query = query.NumericQuery
     model_type = int
 
+    def normalize(self, value):
+        try:
+            return self.model_type(round(float(value)))
+        except ValueError:
+            return self.null
+        except TypeError:
+            return self.null
+
 
 class PaddedInt(Integer):
     """An integer field that is formatted with a given number of digits,

beets/plugins.py

@@ -172,7 +172,7 @@ class BeetsPlugin(object):
         """
         return beets.autotag.hooks.Distance()
 
-    def candidates(self, items, artist, album, va_likely):
+    def candidates(self, items, artist, album, va_likely, extra_tags=None):
         """Should return a sequence of AlbumInfo objects that match the
         album whose items are provided.
         """
@@ -379,11 +379,12 @@ def album_distance(items, album_info, mapping):
     return dist
 
 
-def candidates(items, artist, album, va_likely):
+def candidates(items, artist, album, va_likely, extra_tags=None):
     """Gets MusicBrainz candidates for an album from each plugin.
     """
     for plugin in find_plugins():
-        for candidate in plugin.candidates(items, artist, album, va_likely):
+        for candidate in plugin.candidates(items, artist, album, va_likely,
+                                           extra_tags):
             yield candidate
 
 
@@ -714,7 +715,7 @@ class MetadataSourcePlugin(object):
                 return id_
         return None
 
-    def candidates(self, items, artist, album, va_likely):
+    def candidates(self, items, artist, album, va_likely, extra_tags=None):
         """Returns a list of AlbumInfo objects for Search API results
         matching an ``album`` and ``artist`` (if not various).
 

beets/util/artresizer.py

@@ -40,14 +40,19 @@ else:
 log = logging.getLogger('beets')
 
 
-def resize_url(url, maxwidth):
+def resize_url(url, maxwidth, quality=0):
     """Return a proxied image URL that resizes the original image to
     maxwidth (preserving aspect ratio).
     """
-    return '{0}?{1}'.format(PROXY_URL, urlencode({
+    params = {
         'url': url.replace('http://', ''),
         'w': maxwidth,
-    }))
+    }
+
+    if quality > 0:
+        params['q'] = quality
+
+    return '{0}?{1}'.format(PROXY_URL, urlencode(params))
 
 
 def temp_file_for(path):
@@ -59,7 +64,7 @@ def temp_file_for(path):
         return util.bytestring_path(f.name)
 
 
-def pil_resize(maxwidth, path_in, path_out=None):
+def pil_resize(maxwidth, path_in, path_out=None, quality=0):
     """Resize using Python Imaging Library (PIL).  Return the output path
     of resized image.
     """
@@ -72,7 +77,7 @@ def pil_resize(maxwidth, path_in, path_out=None):
         im = Image.open(util.syspath(path_in))
         size = maxwidth, maxwidth
         im.thumbnail(size, Image.ANTIALIAS)
-        im.save(util.py3_path(path_out))
+        im.save(util.py3_path(path_out), quality=quality)
         return path_out
     except IOError:
         log.error(u"PIL cannot create thumbnail for '{0}'",
@@ -80,7 +85,7 @@ def pil_resize(maxwidth, path_in, path_out=None):
         return path_in
 
 
-def im_resize(maxwidth, path_in, path_out=None):
+def im_resize(maxwidth, path_in, path_out=None, quality=0):
     """Resize using ImageMagick.
 
     Use the ``magick`` program or ``convert`` on older versions. Return
@@ -93,10 +98,15 @@ def im_resize(maxwidth, path_in, path_out=None):
     # "-resize WIDTHx>" shrinks images with the width larger
     # than the given width while maintaining the aspect ratio
     # with regards to the height.
-    cmd = ArtResizer.shared.im_convert_cmd + \
-        [util.syspath(path_in, prefix=False),
-            '-resize', '{0}x>'.format(maxwidth),
-            util.syspath(path_out, prefix=False)]
+    cmd = ArtResizer.shared.im_convert_cmd + [
+        util.syspath(path_in, prefix=False),
+        '-resize', '{0}x>'.format(maxwidth),
+    ]
+
+    if quality > 0:
+        cmd += ['-quality', '{0}'.format(quality)]
+
+    cmd.append(util.syspath(path_out, prefix=False))
 
     try:
         util.command_output(cmd)
@@ -190,18 +200,19 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
                 self.im_convert_cmd = ['magick']
                 self.im_identify_cmd = ['magick', 'identify']
 
-    def resize(self, maxwidth, path_in, path_out=None):
+    def resize(self, maxwidth, path_in, path_out=None, quality=0):
         """Manipulate an image file according to the method, returning a
         new path. For PIL or IMAGEMAGIC methods, resizes the image to a
-        temporary file. For WEBPROXY, returns `path_in` unmodified.
+        temporary file and encodes with the specified quality level.
+        For WEBPROXY, returns `path_in` unmodified.
         """
         if self.local:
             func = BACKEND_FUNCS[self.method[0]]
-            return func(maxwidth, path_in, path_out)
+            return func(maxwidth, path_in, path_out, quality=quality)
         else:
             return path_in
 
-    def proxy_url(self, maxwidth, url):
+    def proxy_url(self, maxwidth, url, quality=0):
         """Modifies an image URL according the method, returning a new
         URL. For WEBPROXY, a URL on the proxy server is returned.
         Otherwise, the URL is returned unmodified.
@@ -209,7 +220,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
         if self.local:
             return url
         else:
-            return resize_url(url, maxwidth)
+            return resize_url(url, maxwidth, quality)
 
     @property
     def local(self):

beetsplug/beatport.py

@@ -355,7 +355,7 @@ class BeatportPlugin(BeetsPlugin):
             config=self.config
         )
 
-    def candidates(self, items, artist, release, va_likely):
+    def candidates(self, items, artist, release, va_likely, extra_tags=None):
         """Returns a list of AlbumInfo objects for beatport search results
         matching release and artist (if not various).
         """

beetsplug/bpd/__init__.py

@@ -639,6 +639,8 @@ class BaseServer(object):
             self.playlist.pop(old_index)
             if self.current_index > old_index:
                 self.current_index -= 1
+            self.playlist_version += 1
+            self._send_event("playlist")
         if self.current_index >= len(self.playlist):
             # Fallen off the end. Move to stopped state or loop.
             if self.repeat:

beetsplug/chroma.py

@@ -191,7 +191,7 @@ class AcoustidPlugin(plugins.BeetsPlugin):
         dist.add_expr('track_id', info.track_id not in recording_ids)
         return dist
 
-    def candidates(self, items, artist, album, va_likely):
+    def candidates(self, items, artist, album, va_likely, extra_tags=None):
         albums = []
         for relid in prefix(_all_releases(items), MAX_RELEASES):
             album = hooks.album_for_mbid(relid)

beetsplug/cue.py

@@ -24,7 +24,7 @@ class CuePlugin(BeetsPlugin):
 
         # self.register_listener('import_task_start', self.look_for_cues)
 
-    def candidates(self, items, artist, album, va_likely):
+    def candidates(self, items, artist, album, va_likely, extra_tags=None):
         import pdb
         pdb.set_trace()
 

beetsplug/discogs.py

@@ -57,7 +57,8 @@ class DiscogsPlugin(BeetsPlugin):
             'tokenfile': 'discogs_token.json',
             'source_weight': 0.5,
             'user_token': '',
-            'separator': u', '
+            'separator': u', ',
+            'index_tracks': False,
         })
         self.config['apikey'].redact = True
         self.config['apisecret'].redact = True
@@ -174,7 +175,7 @@ class DiscogsPlugin(BeetsPlugin):
             config=self.config
         )
 
-    def candidates(self, items, artist, album, va_likely):
+    def candidates(self, items, artist, album, va_likely, extra_tags=None):
         """Returns a list of AlbumInfo objects for discogs search results
         matching an album and artist (if not various).
         """
@@ -397,14 +398,28 @@ class DiscogsPlugin(BeetsPlugin):
         tracks = []
         index_tracks = {}
         index = 0
+        # Distinct works and intra-work divisions, as defined by index tracks.
+        divisions, next_divisions = [], []
         for track in clean_tracklist:
             # Only real tracks have `position`. Otherwise, it's an index track.
             if track['position']:
                 index += 1
-                track_info = self.get_track_info(track, index)
+                if next_divisions:
+                    # End of a block of index tracks: update the current
+                    # divisions.
+                    divisions += next_divisions
+                    del next_divisions[:]
+                track_info = self.get_track_info(track, index, divisions)
                 track_info.track_alt = track['position']
                 tracks.append(track_info)
             else:
+                next_divisions.append(track['title'])
+                # We expect new levels of division at the beginning of the
+                # tracklist (and possibly elsewhere).
+                try:
+                    divisions.pop()
+                except IndexError:
+                    pass
                 index_tracks[index + 1] = track['title']
 
         # Fix up medium and medium_index for each track. Discogs position is
@@ -539,10 +554,13 @@ class DiscogsPlugin(BeetsPlugin):
 
         return tracklist
 
-    def get_track_info(self, track, index):
+    def get_track_info(self, track, index, divisions):
         """Returns a TrackInfo object for a discogs track.
         """
         title = track['title']
+        if self.config['index_tracks']:
+            prefix = ', '.join(divisions)
+            title = ': '.join([prefix, title])
         track_id = None
         medium, medium_index, _ = self.get_track_index(track['position'])
         artist, artist_id = MetadataSourcePlugin.get_artist(

beetsplug/embedart.py

@@ -59,7 +59,8 @@ class EmbedCoverArtPlugin(BeetsPlugin):
             'auto': True,
             'compare_threshold': 0,
             'ifempty': False,
-            'remove_art_file': False
+            'remove_art_file': False,
+            'quality': 0,
         })
 
         if self.config['maxwidth'].get(int) and not ArtResizer.shared.local:
@@ -86,6 +87,7 @@ class EmbedCoverArtPlugin(BeetsPlugin):
             u"-y", u"--yes", action="store_true", help=u"skip confirmation"
         )
         maxwidth = self.config['maxwidth'].get(int)
+        quality = self.config['quality'].get(int)
         compare_threshold = self.config['compare_threshold'].get(int)
         ifempty = self.config['ifempty'].get(bool)
 
@@ -104,8 +106,9 @@ class EmbedCoverArtPlugin(BeetsPlugin):
                     return
 
                 for item in items:
-                    art.embed_item(self._log, item, imagepath, maxwidth, None,
-                                   compare_threshold, ifempty)
+                    art.embed_item(self._log, item, imagepath, maxwidth,
+                                   None, compare_threshold, ifempty,
+                                   quality=quality)
             else:
                 albums = lib.albums(decargs(args))
 
@@ -114,8 +117,9 @@ class EmbedCoverArtPlugin(BeetsPlugin):
                     return
 
                 for album in albums:
-                    art.embed_album(self._log, album, maxwidth, False,
-                                    compare_threshold, ifempty)
+                    art.embed_album(self._log, album, maxwidth,
+                                    False, compare_threshold, ifempty,
+                                    quality=quality)
                     self.remove_artfile(album)
 
         embed_cmd.func = embed_func

beetsplug/fetchart.py

@@ -21,6 +21,7 @@ from contextlib import closing
 import os
 import re
 from tempfile import NamedTemporaryFile
+from collections import OrderedDict
 
 import requests
 
@@ -135,7 +136,8 @@ class Candidate(object):
 
     def resize(self, plugin):
         if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE:
-            self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path)
+            self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path,
+                                                 quality=plugin.quality)
 
 
 def _logged_get(log, *args, **kwargs):
@@ -164,9 +166,14 @@ def _logged_get(log, *args, **kwargs):
         message = 'getting URL'
 
     req = requests.Request('GET', *args, **req_kwargs)
+
     with requests.Session() as s:
         s.headers = {'User-Agent': 'beets'}
         prepped = s.prepare_request(req)
+        settings = s.merge_environment_settings(
+            prepped.url, {}, None, None, None
+        )
+        send_kwargs.update(settings)
         log.debug('{}: {}', message, prepped.url)
         return s.send(prepped, **send_kwargs)
 
@@ -505,12 +512,18 @@ class ITunesStore(RemoteArtSource):
                             payload['term'])
             return
 
+        if self._config['high_resolution']:
+            image_suffix = '100000x100000-999'
+        else:
+            image_suffix = '1200x1200bb'
+
         for c in candidates:
             try:
                 if (c['artistName'] == album.albumartist
                         and c['collectionName'] == album.album):
                     art_url = c['artworkUrl100']
-                    art_url = art_url.replace('100x100', '1200x1200')
+                    art_url = art_url.replace('100x100bb',
+                                              image_suffix)
                     yield self._candidate(url=art_url,
                                           match=Candidate.MATCH_EXACT)
             except KeyError as e:
@@ -520,7 +533,8 @@ class ITunesStore(RemoteArtSource):
 
         try:
             fallback_art_url = candidates[0]['artworkUrl100']
-            fallback_art_url = fallback_art_url.replace('100x100', '1200x1200')
+            fallback_art_url = fallback_art_url.replace('100x100bb',
+                                                        image_suffix)
             yield self._candidate(url=fallback_art_url,
                                   match=Candidate.MATCH_FALLBACK)
         except KeyError as e:
@@ -729,11 +743,72 @@ class FileSystem(LocalArtSource):
                                       match=Candidate.MATCH_FALLBACK)
 
 
+class LastFM(RemoteArtSource):
+    NAME = u"Last.fm"
+
+    # Sizes in priority order.
+    SIZES = OrderedDict([
+        ('mega', (300, 300)),
+        ('extralarge', (300, 300)),
+        ('large', (174, 174)),
+        ('medium', (64, 64)),
+        ('small', (34, 34)),
+    ])
+
+    if util.SNI_SUPPORTED:
+        API_URL = 'https://ws.audioscrobbler.com/2.0'
+    else:
+        API_URL = 'http://ws.audioscrobbler.com/2.0'
+
+    def __init__(self, *args, **kwargs):
+        super(LastFM, self).__init__(*args, **kwargs)
+        self.key = self._config['lastfm_key'].get(),
+
+    def get(self, album, plugin, paths):
+        if not album.mb_albumid:
+            return
+
+        try:
+            response = self.request(self.API_URL, params={
+                'method': 'album.getinfo',
+                'api_key': self.key,
+                'mbid': album.mb_albumid,
+                'format': 'json',
+            })
+        except requests.RequestException:
+            self._log.debug(u'lastfm: error receiving response')
+            return
+
+        try:
+            data = response.json()
+
+            if 'error' in data:
+                if data['error'] == 6:
+                    self._log.debug('lastfm: no results for {}',
+                                    album.mb_albumid)
+                else:
+                    self._log.error(
+                        'lastfm: failed to get album info: {} ({})',
+                        data['message'], data['error'])
+            else:
+                images = {image['size']: image['#text']
+                          for image in data['album']['image']}
+
+                # Provide candidates in order of size.
+                for size in self.SIZES.keys():
+                    if size in images:
+                        yield self._candidate(url=images[size],
+                                              size=self.SIZES[size])
+        except ValueError:
+            self._log.debug(u'lastfm: error loading response: {}'
+                            .format(response.text))
+            return
+
 # Try each source in turn.
 
 SOURCES_ALL = [u'filesystem',
                u'coverart', u'itunes', u'amazon', u'albumart',
-               u'wikipedia', u'google', u'fanarttv']
+               u'wikipedia', u'google', u'fanarttv', u'lastfm']
 
 ART_SOURCES = {
     u'filesystem': FileSystem,
@@ -744,6 +819,7 @@ ART_SOURCES = {
     u'wikipedia': Wikipedia,
     u'google': GoogleImages,
     u'fanarttv': FanartTV,
+    u'lastfm': LastFM,
 }
 SOURCE_NAMES = {v: k for k, v in ART_SOURCES.items()}
 
@@ -765,6 +841,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin):
             'auto': True,
             'minwidth': 0,
             'maxwidth': 0,
+            'quality': 0,
             'enforce_ratio': False,
             'cautious': False,
             'cover_names': ['cover', 'front', 'art', 'album', 'folder'],
@@ -773,13 +850,17 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin):
             'google_key': None,
             'google_engine': u'001442825323518660753:hrh5ch1gjzm',
             'fanarttv_key': None,
+            'lastfm_key': None,
             'store_source': False,
+            'high_resolution': False,
         })
         self.config['google_key'].redact = True
         self.config['fanarttv_key'].redact = True
+        self.config['lastfm_key'].redact = True
 
         self.minwidth = self.config['minwidth'].get(int)
         self.maxwidth = self.config['maxwidth'].get(int)
+        self.quality = self.config['quality'].get(int)
 
         # allow both pixel and percentage-based margin specifications
         self.enforce_ratio = self.config['enforce_ratio'].get(
@@ -815,6 +896,9 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin):
         if not self.config['google_key'].get() and \
                 u'google' in available_sources:
             available_sources.remove(u'google')
+        if not self.config['lastfm_key'].get() and \
+                u'lastfm' in available_sources:
+            available_sources.remove(u'lastfm')
         available_sources = [(s, c)
                              for s in available_sources
                              for c in ART_SOURCES[s].VALID_MATCHING_CRITERIA]
@@ -895,7 +979,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin):
         cmd.parser.add_option(
             u'-q', u'--quiet', dest='quiet',
             action='store_true', default=False,
-            help=u'shows only quiet art'
+            help=u'quiet mode: do not output albums that already have artwork'
         )
 
         def func(lib, opts, args):
@@ -909,9 +993,10 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin):
     def art_for_album(self, album, paths, local_only=False):
         """Given an Album object, returns a path to downloaded art for the
         album (or None if no art is found). If `maxwidth`, then images are
-        resized to this maximum pixel size. If `local_only`, then only local
-        image files from the filesystem are returned; no network requests
-        are made.
+        resized to this maximum pixel size. If `quality` then resized images
+        are saved at the specified quality level. If `local_only`, then only
+        local image files from the filesystem are returned; no network
+        requests are made.
         """
         out = None
 

beetsplug/fish.py

@@ -0,0 +1,276 @@
+# -*- coding: utf-8 -*-
+# This file is part of beets.
+# Copyright 2015, winters jean-marie.
+# Copyright 2020, Justin Mayer <https://justinmayer.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""This plugin generates tab completions for Beets commands for the Fish shell
+<https://fishshell.com/>, including completions for Beets commands, plugin
+commands, and option flags. Also generated are completions for all the album
+and track fields, suggesting for example `genre:` or `album:` when querying the
+Beets database. Completions for the *values* of those fields are not generated
+by default but can be added via the `-e` / `--extravalues` flag. For example:
+`beet fish -e genre -e albumartist`
+"""
+
+from __future__ import division, absolute_import, print_function
+
+from beets.plugins import BeetsPlugin
+from beets import library, ui
+from beets.ui import commands
+from operator import attrgetter
+import os
+BL_NEED2 = """complete -c beet -n '__fish_beet_needs_command' {} {}\n"""
+BL_USE3 = """complete -c beet -n '__fish_beet_using_command {}' {} {}\n"""
+BL_SUBS = """complete -c beet -n '__fish_at_level {} ""' {}  {}\n"""
+BL_EXTRA3 = """complete -c beet -n '__fish_beet_use_extra {}' {} {}\n"""
+
+HEAD = '''
+function __fish_beet_needs_command
+    set cmd (commandline -opc)
+    if test (count $cmd) -eq 1
+        return 0
+    end
+    return 1
+end
+
+function __fish_beet_using_command
+    set cmd (commandline -opc)
+    set needle (count $cmd)
+    if test $needle -gt 1
+        if begin test $argv[1] = $cmd[2];
+            and not contains -- $cmd[$needle] $FIELDS; end
+                return 0
+        end
+    end
+    return 1
+end
+
+function __fish_beet_use_extra
+    set cmd (commandline -opc)
+    set needle (count $cmd)
+    if test $argv[2]  = $cmd[$needle]
+        return 0
+    end
+    return 1
+end
+'''
+
+
+class FishPlugin(BeetsPlugin):
+
+    def commands(self):
+        cmd = ui.Subcommand('fish', help='generate Fish shell tab completions')
+        cmd.func = self.run
+        cmd.parser.add_option('-f', '--noFields', action='store_true',
+                              default=False,
+                              help='omit album/track field completions')
+        cmd.parser.add_option(
+            '-e',
+            '--extravalues',
+            action='append',
+            type='choice',
+            choices=library.Item.all_keys() +
+            library.Album.all_keys(),
+            help='include specified field *values* in completions')
+        return [cmd]
+
+    def run(self, lib, opts, args):
+        # Gather the commands from Beets core and its plugins.
+        # Collect the album and track fields.
+        # If specified, also collect the values for these fields.
+        # Make a giant string of all the above, formatted in a way that
+        # allows Fish to do tab completion for the `beet` command.
+        home_dir = os.path.expanduser("~")
+        completion_dir = os.path.join(home_dir, '.config/fish/completions')
+        try:
+            os.makedirs(completion_dir)
+        except OSError:
+            if not os.path.isdir(completion_dir):
+                raise
+        completion_file_path = os.path.join(completion_dir, 'beet.fish')
+        nobasicfields = opts.noFields  # Do not complete for album/track fields
+        extravalues = opts.extravalues  # e.g., Also complete artists names
+        beetcmds = sorted(
+            (commands.default_commands +
+             commands.plugins.commands()),
+            key=attrgetter('name'))
+        fields = sorted(set(
+            library.Album.all_keys() + library.Item.all_keys()))
+        # Collect commands, their aliases, and their help text
+        cmd_names_help = []
+        for cmd in beetcmds:
+            names = [alias for alias in cmd.aliases]
+            names.append(cmd.name)
+            for name in names:
+                cmd_names_help.append((name, cmd.help))
+        # Concatenate the string
+        totstring = HEAD + "\n"
+        totstring += get_cmds_list([name[0] for name in cmd_names_help])
+        totstring += '' if nobasicfields else get_standard_fields(fields)
+        totstring += get_extravalues(lib, extravalues) if extravalues else ''
+        totstring += "\n" + "# ====== {} =====".format(
+            "setup basic beet completion") + "\n" * 2
+        totstring += get_basic_beet_options()
+        totstring += "\n" + "# ====== {} =====".format(
+            "setup field completion for subcommands") + "\n"
+        totstring += get_subcommands(
+            cmd_names_help, nobasicfields, extravalues)
+        # Set up completion for all the command options
+        totstring += get_all_commands(beetcmds)
+
+        with open(completion_file_path, 'w') as fish_file:
+            fish_file.write(totstring)
+
+
+def get_cmds_list(cmds_names):
+    # Make a list of all Beets core & plugin commands
+    substr = ''
+    substr += (
+        "set CMDS " + " ".join(cmds_names) + ("\n" * 2)
+    )
+    return substr
+
+
+def get_standard_fields(fields):
+    # Make a list of album/track fields and append with ':'
+    fields = (field + ":" for field in fields)
+    substr = ''
+    substr += (
+        "set FIELDS " + " ".join(fields) + ("\n" * 2)
+    )
+    return substr
+
+
+def get_extravalues(lib, extravalues):
+    # Make a list of all values from an album/track field.
+    # 'beet ls albumartist: <TAB>' yields completions for ABBA, Beatles, etc.
+    word = ''
+    values_set = get_set_of_values_for_field(lib, extravalues)
+    for fld in extravalues:
+        extraname = fld.upper() + 'S'
+        word += (
+            "set  " + extraname + " " + " ".join(sorted(values_set[fld]))
+            + ("\n" * 2)
+        )
+    return word
+
+
+def get_set_of_values_for_field(lib, fields):
+    # Get unique values from a specified album/track field
+    fields_dict = {}
+    for each in fields:
+        fields_dict[each] = set()
+    for item in lib.items():
+        for field in fields:
+            fields_dict[field].add(wrap(item[field]))
+    return fields_dict
+
+
+def get_basic_beet_options():
+    word = (
+        BL_NEED2.format("-l format-item",
+                        "-f -d 'print with custom format'") +
+        BL_NEED2.format("-l format-album",
+                        "-f -d 'print with custom format'") +
+        BL_NEED2.format("-s  l  -l library",
+                        "-f -r -d 'library database file to use'") +
+        BL_NEED2.format("-s  d  -l directory",
+                        "-f -r -d 'destination music directory'") +
+        BL_NEED2.format("-s  v  -l verbose",
+                        "-f -d 'print debugging information'") +
+
+        BL_NEED2.format("-s  c  -l config",
+                        "-f -r -d 'path to configuration file'") +
+        BL_NEED2.format("-s  h  -l help",
+                        "-f -d 'print this help message and exit'"))
+    return word
+
+
+def get_subcommands(cmd_name_and_help, nobasicfields, extravalues):
+    # Formatting for Fish to complete our fields/values
+    word = ""
+    for cmdname, cmdhelp in cmd_name_and_help:
+        word += "\n" + "# ------ {} -------".format(
+            "fieldsetups for  " + cmdname) + "\n"
+        word += (
+            BL_NEED2.format(
+                ("-a " + cmdname),
+                ("-f " + "-d " + wrap(clean_whitespace(cmdhelp)))))
+
+        if nobasicfields is False:
+            word += (
+                BL_USE3.format(
+                    cmdname,
+                    ("-a " + wrap("$FIELDS")),
+                    ("-f " + "-d " + wrap("fieldname"))))
+
+        if extravalues:
+            for f in extravalues:
+                setvar = wrap("$" + f.upper() + "S")
+                word += " ".join(BL_EXTRA3.format(
+                    (cmdname + " " + f + ":"),
+                    ('-f ' + '-A ' + '-a ' + setvar),
+                    ('-d ' + wrap(f))).split()) + "\n"
+    return word
+
+
+def get_all_commands(beetcmds):
+    # Formatting for Fish to complete command options
+    word = ""
+    for cmd in beetcmds:
+        names = [alias for alias in cmd.aliases]
+        names.append(cmd.name)
+        for name in names:
+            word += "\n"
+            word += ("\n" * 2) + "# ====== {} =====".format(
+                "completions for  " + name) + "\n"
+
+            for option in cmd.parser._get_all_options()[1:]:
+                cmd_l = (" -l " + option._long_opts[0].replace('--', '')
+                         )if option._long_opts else ''
+                cmd_s = (" -s " + option._short_opts[0].replace('-', '')
+                         ) if option._short_opts else ''
+                cmd_need_arg = ' -r ' if option.nargs in [1] else ''
+                cmd_helpstr = (" -d " + wrap(' '.join(option.help.split()))
+                               ) if option.help else ''
+                cmd_arglist = (' -a ' + wrap(" ".join(option.choices))
+                               ) if option.choices else ''
+
+                word += " ".join(BL_USE3.format(
+                    name,
+                    (cmd_need_arg + cmd_s + cmd_l + " -f " + cmd_arglist),
+                    cmd_helpstr).split()) + "\n"
+
+            word = (word + " ".join(BL_USE3.format(
+                name,
+                ("-s " + "h " + "-l " + "help" + " -f "),
+                ('-d ' + wrap("print help") + "\n")
+            ).split()))
+    return word
+
+
+def clean_whitespace(word):
+    # Remove excess whitespace and tabs in a string
+    return " ".join(word.split())
+
+
+def wrap(word):
+    # Need " or ' around strings but watch out if they're in the string
+    sptoken = '\"'
+    if ('"') in word and ("'") in word:
+        word.replace('"', sptoken)
+        return '"' + word + '"'
+
+    tok = '"' if "'" in word else "'"
+    return tok + word + tok

beetsplug/keyfinder.py

@@ -18,6 +18,7 @@
 
 from __future__ import division, absolute_import, print_function
 
+import os.path
 import subprocess
 
 from beets import ui
@@ -52,15 +53,19 @@ class KeyFinderPlugin(BeetsPlugin):
 
     def find_key(self, items, write=False):
         overwrite = self.config['overwrite'].get(bool)
-        bin = self.config['bin'].as_str()
+        command = [self.config['bin'].as_str()]
+        # The KeyFinder GUI program needs the -f flag before the path.
+        # keyfinder-cli is similar, but just wants the path with no flag.
+        if 'keyfinder-cli' not in os.path.basename(command[0]).lower():
+            command.append('-f')
 
         for item in items:
             if item['initial_key'] and not overwrite:
                 continue
 
             try:
-                output = util.command_output([bin, '-f',
-                                              util.syspath(item.path)]).stdout
+                output = util.command_output(command + [util.syspath(
+                                                        item.path)]).stdout
             except (subprocess.CalledProcessError, OSError) as exc:
                 self._log.error(u'execution failed: {0}', exc)
                 continue

beetsplug/lyrics.py

@@ -187,6 +187,9 @@ def search_pairs(item):
     In addition to the artist and title obtained from the `item` the
     method tries to strip extra information like paranthesized suffixes
     and featured artists from the strings and add them as candidates.
+    The artist sort name is added as a fallback candidate to help in
+    cases where artist name includes special characters or is in a
+    non-latin script.
     The method also tries to split multiple titles separated with `/`.
     """
     def generate_alternatives(string, patterns):
@@ -200,12 +203,16 @@ def search_pairs(item):
                 alternatives.append(match.group(1))
         return alternatives
 
-    title, artist = item.title, item.artist
+    title, artist, artist_sort = item.title, item.artist, item.artist_sort
 
     patterns = [
         # Remove any featuring artists from the artists name
         r"(.*?) {0}".format(plugins.feat_tokens())]
     artists = generate_alternatives(artist, patterns)
+    # Use the artist_sort as fallback only if it differs from artist to avoid
+    # repeated remote requests with the same search terms
+    if artist != artist_sort:
+        artists.append(artist_sort)
 
     patterns = [
         # Remove a parenthesized suffix from a title string. Common
@@ -373,9 +380,13 @@ class Genius(Backend):
 
         # At least Genius is nice and has a tag called 'lyrics'!
         # Updated css where the lyrics are based in HTML.
-        lyrics = html.find("div", class_="lyrics").get_text()
+        lyrics_div = html.find("div", class_="lyrics")
+        if lyrics_div is None:
+            self._log.debug(u'Genius lyrics for {0} not found',
+                            page_url)
+            return None
 
-        return lyrics
+        return lyrics_div.get_text()
 
     def fetch(self, artist, title):
         search_url = self.base_url + "/search"
@@ -395,13 +406,21 @@ class Genius(Backend):
 
         song_info = None
         for hit in json["response"]["hits"]:
-            if hit["result"]["primary_artist"]["name"] == artist:
+            # Genius uses zero-width characters to denote lowercase
+            # artist names.
+            hit_artist = hit["result"]["primary_artist"]["name"]. \
+                strip(u'\u200b').lower()
+
+            if hit_artist == artist.lower():
                 song_info = hit
                 break
 
         if song_info:
+            self._log.debug(u'fetched: {0}', song_info["result"]["url"])
             song_api_path = song_info["result"]["api_path"]
             return self.lyrics_from_song_api_path(song_api_path)
+        else:
+            self._log.debug(u'genius: no matching artist')
 
 
 class LyricsWiki(SymbolsReplaced):

beetsplug/parentwork.py

@@ -89,10 +89,11 @@ class ParentWorkPlugin(BeetsPlugin):
             write = ui.should_write()
 
             for item in lib.items(ui.decargs(args)):
-                self.find_work(item, force_parent)
-                item.store()
-                if write:
-                    item.try_write()
+                changed = self.find_work(item, force_parent)
+                if changed:
+                    item.store()
+                    if write:
+                        item.try_write()
         command = ui.Subcommand(
             'parentwork',
             help=u'fetche parent works, composers and dates')
@@ -198,7 +199,7 @@ add one at https://musicbrainz.org/recording/{}', item, item.mb_trackid)
 
         if work_date:
             item['work_date'] = work_date
-        ui.show_model_changes(
+        return ui.show_model_changes(
             item, fields=['parentwork', 'parentwork_disambig',
                           'mb_parentworkid', 'parent_composer',
                           'parent_composer_sort', 'work_date'])

beetsplug/replaygain.py

@@ -22,6 +22,7 @@ import math
 import sys
 import warnings
 import enum
+import re
 import xml.parsers.expat
 from six.moves import zip
 
@@ -66,6 +67,11 @@ def call(args, **kwargs):
         raise ReplayGainError(u"argument encoding failed")
 
 
+def after_version(version_a, version_b):
+    return tuple(int(s) for s in version_a.split('.')) \
+            >= tuple(int(s) for s in version_b.split('.'))
+
+
 def db_to_lufs(db):
     """Convert db to LUFS.
 
@@ -147,8 +153,12 @@ class Bs1770gainBackend(Backend):
 
         cmd = 'bs1770gain'
         try:
-            call([cmd, "--help"])
+            version_out = call([cmd, '--version'])
             self.command = cmd
+            self.version = re.search(
+                'bs1770gain ([0-9]+.[0-9]+.[0-9]+), ',
+                version_out.stdout.decode('utf-8')
+            ).group(1)
         except OSError:
             raise FatalReplayGainError(
                 u'Is bs1770gain installed?'
@@ -241,17 +251,23 @@ class Bs1770gainBackend(Backend):
         if self.__method != "":
             # backward compatibility to `method` option
             method = self.__method
+            gain_adjustment = target_level \
+                - [k for k, v in self.methods.items() if v == method][0]
         elif target_level in self.methods:
             method = self.methods[target_level]
             gain_adjustment = 0
         else:
-            method = self.methods[-23]
-            gain_adjustment = target_level - lufs_to_db(-23)
+            lufs_target = -23
+            method = self.methods[lufs_target]
+            gain_adjustment = target_level - lufs_target
 
         # Construct shell command.
         cmd = [self.command]
         cmd += ["--" + method]
         cmd += ['--xml', '-p']
+        if after_version(self.version, '0.6.0'):
+            cmd += ['--unit=ebu']            # set units to LU
+            cmd += ['--suppress-progress']   # don't print % to XML output
 
         # Workaround for Windows: the underlying tool fails on paths
         # with the \\?\ prefix, so we don't use it here. This
@@ -286,6 +302,7 @@ class Bs1770gainBackend(Backend):
         album_gain = {}  # mutable variable so it can be set from handlers
         parser = xml.parsers.expat.ParserCreate(encoding='utf-8')
         state = {'file': None, 'gain': None, 'peak': None}
+        album_state = {'gain': None, 'peak': None}
 
         def start_element_handler(name, attrs):
             if name == u'track':
@@ -294,9 +311,13 @@ class Bs1770gainBackend(Backend):
                     raise ReplayGainError(
                         u'duplicate filename in bs1770gain output')
             elif name == u'integrated':
-                state['gain'] = float(attrs[u'lu'])
+                if 'lu' in attrs:
+                    state['gain'] = float(attrs[u'lu'])
             elif name == u'sample-peak':
-                state['peak'] = float(attrs[u'factor'])
+                if 'factor' in attrs:
+                    state['peak'] = float(attrs[u'factor'])
+                elif 'amplitude' in attrs:
+                    state['peak'] = float(attrs[u'amplitude'])
 
         def end_element_handler(name):
             if name == u'track':
@@ -312,6 +333,17 @@ class Bs1770gainBackend(Backend):
                                           'the output of bs1770gain')
                 album_gain["album"] = Gain(state['gain'], state['peak'])
                 state['gain'] = state['peak'] = None
+            elif len(per_file_gain) == len(path_list):
+                if state['gain'] is not None:
+                    album_state['gain'] = state['gain']
+                if state['peak'] is not None:
+                    album_state['peak'] = state['peak']
+                if album_state['gain'] is not None \
+                        and album_state['peak'] is not None:
+                    album_gain["album"] = Gain(
+                        album_state['gain'], album_state['peak'])
+                state['gain'] = state['peak'] = None
+
         parser.StartElementHandler = start_element_handler
         parser.EndElementHandler = end_element_handler
 
@@ -1294,10 +1326,10 @@ class ReplayGainPlugin(BeetsPlugin):
 
         if (any([self.should_use_r128(item) for item in album.items()]) and not
                 all(([self.should_use_r128(item) for item in album.items()]))):
-            raise ReplayGainError(
-                u"Mix of ReplayGain and EBU R128 detected"
-                u" for some tracks in album {0}".format(album)
-            )
+            self._log.error(
+                u"Cannot calculate gain for album {0} (incompatible formats)",
+                album)
+            return
 
         tag_vals = self.tag_specific_values(album.items())
         store_track_gain, store_album_gain, target_level, peak = tag_vals

beetsplug/subsonicupdate.py

@@ -25,16 +25,59 @@ a "subsonic" section like the following:
 """
 from __future__ import division, absolute_import, print_function
 
-from beets.plugins import BeetsPlugin
-from beets import config
-import requests
-import string
 import hashlib
 import random
+import string
+
+import requests
+
+from beets import config
+from beets.plugins import BeetsPlugin
 
 __author__ = 'https://github.com/maffo999'
 
 
+def create_token():
+    """Create salt and token from given password.
+
+    :return: The generated salt and hashed token
+    """
+    password = config['subsonic']['pass'].as_str()
+
+    # Pick the random sequence and salt the password
+    r = string.ascii_letters + string.digits
+    salt = "".join([random.choice(r) for _ in range(6)])
+    salted_password = password + salt
+    token = hashlib.md5().update(salted_password.encode('utf-8')).hexdigest()
+
+    # Put together the payload of the request to the server and the URL
+    return salt, token
+
+
+def format_url():
+    """Get the Subsonic URL to trigger a scan. Uses either the url
+    config option or the deprecated host, port, and context_path config
+    options together.
+
+    :return: Endpoint for updating Subsonic
+    """
+
+    url = config['subsonic']['url'].as_str()
+    if url and url.endsWith('/'):
+        url = url[:-1]
+
+    # @deprecated("Use url config option instead")
+    if not url:
+        host = config['subsonic']['host'].as_str()
+        port = config['subsonic']['port'].get(int)
+        context_path = config['subsonic']['contextpath'].as_str()
+        if context_path == '/':
+            context_path = ''
+        url = "http://{}:{}{}".format(host, port, context_path)
+
+    return url + '/rest/startScan'
+
+
 class SubsonicUpdate(BeetsPlugin):
     def __init__(self):
         super(SubsonicUpdate, self).__init__()
@@ -46,42 +89,32 @@ class SubsonicUpdate(BeetsPlugin):
             'user': 'admin',
             'pass': 'admin',
             'contextpath': '/',
+            'url': 'http://localhost:4040',
         })
+
         config['subsonic']['pass'].redact = True
-        self.register_listener('import', self.loaded)
+        self.register_listener('import', self.start_scan)
 
-    def loaded(self):
-        host = config['subsonic']['host'].as_str()
-        port = config['subsonic']['port'].get(int)
+    def start_scan(self):
         user = config['subsonic']['user'].as_str()
-        passw = config['subsonic']['pass'].as_str()
-        contextpath = config['subsonic']['contextpath'].as_str()
+        url = format_url()
+        salt, token = create_token()
 
-        # To avoid sending plaintext passwords, authentication will be
-        # performed via username, a token, and a 6 random
-        # letters/numbers sequence.
-        # The token is the concatenation of your password and the 6 random
-        # letters/numbers (the salt) which is hashed with MD5.
-
-        # Pick the random sequence and salt the password
-        r = string.ascii_letters + string.digits
-        salt = "".join([random.choice(r) for n in range(6)])
-        t = passw + salt
-        token = hashlib.md5()
-        token.update(t.encode('utf-8'))
-
-        # Put together the payload of the request to the server and the URL
         payload = {
             'u': user,
-            't': token.hexdigest(),
+            't': token,
             's': salt,
             'v': '1.15.0',  # Subsonic 6.1 and newer.
             'c': 'beets'
         }
-        if contextpath == '/':
-            contextpath = ''
-        url = "http://{}:{}{}/rest/startScan".format(host, port, contextpath)
+
         response = requests.post(url, params=payload)
 
-        if response.status_code != 200:
-            self._log.error(u'Generic error, please try again later.')
+        if response.status_code == 403:
+            self._log.error(u'Server authentication failed')
+        elif response.status_code == 200:
+            self._log.debug(u'Updating Subsonic')
+        else:
+            self._log.error(
+                u'Generic error, please try again later [Status Code: {}]'
+                .format(response.status_code))

beetsplug/web/__init__.py

@@ -169,7 +169,7 @@ class IdListConverter(BaseConverter):
         return ids
 
     def to_url(self, value):
-        return ','.join(value)
+        return ','.join(str(v) for v in value)
 
 
 class QueryConverter(PathConverter):
@@ -177,10 +177,11 @@ class QueryConverter(PathConverter):
     """
 
     def to_python(self, value):
-        return value.split('/')
+        queries = value.split('/')
+        return [query.replace('\\', os.sep) for query in queries]
 
     def to_url(self, value):
-        return ','.join(value)
+        return ','.join([v.replace(os.sep, '\\') for v in value])
 
 
 class EverythingConverter(PathConverter):

docs/changelog.rst

@@ -8,6 +8,19 @@ New features:
 
 * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server.
 * :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and 
+* A new :ref:`extra_tags` configuration option allows more tagged metadata
+  to be included in MusicBrainz queries.
+* A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets
+* :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality``
+  option that controls the quality of the image output when the image is
+  resized.
+* :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_
+  Thanks to :user:`BrainDamage`.
+* :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to
+  allow downloading of higher resolution iTunes artwork (at the expense of
+  file size).
+  :bug: `3391`
+* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and
   `discogs_artistid`
   :bug: `3413`
 * :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; 
@@ -91,9 +104,29 @@ New features:
   :bug:`3387`
 * :doc:`/plugins/hook` now treats non-zero exit codes as errors.
   :bug:`3409`
+* :doc:`/plugins/subsonicupdate`: A new ``url`` configuration replaces the
+  older (and now deprecated) separate ``host``, ``port``, and ``contextpath``
+  config options. As a consequence, the plugin can now talk to Subsonic over
+  HTTPS.
+  Thanks to :user:`jef`.
+  :bug:`3449`
+* :doc:`/plugins/discogs`: The new ``index_tracks`` option enables
+  incorporation of work names and intra-work divisions into imported track
+  titles.
+  Thanks to :user:`cole-miller`.
+  :bug:`3459`
+* :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_.
+  :bug:`3530`
+* :doc:`/plugins/web`: The query API now interprets backslashes as path
+  separators to support path queries.
+  Thanks to :user:`nmeum`.
+  :bug:`3567`
 
 Fixes:
 
+* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take 
+  environment variables such as proxy servers into account when making requests
+  :bug:`3450`
 * :doc:`/plugins/inline`: In function-style field definitions that refer to
   flexible attributes, values could stick around from one function invocation
   to the next. This meant that, when displaying a list of objects, later
@@ -132,6 +165,38 @@ Fixes:
   wiping out their beets database.
   Thanks to :user:`logan-arens`.
   :bug:`1934`
+* :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode.
+  Thanks to :user:`aereaux`.
+  :bug:`3437`
+* :doc:`/plugins/lyrics`: Fix a corner-case with Genius lowercase artist names
+  :bug:`3446`
+* :doc:`/plugins/replaygain`: Support ``bs1770gain`` v0.6.0 and up
+  :bug:`3480`
+* :doc:`/plugins/parentwork`: Don't save tracks when nothing has changed.
+  :bug:`3492`
+* Added a warning when configuration files defined in the `include` directive
+  of the configuration file fail to be imported.
+  :bug:`3498`
+* Added the normalize method to the dbcore.types.INTEGER class which now
+  properly returns integer values, which should avoid problems where fields
+  like ``bpm`` would sometimes store non-integer values.
+  :bug:`762` :bug:`3507` :bug:`3508`
+* Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method
+  failing with AttributeError when called. It is now an instance method.
+  :bug:`3516` :bug:`3517`
+* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper.
+  Thanks to :user:`thejli21`.
+  :bug:`3535` :bug:`3554`
+* :doc:`/plugins/lyrics`: Use the artist sort name to search for lyrics, which
+  can help find matches when the artist name has special characters.
+  Thanks to :user:`hashhar`.
+  :bug:`3340` :bug:`3558`
+* :doc:`/plugins/replaygain`: Trying to calculate volume gain for an album
+  consisting of some formats using ``ReplayGain`` and some using ``R128``
+  will no longer crash; instead it is skipped and and a message is logged.
+  The log message has also been rewritten for to improve clarity.
+  Thanks to :user:`autrimpo`.
+  :bug:`3533`
 
 For plugin developers:
 
@@ -181,11 +246,15 @@ For packagers:
 * We attempted to fix an unreliable test, so a patch to `skip <https://sources.debian.org/src/beets/1.4.7-2/debian/patches/skip-broken-test/>`_
   or `repair <https://build.opensuse.org/package/view_file/openSUSE:Factory/beets/fix_test_command_line_option_relative_to_working_dir.diff?expand=1>`_
   the test may no longer be necessary.
+* This version drops support for Python 3.4.
 
+.. _Fish shell: https://fishshell.com/
 .. _MediaFile: https://github.com/beetbox/mediafile
 .. _Confuse: https://github.com/beetbox/confuse
 .. _works: https://musicbrainz.org/doc/Work
 .. _Deezer: https://www.deezer.com
+.. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli
+.. _last.fm: https://last.fm
 
 
 1.4.9 (May 30, 2019)

docs/plugins/chroma.rst

@@ -96,7 +96,9 @@ Usage
 
 Once you have all the dependencies sorted out, enable the ``chroma`` plugin in
 your configuration (see :ref:`using-plugins`) to benefit from fingerprinting
-the next time you run ``beet import``.
+the next time you run ``beet import``. (The plugin doesn't produce any obvious
+output by default. If you want to confirm that it's enabled, you can try
+running in verbose mode once with ``beet -v import``.)
 
 You can also use the ``beet fingerprint`` command to generate fingerprints for
 items already in your library. (Provide a query to fingerprint a subset of your

docs/plugins/discogs.rst

@@ -48,6 +48,33 @@ Configuration
 
 This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`.
 
+There is one additional option in the ``discogs:`` section, ``index_tracks``.
+Index tracks (see the `Discogs guidelines
+<https://support.discogs.com/hc/en-us/articles/360005055373-Database-Guidelines-12-Tracklisting#12.13>`_),
+along with headers, mark divisions between distinct works on the same release
+or within works. When ``index_tracks`` is enabled::
+
+    discogs:
+        index_tracks: yes
+
+beets will incorporate the names of the divisions containing each track into
+the imported track's title. For example, importing
+`this album
+<https://www.discogs.com/Handel-Sutherland-Kirkby-Kwella-Nelson-Watkinson-Bowman-Rolfe-Johnson-Elliott-Partridge-Thomas-The-A/release/2026070>`_
+would result in track names like::
+
+    Messiah, Part I: No.1: Sinfony
+    Messiah, Part II: No.22: Chorus- Behold The Lamb Of God
+    Athalia, Act I, Scene I: Sinfonia
+
+whereas with ``index_tracks`` disabled you'd get::
+
+    No.1: Sinfony
+    No.22: Chorus- Behold The Lamb Of God
+    Sinfonia
+
+This option is useful when importing classical music.
+
 Troubleshooting
 ---------------
 

docs/plugins/embedart.rst

@@ -58,6 +58,13 @@ file. The available options are:
   the aspect ratio is preserved. See also :ref:`image-resizing` for further
   caveats about image resizing.
   Default: 0 (disabled).
+- **quality**: The JPEG quality level to use when compressing images (when
+  ``maxwidth`` is set). This should be either a number from 1 to 100 or 0 to
+  use the default quality. 65–75 is usually a good starting point. The default
+  behavior depends on the imaging tool used for scaling: ImageMagick tries to
+  estimate the input image quality and uses 92 if it cannot be determined, and
+  PIL defaults to 75.
+  Default: 0 (disabled)
 - **remove_art_file**: Automatically remove the album art file for the album
   after it has been embedded. This option is best used alongside the
   :doc:`FetchArt </plugins/fetchart>` plugin to download art with the purpose of

docs/plugins/fetchart.rst

@@ -42,6 +42,13 @@ file. The available options are:
 - **maxwidth**: A maximum image width to downscale fetched images if they are
   too big. The resize operation reduces image width to at most ``maxwidth``
   pixels. The height is recomputed so that the aspect ratio is preserved.
+- **quality**: The JPEG quality level to use when compressing images (when
+  ``maxwidth`` is set). This should be either a number from 1 to 100 or 0 to
+  use the default quality. 65–75 is usually a good starting point. The default
+  behavior depends on the imaging tool used for scaling: ImageMagick tries to
+  estimate the input image quality and uses 92 if it cannot be determined, and
+  PIL defaults to 75.
+  Default: 0 (disabled)
 - **enforce_ratio**: Only images with a width:height ratio of 1:1 are
   considered as valid album art candidates if set to ``yes``.
   It is also possible to specify a certain deviation to the exact ratio to
@@ -51,9 +58,9 @@ file. The available options are:
 - **sources**: List of sources to search for images. An asterisk `*` expands
   to all available sources.
   Default: ``filesystem coverart itunes amazon albumart``, i.e., everything but
-  ``wikipedia``, ``google`` and ``fanarttv``. Enable those sources for more
-  matches at the cost of some speed. They are searched in the given order,
-  thus in the default config, no remote (Web) art source are queried if
+  ``wikipedia``, ``google``, ``fanarttv`` and ``lastfm``. Enable those sources
+  for more matches at the cost of some speed. They are searched in the given
+  order, thus in the default config, no remote (Web) art source are queried if
   local art is found in the filesystem. To use a local image as fallback,
   move it to the end of the list. For even more fine-grained control over
   the search order, see the section on :ref:`album-art-sources` below.
@@ -64,9 +71,14 @@ file. The available options are:
   Default: The `beets custom search engine`_, which searches the entire web.
 - **fanarttv_key**: The personal API key for requesting art from
   fanart.tv. See below.
+- **lastfm_key**: The personal API key for requesting art from Last.fm. See
+  below.
 - **store_source**: If enabled, fetchart stores the artwork's source in a
   flexible tag named ``art_source``. See below for the rationale behind this.
   Default: ``no``.
+- **high_resolution**: If enabled, fetchart retrieves artwork in the highest
+  resolution it can find (warning: image files can sometimes reach >20MB).
+  Default: ``no``.
 
 Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_
 or `Pillow`_.
@@ -114,8 +126,9 @@ art::
 
     $ beet fetchart [-q] [query]
 
-By default the command will display all results, the ``-q`` or ``--quiet``
-switch will only display results for album arts that are still missing.
+By default the command will display all albums matching the ``query``. When the
+``-q`` or ``--quiet`` switch is given, only albums for which artwork has been
+fetched, or for which artwork could not be found will be printed.
 
 .. _image-resizing:
 
@@ -211,6 +224,15 @@ personal key will give you earlier access to new art.
 
 .. _on their blog: https://fanart.tv/2015/01/personal-api-keys/
 
+Last.fm
+'''''''
+
+To use the Last.fm backend, you need to `register for a Last.fm API key`_. Set
+the ``lastfm_key`` configuration option to your API key, then add ``lastfm`` to
+the list of sources in your configutation.
+
+.. _register for a Last.fm API key: https://www.last.fm/api/account/create
+
 Storing the Artwork's Source
 ----------------------------
 

docs/plugins/fish.rst

@@ -0,0 +1,52 @@
+Fish Plugin
+===========
+
+The ``fish`` plugin adds a ``beet fish`` command that creates a `Fish shell`_
+tab-completion file named ``beet.fish`` in ``~/.config/fish/completions``.
+This enables tab-completion of ``beet`` commands for the `Fish shell`_.
+
+.. _Fish shell: https://fishshell.com/
+
+Configuration
+-------------
+
+Enable the ``fish`` plugin (see :ref:`using-plugins`) on a system running the
+`Fish shell`_.
+
+Usage
+-----
+
+Type ``beet fish`` to generate the ``beet.fish`` completions file at:
+``~/.config/fish/completions/``. If you later install or disable plugins, run
+``beet fish`` again to update the completions based on the enabled plugins.
+
+For users not accustomed to tab completion… After you type ``beet`` followed by
+a space in your shell prompt and then the ``TAB`` key, you should see a list of
+the beets commands (and their abbreviated versions) that can be invoked in your
+current environment. Similarly, typing ``beet -<TAB>`` will show you all the
+option flags available to you, which also applies to subcommands such as
+``beet import -<TAB>``. If you type ``beet ls`` followed by a space and then the
+and the ``TAB`` key, you will see a list of all the album/track fields that can
+be used in beets queries. For example, typing ``beet ls ge<TAB>`` will complete
+to ``genre:`` and leave you ready to type the rest of your query.
+
+Options
+-------
+
+In addition to beets commands, plugin commands, and option flags, the generated
+completions also include by default all the album/track fields. If you only want
+the former and do not want the album/track fields included in the generated
+completions, use ``beet fish -f`` to only generate completions for beets/plugin
+commands and option flags.
+
+If you want generated completions to also contain album/track field *values* for
+the items in your library, you can use the ``-e`` or ``--extravalues`` option.
+For example: ``beet fish -e genre`` or ``beet fish -e genre -e albumartist``
+In the latter case, subsequently typing ``beet list genre: <TAB>`` will display
+a list of all the genres in your library and ``beet list albumartist: <TAB>``
+will show a list of the album artists in your library. Keep in mind that all of
+these values will be put into the generated completions file, so use this option
+with care when specified fields contain a large number of values. Libraries with,
+for example, very large numbers of genres/artists may result in higher memory
+utilization, completion latency, et cetera. This option is not meant to replace
+database queries altogether.

docs/plugins/index.rst

@@ -78,6 +78,7 @@ following to your configuration::
    export
    fetchart
    filefilter
+   fish
    freedesktop
    fromfilename
    ftintitle
@@ -185,6 +186,7 @@ Interoperability
 
 * :doc:`badfiles`: Check audio file integrity.
 * :doc:`embyupdate`: Automatically notifies `Emby`_ whenever the beets library changes.
+* :doc:`fish`: Adds `Fish shell`_ tab autocompletion to ``beet`` commands.
 * :doc:`importfeeds`: Keep track of imported files via ``.m3u`` playlist file(s) or symlinks.
 * :doc:`ipfs`: Import libraries from friends and get albums from them via ipfs.
 * :doc:`kodiupdate`: Automatically notifies `Kodi`_ whenever the beets library
@@ -199,12 +201,16 @@ Interoperability
 * :doc:`sonosupdate`: Automatically notifies `Sonos`_ whenever the beets library
   changes.
 * :doc:`thumbnails`: Get thumbnails with the cover art on your album folders.
+* :doc:`subsonicupdate`: Automatically notifies `Subsonic`_ whenever the beets
+  library changes.
 
 
 .. _Emby: https://emby.media
+.. _Fish shell: https://fishshell.com/
 .. _Plex: https://plex.tv
 .. _Kodi: https://kodi.tv
 .. _Sonos: https://sonos.com
+.. _Subsonic: http://www.subsonic.org/
 
 Miscellaneous
 -------------
@@ -295,7 +301,22 @@ Here are a few of the plugins written by the beets community:
 * `beet-summarize`_ can compute lots of counts and statistics about your music
   library.
 
-* `beets-mosaic`_ generates a montage of a mosiac from cover art.
+* `beets-mosaic`_ generates a montage of a mosaic from cover art.
+
+* `beets-goingrunning`_ generates playlists to go with your running sessions.
+
+* `beets-xtractor`_ extracts low- and high-level musical information from your songs.
+
+* `beets-yearfixer`_ attempts to fix all missing ``original_year`` and ``year`` fields.
+
+* `beets-autofix`_ automates repetitive tasks to keep your library in order.
+
+* `beets-describe`_ gives you the full picture of a single attribute of your library items.
+
+* `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM).
+
+* `beets-originquery`_ augments MusicBrainz queries with locally-sourced data
+  to improve autotagger results.
 
 .. _beets-barcode: https://github.com/8h2a/beets-barcode
 .. _beets-check: https://github.com/geigerzaehler/beets-check
@@ -319,3 +340,10 @@ Here are a few of the plugins written by the beets community:
 .. _beets-ydl: https://github.com/vmassuchetto/beets-ydl
 .. _beet-summarize: https://github.com/steven-murray/beet-summarize
 .. _beets-mosaic: https://github.com/SusannaMaria/beets-mosaic
+.. _beets-goingrunning: https://pypi.org/project/beets-goingrunning
+.. _beets-xtractor: https://github.com/adamjakab/BeetsPluginXtractor
+.. _beets-yearfixer: https://github.com/adamjakab/BeetsPluginYearFixer
+.. _beets-autofix: https://github.com/adamjakab/BeetsPluginAutofix
+.. _beets-describe: https://github.com/adamjakab/BeetsPluginDescribe
+.. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser
+.. _beets-originquery: https://github.com/x1ppy/beets-originquery

docs/plugins/keyfinder.rst

@@ -1,9 +1,9 @@
 Key Finder Plugin
 =================
 
-The `keyfinder` plugin uses the `KeyFinder`_ program to detect the
-musical key of track from its audio data and store it in the
-`initial_key` field of your database.  It does so
+The `keyfinder` plugin uses either the `KeyFinder`_ or `keyfinder-cli`_
+program to  detect the musical key of a track from its audio data and store
+it in the `initial_key` field of your database.  It does so
 automatically when importing music or through the ``beet keyfinder
 [QUERY]`` command.
 
@@ -20,13 +20,16 @@ configuration file. The available options are:
   import. Otherwise, you need to use the ``beet keyfinder`` command
   explicitly.
   Default: ``yes``
-- **bin**: The name of the `KeyFinder`_ program on your system or
-  a path to the binary. If you installed the KeyFinder GUI on a Mac, for
-  example, you want something like
+- **bin**: The name of the program use for key analysis. You can use either
+  `KeyFinder`_ or `keyfinder-cli`_.
+  If you installed the KeyFinder GUI on a Mac, for example, you want
+  something like
   ``/Applications/KeyFinder.app/Contents/MacOS/KeyFinder``.
+  If using `keyfinder-cli`_, the binary must be named ``keyfinder-cli``.
   Default: ``KeyFinder`` (i.e., search for the program in your ``$PATH``)..
 - **overwrite**: Calculate a key even for files that already have an
   `initial_key` value.
   Default: ``no``.
 
 .. _KeyFinder: https://www.ibrahimshaath.co.uk/keyfinder/
+.. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli/

docs/plugins/playlist.rst

@@ -13,7 +13,7 @@ Then configure your playlists like this::
         playlist_dir: ~/.mpd/playlists
         forward_slash: no
 
-It is possible to query the library based on a playlist by speicifying its
+It is possible to query the library based on a playlist by specifying its
 absolute path::
 
     $ beet ls playlist:/path/to/someplaylist.m3u

docs/plugins/subsonicupdate.rst

@@ -13,11 +13,9 @@ You can do that using a ``subsonic:`` section in your ``config.yaml``,
 which looks like this::
 
     subsonic:
-        host: X.X.X.X
-        port: 4040
+        url: https://example.com:443/subsonic
         user: username
         pass: password
-        contextpath: /subsonic
 
 With that all in place, beets will send a Rest API to your Subsonic
 server every time you import new music.
@@ -30,8 +28,7 @@ Configuration
 
 The available options under the ``subsonic:`` section are:
 
-- **host**: The Subsonic server name/IP. Default: ``localhost``
-- **port**: The Subsonic server port. Default: ``4040``
+- **url**: The Subsonic server resource. Default: ``http://localhost:4040``
 - **user**: The Subsonic user. Default: ``admin``
-- **pass**: The Subsonic user password. Default: ``admin``
-- **contextpath**: The Subsonic context path. Default: ``/``
+- **pass**: The Subsonic user password. (This may either be a clear-text
+  password or hex-encoded with the prefix ``enc:``.) Default: ``admin``

docs/plugins/web.rst

@@ -210,7 +210,8 @@ If the server runs UNIX, you'll need to include an extra leading slash:
 ``GET /item/query/querystring``
 +++++++++++++++++++++++++++++++
 
-Returns a list of tracks matching the query. The *querystring* must be a valid query as described in :doc:`/reference/query`. ::
+Returns a list of tracks matching the query. The *querystring* must be a
+valid query as described in :doc:`/reference/query`. ::
 
     {
       "results": [
@@ -219,6 +220,11 @@ Returns a list of tracks matching the query. The *querystring* must be a valid q
       ]
     }
 
+Path elements are joined as parts of a query. For example,
+``/item/query/foo/bar`` will be converted to the query ``foo,bar``.
+To specify literal path separators in a query, use a backslash instead of a
+slash.
+
 
 ``GET /item/6/file``
 ++++++++++++++++++++

docs/reference/cli.rst

@@ -71,7 +71,7 @@ box. To extract `rar` files, install the `rarfile`_ package and the
 
 Optional command flags:
 
-* By default, the command copies files your the library directory and
+* By default, the command copies files to your library directory and
   updates the ID3 tags on your music. In order to move the files, instead of
   copying, use the ``-m`` (move) option. If you'd like to leave your music
   files untouched, try the ``-C`` (don't copy) and ``-W`` (don't write tags)

docs/reference/config.rst

@@ -701,6 +701,26 @@ MusicBrainz server.
 
 Default: ``5``.
 
+.. _extra_tags:
+
+extra_tags
+~~~~~~~~~~
+
+By default, beets will use only the artist, album, and track count to query
+MusicBrainz. Additional tags to be queried can be supplied with the
+``extra_tags`` setting. For example::
+
+    musicbrainz:
+        extra_tags: [year, catalognum, country, media, label]
+
+This setting should improve the autotagger results if the metadata with the
+given tags match the metadata returned by MusicBrainz.
+
+Note that the only tags supported by this setting are the ones listed in the
+above example.
+
+Default: ``[]``
+
 .. _match-config:
 
 Autotagger Matching Options

setup.py

@@ -118,7 +118,8 @@ setup(
         'responses',
         'pyxdg',
         'python-mpd2',
-        'discogs-client'
+        'discogs-client',
+        'requests_oauthlib'
     ] + (
         # Tests for the thumbnails plugin need pathlib on Python 2 too.
         ['pathlib'] if (sys.version_info < (3, 4, 0)) else []
@@ -172,7 +173,6 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

test/rsrc/lyrics/geniuscom/sample.txt

@@ -0,0 +1,270 @@
+<!DOCTYPE html>
+<html class="snarly apple_music_player--enabled bagon_song_page--enabled song_stories_public_launch--enabled react_forums--disabled" xmlns="http://www.w3.org/1999/xhtml" xmlns:fb="http://www.facebook.com/2008/fbml" lang="en" xml:lang="en">
+  <head>
+    <base target='_top' href="//g-example.com/">
+
+    <script type="text/javascript">
+    //<![CDATA[
+
+      var _sf_startpt=(new Date()).getTime();
+      if (window.performance && performance.mark) {
+        window.performance.mark('parse_start');
+      }
+
+    //]]>
+    </script>
+
+    <title>SAMPLE – SONG Lyrics | g-example Lyrics</title>
+
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta content='width=device-width,initial-scale=1' name='viewport'>
+
+    <meta property="og:site_name" content="g-example"/>
+
+    <link title="g-example" type="application/opensearchdescription+xml" rel="search" href="https://g-example.com/opensearch.xml">
+
+    <script async src="https://www.youtube.com/iframe_api"></script>
+    <script defer src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script>
+
+      <meta content="https://g-example.com/SAMPLE-SONG-lyrics" property="og:url" />
+      
+      <link href="ios-app://#/g-example/songs/#" rel="alternate" />
+      <meta content="/songs/3113595" name="newrelic-resource-path" />
+      <link href="https://g-example.com/SAMPLE-SONG-lyrics" rel="canonical" />
+      <link href="https://g-example.com/amp/SAMPLE-SONG-lyrics" rel="amphtml" />
+      
+    <script type="text/javascript">
+      var _qevents = _qevents || [];
+      (function() {
+        var elem = document.createElement('script');
+        elem.src = (document.location.protocol == 'https:' ? 'https://secure' : 'http://edge') + '.quantserve.com/quant.js';
+        elem.async = true;
+        elem.type = 'text/javascript';
+        var scpt = document.getElementsByTagName('script')[0];
+        scpt.parentNode.insertBefore(elem, scpt);
+      })();
+    </script>
+
+    <script type="text/javascript">
+      window.ga = window.ga || function() {
+        (window.ga.q = window.ga.q || []).push(arguments);
+      };
+
+      
+        (function(g, e, n, i, u, s) {
+          g['GoogleAnalyticsObject'] = 'ga';
+          g.ga.l = Date.now();
+          u = e.createElement(n);
+          s = e.getElementsByTagName(n)[0];
+          u.async = true;
+          u.src = i;
+          s.parentNode.insertBefore(u, s);
+        })(window, document, 'script', 'https://www.google-analytics.com/analytics.js');
+
+        ga('create', "UA-10346621-1", 'auto', {'useAmpClientId': true});
+        ga('set', 'dimension1', "false");
+        ga('set', 'dimension2', "songs#show");
+        ga('set', 'dimension3', "r-b");
+        ga('set', 'dimension4', "true");
+        ga('set', 'dimension5', 'false');
+        ga('set', 'dimension6', "none");
+        ga('send', 'pageview');
+      
+    </script>
+  </head>
+
+  <body>    
+
+    <div class="header" ng-controller="HeaderALBUM as header_ALBUM" click-outside="close_mobile_subnav_menu()">
+      <div class="header-primary active">
+        <div class="header-expand_nav_menu" ng-click="toggle_mobile_subnav_menu()"><div class="header-expand_nav_menu-contents"></div></div>
+
+        
+          <div class="logo_container">
+            <a href="https://g-example.com/" class="logo_link">g-example</a>
+          </div>
+        
+
+        <header-actions></header-actions>
+
+      <search-form search-style="header"></search-form>
+      </div>
+    </div>
+
+    <routable-page>
+      <ng-non-bindable>
+
+        <div class="header_with_cover_art">
+          <div class="header_with_cover_art-inner column_layout">
+            <div class="column_layout-column_span column_layout-column_span--primary">
+              <div class="header_with_cover_art-cover_art ">
+                <div class="cover_art">
+                  <img alt="#" class="cover_art-image" src="#" srcset="#" />
+                </div>
+              </div>
+
+              <div class="header_with_cover_art-primary_info_container">
+                <div class="header_with_cover_art-primary_info">
+                  <h1 class="header_with_cover_art-primary_info-title ">SONG</h1>
+                  <h2>
+                    <a href="https://g-example.com/artists/SAMPLE" class="header_with_cover_art-primary_info-primary_artist">
+                      SAMPLE
+                    </a>
+                  </h2>
+                  <h3>
+                    <div class="metadata_unit ">
+                      <span class="metadata_unit-label">Produced by</span>
+                      <span class="metadata_unit-info">
+                        <a href="https://g-example.com/artists/Person1">Person 1</a> & <a href="https://g-example.com/artists/Person 2">Person 2</a>
+                      </span>
+                    </div>
+                  </h3>
+                  <h3>
+                    <div class="metadata_unit ">
+                      <span class="metadata_unit-label">Album</span>
+                      <span class="metadata_unit-info"><a href="https://g-example.com/albums/SAMPLE/ALBUM">ALBUM</a></span>
+                    </div>
+                  </h3>
+                  </div>
+              </div>
+
+            </div>
+          </div>
+        </div>
+
+        <div class="song_body column_layout" initial-content-for="song_body">
+          <div class="column_layout-column_span column_layout-column_span--primary">
+            <div class="song_body-lyrics">
+                <h2 class="text_label text_label--gray text_label--x_small_text_size u-top_margin">SONG Lyrics</h2>
+              <div initial-content-for="lyrics">
+                <div class="totally-not-the-lyrics-div">
+                  !!!! MISSING LYRICS HERE !!!
+                </div>
+              </div>
+              <div initial-content-for="recirculated_content">
+                  <div class="u-xx_large_vertical_margins">
+                    <div class="text_label text_label--gray">More on g-example</div>
+                  </div>
+              </div>
+            </div>
+          </div>
+        </div>         
+        
+        <div class="metadata_unit metadata_unit--table_row">
+          <span class="metadata_unit-label">Released by</span>
+
+          <span class="metadata_unit-info">
+            <a href="https://g-example.com/artists/records">Records</a> & <a href="https://g-example.com/artists/Top">Top</a>
+          </span>
+        </div>
+        
+        <div class="metadata_unit metadata_unit--table_row">
+          <span class="metadata_unit-label">Mixing</span>
+          <span class="metadata_unit-info">
+            <a href="https://g-example.com/artists/Mixed-by-person">Mixed by Person</a>
+          </span>
+        </div>
+
+        <div class="metadata_unit metadata_unit--table_row">
+          <span class="metadata_unit-label">Recorded At</span>
+          <span class="metadata_unit-info metadata_unit-info--text_only">City, Place</span>
+        </div>
+
+        <div class="metadata_unit metadata_unit--table_row">
+          <span class="metadata_unit-label">Release Date</span>
+          <span class="metadata_unit-info metadata_unit-info--text_only">Feb 30, 1290</span>
+        </div>
+            
+        <div class="metadata_unit metadata_unit--table_row">
+          <span class="metadata_unit-label">Interpolated By</span>
+          <span class="metadata_unit-info">
+            
+            <div class="u-x_small_bottom_margin">
+              <a href="#"> # </a>
+            </div>
+            
+          </span>
+        </div>
+
+        <div initial-content-for="album">
+          <div class="u-xx_large_vertical_margins">
+            <div class="song_album u-bottom_margin">
+              <a href="https://g-example.com/albums/SAMPLE/ALBUM" class="song_album-album_art" title="ALBUM">
+                <img alt="#" src="#" srcset="#"/>
+              </a>
+            <div class="song_album-info">
+              <a href="https://g-example.com/albums/SAMPLE/ALBUM" title="ALBUM" class="song_album-info-title">
+                ALBUM
+              </a>
+              <a href="https://g-example.com/artists/SAMPLE" class="song_album-info-artist" title="ALBUM">SAMPLE</a>
+            </div>
+          </div>
+
+      </ng-non-bindable>
+    </routable-page>
+
+    <div class="page_footer page_footer--padding-for-sticky-player">
+      <div class="footer">
+        <div>
+          <a href="/about">About g-example</a>
+          <a href="/contributor_guidelines">Contributor Guidelines</a> 
+        </div>
+
+        <div>
+            <span>g-example</span>
+        </div>
+      </div>
+    </div>
+
+    <script type="text/javascript">_qevents.push({ qacct: "################"});</script>
+    <noscript>
+      <div style="display: none;">
+        <img src="#" height="1" width="1" alt="#"/>
+      </div>
+    </noscript>
+
+    <script type="text/javascript">
+      var _sf_async_config={};
+
+      _sf_async_config.uid = 3877;
+      _sf_async_config.domain = 'g-example.com';
+      _sf_async_config.title = 'SAMPLE – SONG Lyrics | g-example Lyrics';
+      _sf_async_config.sections = 'songs,tag:r-b';
+      _sf_async_config.authors = 'SAMPLE';
+
+      var _cbq = window._cbq || [];
+
+      (function(){
+        function loadChartbeat() {
+          window._sf_endpt=(new Date()).getTime();
+          var e = document.createElement('script');
+          e.setAttribute('language', 'javascript');
+          e.setAttribute('type', 'text/javascript');
+          e.setAttribute('src', '#');
+          document.body.appendChild(e);
+        }
+        var oldonload = window.onload;
+        window.onload = (typeof window.onload != 'function') ?
+          loadChartbeat : function() { oldonload(); loadChartbeat(); };
+      })();
+    </script>
+
+      <!-- Begin comScore Tag -->
+    <script>
+      var _comscore = _comscore || [];
+      _comscore.push({ c1: "2", c2: "17151659" });
+      (function() {
+        var s = document.createElement("script"), el = document.getElementsByTagName("script")[0]; s.async = true;
+        s.src = (document.location.protocol == "https:" ? "https://sb" : "http://b") + ".scorecardresearch.com/beacon.js";
+        el.parentNode.insertBefore(s, el);
+      })();
+    </script>
+    <noscript>
+      <img src="#"/>
+    </noscript>
+    <!-- End comScore Tag -->
+    <noscript>
+      <img height="1" width="1" style="display:none" src="#"/> 
+    </noscript>
+  </body>
+</html>

test/test_autotag.py

@@ -91,7 +91,10 @@ class PluralityTest(_common.TestCase):
                  for i in range(5)]
         likelies, _ = match.current_metadata(items)
         for f in fields:
-            self.assertEqual(likelies[f], '%s_1' % f)
+            if isinstance(likelies[f], int):
+                self.assertEqual(likelies[f], 0)
+            else:
+                self.assertEqual(likelies[f], '%s_1' % f)
 
 
 def _make_item(title, track, artist=u'some artist'):

test/test_dbcore.py

@@ -53,6 +53,7 @@ class ModelFixture1(dbcore.Model):
     _fields = {
         'id': dbcore.types.PRIMARY_ID,
         'field_one': dbcore.types.INTEGER,
+        'field_two': dbcore.types.STRING,
     }
     _types = {
         'some_float_field': dbcore.types.FLOAT,
@@ -355,7 +356,7 @@ class ModelTest(unittest.TestCase):
     def test_items(self):
         model = ModelFixture1(self.db)
         model.id = 5
-        self.assertEqual({('id', 5), ('field_one', 0)},
+        self.assertEqual({('id', 5), ('field_one', 0), ('field_two', '')},
                          set(model.items()))
 
     def test_delete_internal_field(self):
@@ -370,10 +371,28 @@ class ModelTest(unittest.TestCase):
 
 
 class FormatTest(unittest.TestCase):
-    def test_format_fixed_field(self):
+    def test_format_fixed_field_integer(self):
         model = ModelFixture1()
-        model.field_one = u'caf\xe9'
+        model.field_one = 155
         value = model.formatted().get('field_one')
+        self.assertEqual(value, u'155')
+
+    def test_format_fixed_field_integer_normalized(self):
+        """The normalize method of the Integer class rounds floats
+        """
+        model = ModelFixture1()
+        model.field_one = 142.432
+        value = model.formatted().get('field_one')
+        self.assertEqual(value, u'142')
+
+        model.field_one = 142.863
+        value = model.formatted().get('field_one')
+        self.assertEqual(value, u'143')
+
+    def test_format_fixed_field_string(self):
+        model = ModelFixture1()
+        model.field_two = u'caf\xe9'
+        value = model.formatted().get('field_two')
         self.assertEqual(value, u'caf\xe9')
 
     def test_format_flex_field(self):

test/test_importer.py

@@ -79,7 +79,7 @@ class AutotagStub(object):
         autotag.mb.album_for_id = self.mb_album_for_id
         autotag.mb.track_for_id = self.mb_track_for_id
 
-    def match_album(self, albumartist, album, tracks):
+    def match_album(self, albumartist, album, tracks, extra_tags):
         if self.matching == self.IDENT:
             yield self._make_album_match(albumartist, album, tracks)
 

test/test_lyrics.py

@@ -39,6 +39,7 @@ from mock import MagicMock
 log = logging.getLogger('beets.test_lyrics')
 raw_backend = lyrics.Backend({}, log)
 google = lyrics.Google(MagicMock(), log)
+genius = lyrics.Genius(MagicMock(), log)
 
 
 class LyricsPluginTest(unittest.TestCase):
@@ -94,6 +95,27 @@ class LyricsPluginTest(unittest.TestCase):
         self.assertEqual(('Alice and Bob', ['song']),
                          list(lyrics.search_pairs(item))[0])
 
+    def test_search_artist_sort(self):
+        item = Item(artist='CHVRCHΞS', title='song', artist_sort='CHVRCHES')
+        self.assertIn(('CHVRCHΞS', ['song']),
+                      lyrics.search_pairs(item))
+        self.assertIn(('CHVRCHES', ['song']),
+                      lyrics.search_pairs(item))
+
+        # Make sure that the original artist name is still the first entry
+        self.assertEqual(('CHVRCHΞS', ['song']),
+                         list(lyrics.search_pairs(item))[0])
+
+        item = Item(artist='横山克', title='song', artist_sort='Masaru Yokoyama')
+        self.assertIn(('横山克', ['song']),
+                      lyrics.search_pairs(item))
+        self.assertIn(('Masaru Yokoyama', ['song']),
+                      lyrics.search_pairs(item))
+
+        # Make sure that the original artist name is still the first entry
+        self.assertEqual(('横山克', ['song']),
+                         list(lyrics.search_pairs(item))[0])
+
     def test_search_pairs_multi_titles(self):
         item = Item(title='1 / 2', artist='A')
         self.assertIn(('A', ['1 / 2']), lyrics.search_pairs(item))
@@ -214,6 +236,33 @@ class MockFetchUrl(object):
         return content
 
 
+class GeniusMockGet(object):
+
+    def __init__(self, pathval='fetched_path'):
+        self.pathval = pathval
+        self.fetched = None
+
+    def __call__(self, url, headers=False):
+        from requests.models import Response
+        # for the first requests.get() return a path
+        if headers:
+            response = Response()
+            response.status_code = 200
+            response._content = b'{"meta":{"status":200},\
+                                "response":{"song":{"path":"/lyrics/sample"}}}'
+            return response
+        # for the second requests.get() return the genius page
+        else:
+            from mock import PropertyMock
+            self.fetched = url
+            fn = url_to_filename(url)
+            with open(fn, 'r') as f:
+                content = f.read()
+            response = Response()
+            type(response).text = PropertyMock(return_value=content)
+            return response
+
+
 def is_lyrics_content_ok(title, text):
     """Compare lyrics text to expected lyrics for given title."""
     if not text:
@@ -395,6 +444,40 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest):
         google.is_page_candidate(url, url_title, s['title'], u'Sunn O)))')
 
 
+class LyricsGeniusBaseTest(unittest.TestCase):
+
+    def setUp(self):
+        """Set up configuration."""
+        try:
+            __import__('bs4')
+        except ImportError:
+            self.skipTest('Beautiful Soup 4 not available')
+        if sys.version_info[:3] < (2, 7, 3):
+            self.skipTest("Python's built-in HTML parser is not good enough")
+
+
+class LyricsGeniusScrapTest(LyricsGeniusBaseTest):
+
+    """Checks that Genius backend works as intended.
+    """
+    import requests
+
+    def setUp(self):
+        """Set up configuration"""
+        LyricsGeniusBaseTest.setUp(self)
+        self.plugin = lyrics.LyricsPlugin()
+
+    @patch.object(requests, 'get', GeniusMockGet())
+    def test_no_lyrics_div(self):
+        """Ensure that `lyrics_from_song_api_path` doesn't crash when the html
+        for a Genius page contain <div class="lyrics"></div>
+        """
+        # https://github.com/beetbox/beets/issues/3535
+        # expected return value None
+        self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'),
+                         None)
+
+
 class SlugTests(unittest.TestCase):
 
     def test_slug(self):

test/test_query.py

@@ -772,6 +772,21 @@ class NoneQueryTest(unittest.TestCase, TestHelper):
         matched = self.lib.items(NoneQuery(u'rg_track_gain'))
         self.assertInResult(item, matched)
 
+    def test_match_slow(self):
+        item = self.add_item()
+        matched = self.lib.items(NoneQuery(u'rg_track_peak', fast=False))
+        self.assertInResult(item, matched)
+
+    def test_match_slow_after_set_none(self):
+        item = self.add_item(rg_track_gain=0)
+        matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False))
+        self.assertNotInResult(item, matched)
+
+        item['rg_track_gain'] = None
+        item.store()
+        matched = self.lib.items(NoneQuery(u'rg_track_gain', fast=False))
+        self.assertInResult(item, matched)
+
 
 class NotQueryMatchTest(_common.TestCase):
     """Test `query.NotQuery` matching against a single item, using the same

test/test_replaygain.py

@@ -40,7 +40,7 @@ if any(has_program(cmd, ['-v']) for cmd in ['mp3gain', 'aacgain']):
 else:
     GAIN_PROG_AVAILABLE = False
 
-if has_program('bs1770gain', ['--replaygain']):
+if has_program('bs1770gain'):
     LOUDNESS_PROG_AVAILABLE = True
 else:
     LOUDNESS_PROG_AVAILABLE = False
@@ -151,7 +151,9 @@ class ReplayGainCliTestBase(TestHelper):
         self.assertEqual(max(gains), min(gains))
 
         self.assertNotEqual(max(gains), 0.0)
-        self.assertNotEqual(max(peaks), 0.0)
+        if not self.backend == "bs1770gain":
+            # Actually produces peaks == 0.0 ~ self.add_album_fixture
+            self.assertNotEqual(max(peaks), 0.0)
 
     def test_cli_writes_only_r128_tags(self):
         if self.backend == "command":
@@ -228,7 +230,9 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase):
 
         # Patch call to return nothing, bypassing the bs1770gain installation
         # check.
-        call_patch.return_value = CommandOutput(stdout=b"", stderr=b"")
+        call_patch.return_value = CommandOutput(
+            stdout=b'bs1770gain 0.0.0, ', stderr=b''
+        )
         try:
             self.load_plugins('replaygain')
         except Exception:
@@ -250,7 +254,7 @@ class ReplayGainLdnsCliMalformedTest(TestHelper, unittest.TestCase):
     @patch('beetsplug.replaygain.call')
     def test_malformed_output(self, call_patch):
         # Return malformed XML (the ampersand should be &)
-        call_patch.return_value = CommandOutput(stdout="""
+        call_patch.return_value = CommandOutput(stdout=b"""
             <album>
                 <track total="1" number="1" file="&">
                     <integrated lufs="0" lu="0" />