diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index bc9cc77ec..ce88fa3bd 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -89,7 +89,7 @@ def parse_query_part(part, query_classes={}, prefixes={}, assert match # Regex should always match negate = bool(match.group(1)) key = match.group(2) - term = match.group(3).replace('\:', ':') + term = match.group(3).replace('\\:', ':') # Check whether there's a prefix in the query and use the # corresponding query type. diff --git a/beets/plugins.py b/beets/plugins.py index 1bd2cacd5..6dec7ef2a 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -488,7 +488,7 @@ def feat_tokens(for_artist=True): feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.'] if for_artist: feat_words += ['with', 'vs', 'and', 'con', '&'] - return '(?<=\s)(?:{0})(?=\s)'.format( + return r'(?<=\s)(?:{0})(?=\s)'.format( '|'.join(re.escape(x) for x in feat_words) ) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index af2b79a19..d3c3dafc2 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1193,10 +1193,11 @@ def _open_library(config): get_replacements(), ) lib.get_item(0) # Test database connection. - except (sqlite3.OperationalError, sqlite3.DatabaseError): + except (sqlite3.OperationalError, sqlite3.DatabaseError) as db_error: log.debug(u'{}', traceback.format_exc()) - raise UserError(u"database file {0} could not be opened".format( - util.displayable_path(dbpath) + raise UserError(u"database file {0} cannot not be opened: {1}".format( + util.displayable_path(dbpath), + db_error )) log.debug(u'library database: {0}\n' u'library directory: {1}', diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 0c288b9d8..5cce11bc0 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -24,7 +24,9 @@ import json import os import subprocess import tempfile +import sys +from multiprocessing.pool import ThreadPool from distutils.spawn import find_executable import requests @@ -104,10 +106,20 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) - for item in items: - analysis = self._get_analysis(item) - if analysis: - self._submit_data(item, analysis) + if sys.version_info[0] < 3: + for item in items: + self.analyze_submit(item) + else: + # Analyze in parallel using a thread pool. + pool = ThreadPool() + pool.map(self.analyze_submit, items) + pool.close() + pool.join() + + def analyze_submit(self, item): + analysis = self._get_analysis(item) + if analysis: + self._submit_data(item, analysis) def _get_analysis(self, item): mbid = item['mb_trackid'] diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index c4be2a3df..db993d612 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -60,7 +60,7 @@ def span_from_str(span_str): d = (yearfrom - yearfrom % 100) + d return d - years = [int(x) for x in re.findall('\d+', span_str)] + years = [int(x) for x in re.findall(r'\d+', span_str)] if not years: raise ui.UserError(u"invalid range defined for year bucket '%s': no " u"year found" % span_str) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 0e106694d..cb62059cd 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -35,12 +35,6 @@ from beets.util import confit from beets.util import syspath, bytestring_path, py3_path import six -try: - import itunes - HAVE_ITUNES = True -except ImportError: - HAVE_ITUNES = False - CONTENT_TYPES = { 'image/jpeg': [b'jpg', b'jpeg'], 'image/png': [b'png'] @@ -458,37 +452,65 @@ class FanartTV(RemoteArtSource): class ITunesStore(RemoteArtSource): NAME = u"iTunes Store" + API_URL = u'https://itunes.apple.com/search' def get(self, album, plugin, paths): """Return art URL from iTunes Store given an album title. """ if not (album.albumartist and album.album): return - search_string = (album.albumartist + ' ' + album.album).encode('utf-8') + + payload = { + 'term': album.albumartist + u' ' + album.album, + 'entity': u'album', + 'media': u'music', + 'limit': 200 + } try: - # Isolate bugs in the iTunes library while searching. + r = self.request(self.API_URL, params=payload) + r.raise_for_status() + except requests.RequestException as e: + self._log.debug(u'iTunes search failed: {0}', e) + return + + try: + candidates = r.json()['results'] + except ValueError as e: + self._log.debug(u'Could not decode json response: {0}', e) + return + except KeyError as e: + self._log.debug(u'{} not found in json. Fields are {} ', + e, + list(r.json().keys())) + return + + if not candidates: + self._log.debug(u'iTunes search for {!r} got no results', + payload['term']) + return + + for c in candidates: try: - results = itunes.search_album(search_string) - except Exception as exc: - self._log.debug(u'iTunes search failed: {0}', exc) - return + if (c['artistName'] == album.albumartist + and c['collectionName'] == album.album): + art_url = c['artworkUrl100'] + art_url = art_url.replace('100x100', '1200x1200') + yield self._candidate(url=art_url, + match=Candidate.MATCH_EXACT) + except KeyError as e: + self._log.debug(u'Malformed itunes candidate: {} not found in {}', # NOQA E501 + e, + list(c.keys())) - # Get the first match. - if results: - itunes_album = results[0] - else: - self._log.debug(u'iTunes search for {:r} got no results', - search_string) - return - - if itunes_album.get_artwork()['100']: - small_url = itunes_album.get_artwork()['100'] - big_url = small_url.replace('100x100', '1200x1200') - yield self._candidate(url=big_url, match=Candidate.MATCH_EXACT) - else: - self._log.debug(u'album has no artwork in iTunes Store') - except IndexError: - self._log.debug(u'album not found in iTunes Store') + try: + fallback_art_url = candidates[0]['artworkUrl100'] + fallback_art_url = fallback_art_url.replace('100x100', '1200x1200') + yield self._candidate(url=fallback_art_url, + match=Candidate.MATCH_FALLBACK) + except KeyError as e: + self._log.debug(u'Malformed itunes candidate: {} not found in {}', + e, + list(c.keys())) class Wikipedia(RemoteArtSource): @@ -756,8 +778,6 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.register_listener('import_task_files', self.assign_art) available_sources = list(SOURCES_ALL) - if not HAVE_ITUNES and u'itunes' in available_sources: - available_sources.remove(u'itunes') if not self.config['google_key'].get() and \ u'google' in available_sources: available_sources.remove(u'google') diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index 98f368cb4..c2fda19d4 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -30,16 +30,13 @@ import gmusicapi.clients class Gmusic(BeetsPlugin): def __init__(self): super(Gmusic, self).__init__() - # Checks for OAuth2 credentials, - # if they don't exist - performs authorization self.m = Musicmanager() - if os.path.isfile(gmusicapi.clients.OAUTH_FILEPATH): - self.m.login() - else: - self.m.perform_oauth() - self.config.add({ u'auto': False, + u'uploader_id': '', + u'uploader_name': '', + u'device_id': '', + u'oauth_file': gmusicapi.clients.OAUTH_FILEPATH, }) if self.config['auto']: self.import_stages = [self.autoupload] @@ -50,8 +47,7 @@ class Gmusic(BeetsPlugin): gupload.func = self.upload search = Subcommand('gmusic-songs', - help=u'list of songs in Google Play Music library' - ) + help=u'list of songs in Google Play Music library') search.parser.add_option('-t', '--track', dest='track', action='store_true', help='Search by track name') @@ -61,9 +57,25 @@ class Gmusic(BeetsPlugin): search.func = self.search return [gupload, search] + def authenticate(self): + if self.m.is_authenticated(): + return + # Checks for OAuth2 credentials, + # if they don't exist - performs authorization + oauth_file = self.config['oauth_file'].as_str() + if os.path.isfile(oauth_file): + uploader_id = self.config['uploader_id'] + uploader_name = self.config['uploader_name'] + self.m.login(oauth_credentials=oauth_file, + uploader_id=uploader_id.as_str().upper() or None, + uploader_name=uploader_name.as_str() or None) + else: + self.m.perform_oauth(oauth_file) + def upload(self, lib, opts, args): items = lib.items(ui.decargs(args)) files = self.getpaths(items) + self.authenticate() ui.print_(u'Uploading your files...') self.m.upload(filepaths=files) ui.print_(u'Your files were successfully added to library') @@ -71,6 +83,7 @@ class Gmusic(BeetsPlugin): def autoupload(self, session, task): items = task.imported_items() files = self.getpaths(items) + self.authenticate() self._log.info(u'Uploading files to Google Play Music...', files) self.m.upload(filepaths=files) self._log.info(u'Your files were successfully added to your ' @@ -82,14 +95,18 @@ class Gmusic(BeetsPlugin): def search(self, lib, opts, args): password = config['gmusic']['password'] email = config['gmusic']['email'] + uploader_id = config['gmusic']['uploader_id'] + device_id = config['gmusic']['device_id'] password.redact = True email.redact = True # Since Musicmanager doesn't support library management # we need to use mobileclient interface mobile = Mobileclient() try: - mobile.login(email.as_str(), password.as_str(), - Mobileclient.FROM_MAC_ADDRESS) + new_device_id = (device_id.as_str() + or uploader_id.as_str().replace(':', '') + or Mobileclient.FROM_MAC_ADDRESS).upper() + mobile.login(email.as_str(), password.as_str(), new_device_id) files = mobile.get_all_songs() except NotLoggedIn: ui.print_( diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 60f537597..6ecdbd1d0 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -131,7 +131,7 @@ def unescape(text): def replchar(m): num = m.group(1) return unichar(int(num)) - out = re.sub(u"&#(\d+);", replchar, out) + out = re.sub(u"&#(\\d+);", replchar, out) return out @@ -537,12 +537,12 @@ class Google(Backend): """ text = re.sub(r"[-'_\s]", '_', text) text = re.sub(r"_+", '_', text).strip('_') - pat = "([^,\(]*)\((.*?)\)" # Remove content within parentheses - text = re.sub(pat, '\g<1>', text).strip() + pat = r"([^,\(]*)\((.*?)\)" # Remove content within parentheses + text = re.sub(pat, r'\g<1>', text).strip() try: text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore') - text = six.text_type(re.sub('[-\s]+', ' ', text.decode('utf-8'))) + text = six.text_type(re.sub(r'[-\s]+', ' ', text.decode('utf-8'))) except UnicodeDecodeError: self._log.exception(u"Failing to normalize '{0}'", text) return text diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 35229281d..ac45aa4f8 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -181,9 +181,9 @@ class Bs1770gainBackend(Backend): i += 1 returnchunk = self.compute_chunk_gain(chunk, is_album) albumgaintot += returnchunk[-1].gain - albumpeaktot += returnchunk[-1].peak + albumpeaktot = max(albumpeaktot, returnchunk[-1].peak) returnchunks = returnchunks + returnchunk[0:-1] - returnchunks.append(Gain(albumgaintot / i, albumpeaktot / i)) + returnchunks.append(Gain(albumgaintot / i, albumpeaktot)) return returnchunks else: return self.compute_chunk_gain(items, is_album) diff --git a/beetsplug/the.py b/beetsplug/the.py index cfb583ced..83d1089de 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -23,8 +23,8 @@ from beets.plugins import BeetsPlugin __author__ = 'baobab@heresiarch.info' __version__ = '1.1' -PATTERN_THE = u'^[the]{3}\s' -PATTERN_A = u'^[a][n]?\s' +PATTERN_THE = u'^[the]{3}\\s' +PATTERN_A = u'^[a][n]?\\s' FORMAT = u'{0}, {1}' diff --git a/docs/changelog.rst b/docs/changelog.rst index 4786178cf..8baa9bce6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,8 @@ New features: :bug:`2442` * Added :doc:`/plugins/subsonicupdate` that can automatically update your Subsonic library. :user:`maffo999` +* replaygain: albumpeak on large collections is calculated as average, not maximum + :bug:`3008` Fixes: diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index d6d9adeff..002471ec1 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -16,12 +16,13 @@ The plugin uses `requests`_ to fetch album art from the Web. Fetching Album Art During Import -------------------------------- -When the plugin is enabled, it automatically gets album art for every album -you import. +When the plugin is enabled, it automatically tries to get album art for every +album you import. By default, beets stores album art image files alongside the music files for an album in a file called ``cover.jpg``. To customize the name of this file, use -the :ref:`art-filename` config option. +the :ref:`art-filename` config option. To embed the art into the files' tags, +use the :doc:`/plugins/embedart`. (You'll want to have both plugins enabled.) Configuration ------------- @@ -49,7 +50,7 @@ file. The available options are: (``enforce_ratio: 0.5%``). Default: ``no``. - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. - Default: ``filesystem coverart amazon albumart``, i.e., everything but + Default: ``filesystem coverart itunes amazon albumart``, i.e., everything but ``wikipedia``, ``google`` and ``fanarttv``. Enable those sources for more matches at the cost of some speed. They are searched in the given order, thus in the default config, no remote (Web) art source are queried if @@ -83,13 +84,13 @@ or `Pillow`_. .. _ImageMagick: http://www.imagemagick.org/ Here's an example that makes plugin select only images that contain *front* or -*back* keywords in their filenames and prioritizes the Amazon source over +*back* keywords in their filenames and prioritizes the iTunes source over others:: fetchart: cautious: true cover_names: front back - sources: amazon * + sources: itunes * Manually Fetching Album Art @@ -142,7 +143,7 @@ Album Art Sources ----------------- By default, this plugin searches for art in the local filesystem as well as on -the Cover Art Archive, Amazon, and AlbumArt.org, in that +the Cover Art Archive, the iTunes Store, Amazon, and AlbumArt.org, in that order. You can reorder the sources or remove some to speed up the process using the ``sources`` configuration option. @@ -222,10 +223,3 @@ album art fetch, you could do The values written to ``art_source`` are the same names used in the ``sources`` configuration value. - -Embedding Album Art -------------------- - -This plugin fetches album art but does not embed images into files' tags. To do -that, use the :doc:`/plugins/embedart`. (You'll want to have both plugins -enabled.) diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index a08c0abfa..a4f4c8e05 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -20,14 +20,16 @@ Then, you can enable the ``gmusic`` plugin in your configuration (see Usage ----- - -To automatically upload all tracks to Google Play Music, add the ``auto: yes`` -parameter to your configuration file like the example below:: +Configuration is required before use. Below is an example configuration:: gmusic: - auto: yes email: user@example.com password: seekrit + auto: yes + uploader_id: 00:11:22:33:AA:BB + device_id: 00112233AABB + oauth_file: ~/.config/beets/oauth.cred + To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: @@ -35,19 +37,7 @@ To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: If you don't include a query, the plugin will upload your entire collection. -To query the songs in your collection, you will need to add your Google -credentials to your beets configuration file. Put your Google username and -password under a section called ``gmusic``, like so:: - - gmusic: - email: user@example.com - password: seekrit - -If you have enabled two-factor authentication in your Google account, you will -need to set up and use an *application-specific password*. You can obtain one -from your Google security settings page. - -Then, use the ``gmusic-songs`` command to list music:: +To list your music collection, use the ``gmusic-songs`` command:: beet gmusic-songs [-at] [ARGS] @@ -59,3 +49,39 @@ example:: For a list of all songs in your library, run ``beet gmusic-songs`` without any arguments. + + +Configuration +------------- +To configure the plugin, make a ``gmusic:`` section in your configuration file. +The available options are: + +- **email**: Your Google account email address. + Default: none. +- **password**: Password to your Google account. Required to query songs in + your collection. + For accounts with 2-step-verification, an + `app password `__ + will need to be generated. An app password for an account without + 2-step-verification is not required but is recommended. + Default: none. +- **auto**: Set to ``yes`` to automatically upload new imports to Google Play + Music. + Default: ``no`` +- **uploader_id**: Unique id as a MAC address, eg ``00:11:22:33:AA:BB``. + This option should be set before the maximum number of authorized devices is + reached. + If provided, use the same id for all future runs on this, and other, beets + installations as to not reach the maximum number of authorized devices. + Default: device's MAC address. +- **device_id**: Unique device ID for authorized devices. It is usually + the same as your MAC address with the colons removed, eg ``00112233AABB``. + This option only needs to be set if you receive an `InvalidDeviceId` + exception. Below the exception will be a list of valid device IDs. + Default: none. +- **oauth_file**: Filepath for oauth credentials file. + Default: `{user_data_dir} `__/gmusicapi/oauth.cred + +Refer to the `Google Play Music Help +`__ +page for more details on authorized devices. diff --git a/setup.cfg b/setup.cfg index be2dbe543..0660b2721 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ logging-clear-handlers=1 min-version=2.7 accept-encodings=utf-8 # Errors we ignore: -# - E121,E123,E126,E226,E24,E704,W503,W504 flake8 default ignores (have to be listed here to not be overridden) +# - E121,E123,E126,E24,E704,W503,W504 flake8 default ignores, excluding E226 (have to be listed here to not be overridden) # - E221: multiple spaces before operator (used to align visually) # - E731: do not assign a lambda expression, use a def # - F405 object may be undefined, or defined from star imports @@ -20,4 +20,4 @@ accept-encodings=utf-8 # - FI14: `__future__` import "unicode_literals" missing # - FI15: `__future__` import "generator_stop" missing # - E741: ambiguous variable name -ignore=E121,E123,E126,E226,E24,E704,W503,W504,E305,C901,E221,E731,F405,FI50,FI51,FI12,FI53,FI14,FI15,E741 +ignore=E121,E123,E126,E24,E704,W503,W504,E305,C901,E221,E731,F405,FI50,FI51,FI12,FI53,FI14,FI15,E741