From b1d5cc1485dd9a1ea845f758bc489b11e6207f3c Mon Sep 17 00:00:00 2001 From: thetarkus Date: Mon, 13 Aug 2018 02:52:16 -0400 Subject: [PATCH 01/28] Add uploader_id, uploader_name, and device_id for gmusicapi auth. --- beetsplug/gmusic.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index 98f368cb4..1d5449ab9 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -30,17 +30,23 @@ import gmusicapi.clients class Gmusic(BeetsPlugin): def __init__(self): super(Gmusic, self).__init__() + self.config.add({ + u'auto': False, + u'uploader_id': '', + u'uploader_name': '', + u'device_id': '', + }) # Checks for OAuth2 credentials, # if they don't exist - performs authorization self.m = Musicmanager() if os.path.isfile(gmusicapi.clients.OAUTH_FILEPATH): - self.m.login() + uploader_id = self.config['uploader_id'] + uploader_name = self.config['uploader_name'] + self.m.login(uploader_id=uploader_id.as_str().upper() or None, + uploader_name=uploader_name.as_str() or None) else: self.m.perform_oauth() - self.config.add({ - u'auto': False, - }) if self.config['auto']: self.import_stages = [self.autoupload] @@ -82,6 +88,7 @@ class Gmusic(BeetsPlugin): def search(self, lib, opts, args): password = config['gmusic']['password'] email = config['gmusic']['email'] + device_id = config['gmusic']['device_id'] password.redact = True email.redact = True # Since Musicmanager doesn't support library management @@ -89,7 +96,7 @@ class Gmusic(BeetsPlugin): mobile = Mobileclient() try: mobile.login(email.as_str(), password.as_str(), - Mobileclient.FROM_MAC_ADDRESS) + device_id.as_str() or Mobileclient.FROM_MAC_ADDRESS) files = mobile.get_all_songs() except NotLoggedIn: ui.print_( From 966108b72c59b1ea865d06706285a89cae1f8d89 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 13 Aug 2018 10:37:45 -0400 Subject: [PATCH 02/28] Enable flake8 E226 This is the rule that prohibits `1+2` in favor of `1 + 2`. I didn't realize it was disabled by default! --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index be2dbe543..0660b2721 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ logging-clear-handlers=1 min-version=2.7 accept-encodings=utf-8 # Errors we ignore: -# - E121,E123,E126,E226,E24,E704,W503,W504 flake8 default ignores (have to be listed here to not be overridden) +# - E121,E123,E126,E24,E704,W503,W504 flake8 default ignores, excluding E226 (have to be listed here to not be overridden) # - E221: multiple spaces before operator (used to align visually) # - E731: do not assign a lambda expression, use a def # - F405 object may be undefined, or defined from star imports @@ -20,4 +20,4 @@ accept-encodings=utf-8 # - FI14: `__future__` import "unicode_literals" missing # - FI15: `__future__` import "generator_stop" missing # - E741: ambiguous variable name -ignore=E121,E123,E126,E226,E24,E704,W503,W504,E305,C901,E221,E731,F405,FI50,FI51,FI12,FI53,FI14,FI15,E741 +ignore=E121,E123,E126,E24,E704,W503,W504,E305,C901,E221,E731,F405,FI50,FI51,FI12,FI53,FI14,FI15,E741 From bac8faad7884c0af9c70bb26ed4bb7a0385c14dc Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 13 Aug 2018 10:41:01 -0400 Subject: [PATCH 03/28] Resolve W605: invalid escape sequence This came up in lots of regexes that weren't using "raw" literals. --- beets/dbcore/queryparse.py | 2 +- beets/plugins.py | 2 +- beetsplug/bucket.py | 2 +- beetsplug/lyrics.py | 8 ++++---- beetsplug/the.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index bc9cc77ec..ce88fa3bd 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -89,7 +89,7 @@ def parse_query_part(part, query_classes={}, prefixes={}, assert match # Regex should always match negate = bool(match.group(1)) key = match.group(2) - term = match.group(3).replace('\:', ':') + term = match.group(3).replace('\\:', ':') # Check whether there's a prefix in the query and use the # corresponding query type. diff --git a/beets/plugins.py b/beets/plugins.py index 1bd2cacd5..6dec7ef2a 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -488,7 +488,7 @@ def feat_tokens(for_artist=True): feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.'] if for_artist: feat_words += ['with', 'vs', 'and', 'con', '&'] - return '(?<=\s)(?:{0})(?=\s)'.format( + return r'(?<=\s)(?:{0})(?=\s)'.format( '|'.join(re.escape(x) for x in feat_words) ) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index c4be2a3df..db993d612 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -60,7 +60,7 @@ def span_from_str(span_str): d = (yearfrom - yearfrom % 100) + d return d - years = [int(x) for x in re.findall('\d+', span_str)] + years = [int(x) for x in re.findall(r'\d+', span_str)] if not years: raise ui.UserError(u"invalid range defined for year bucket '%s': no " u"year found" % span_str) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 60f537597..6ecdbd1d0 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -131,7 +131,7 @@ def unescape(text): def replchar(m): num = m.group(1) return unichar(int(num)) - out = re.sub(u"&#(\d+);", replchar, out) + out = re.sub(u"&#(\\d+);", replchar, out) return out @@ -537,12 +537,12 @@ class Google(Backend): """ text = re.sub(r"[-'_\s]", '_', text) text = re.sub(r"_+", '_', text).strip('_') - pat = "([^,\(]*)\((.*?)\)" # Remove content within parentheses - text = re.sub(pat, '\g<1>', text).strip() + pat = r"([^,\(]*)\((.*?)\)" # Remove content within parentheses + text = re.sub(pat, r'\g<1>', text).strip() try: text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore') - text = six.text_type(re.sub('[-\s]+', ' ', text.decode('utf-8'))) + text = six.text_type(re.sub(r'[-\s]+', ' ', text.decode('utf-8'))) except UnicodeDecodeError: self._log.exception(u"Failing to normalize '{0}'", text) return text diff --git a/beetsplug/the.py b/beetsplug/the.py index cfb583ced..83d1089de 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -23,8 +23,8 @@ from beets.plugins import BeetsPlugin __author__ = 'baobab@heresiarch.info' __version__ = '1.1' -PATTERN_THE = u'^[the]{3}\s' -PATTERN_A = u'^[a][n]?\s' +PATTERN_THE = u'^[the]{3}\\s' +PATTERN_A = u'^[a][n]?\\s' FORMAT = u'{0}, {1}' From 2b031be99c52b5aad75d51815e1f194ed5467fe2 Mon Sep 17 00:00:00 2001 From: Mike Mob Date: Mon, 13 Aug 2018 19:16:01 -0400 Subject: [PATCH 04/28] Update gmusic.rst --- docs/plugins/gmusic.rst | 65 ++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index a08c0abfa..e585ec742 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -20,14 +20,7 @@ Then, you can enable the ``gmusic`` plugin in your configuration (see Usage ----- - -To automatically upload all tracks to Google Play Music, add the ``auto: yes`` -parameter to your configuration file like the example below:: - - gmusic: - auto: yes - email: user@example.com - password: seekrit +**Configuration required before use.** To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: @@ -35,19 +28,7 @@ To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: If you don't include a query, the plugin will upload your entire collection. -To query the songs in your collection, you will need to add your Google -credentials to your beets configuration file. Put your Google username and -password under a section called ``gmusic``, like so:: - - gmusic: - email: user@example.com - password: seekrit - -If you have enabled two-factor authentication in your Google account, you will -need to set up and use an *application-specific password*. You can obtain one -from your Google security settings page. - -Then, use the ``gmusic-songs`` command to list music:: +To list your music collection, use the ``gmusic-songs`` command:: beet gmusic-songs [-at] [ARGS] @@ -59,3 +40,45 @@ example:: For a list of all songs in your library, run ``beet gmusic-songs`` without any arguments. + + +Configuration +------------- +To configure the plugin, make a ``gmusic:`` section in your configuration file. +The available options are: + +- **email**: Your Google account email address. + Default: none. +- **password**: Password to your Google account. Required to query songs in + your collection. + For accounts with 2-step-verification, an + `app password `__ + will need to be generated. An app password for an account without + 2-step-verification is not required but is recommended. + Default: none. +- **auto**: Set to ``yes`` to automatically upload new imports to Google Play + Music. + Default: ``no`` +- **uploader_id**: Unique id as a MAC address, eg ``'00:11:22:33:AA:BB'``. + This option should be set before the maximum number of authorized devices is + reached. + If provided, use the same id for all future runs on this, and other, beets + installations as to not reach the maximum number of authorized devices. + Default: device's MAC address. +- **device_id**: Unique device ID for authorized devices. + This option only needs to be set if you receive an `InvalidDeviceId` + exception. Below the exception will be a list of valid device IDs. + Default: none. + +Refer to the `Google Play Music Help +`__ +page for more details on authorized devices. + +Below is an example configuration:: + + gmusic: + email: user@example.com + password: seekrit + auto: yes + uploader_id: 00:11:22:33:AA:BB + device_id: F96AE4C643A5 From 22a6a0e6ef96055b7e50c58dcff27520d9de4e9e Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 13 Aug 2018 22:44:45 -0300 Subject: [PATCH 05/28] Parallelized absubmit --- beetsplug/absubmit.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 0c288b9d8..b8080fda5 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -25,6 +25,7 @@ import os import subprocess import tempfile +from multiprocessing import Pool from distutils.spawn import find_executable import requests @@ -102,12 +103,16 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): return [cmd] def command(self, lib, opts, args): + # Create threadpool + pool = Pool() # Get items from arguments items = lib.items(ui.decargs(args)) - for item in items: - analysis = self._get_analysis(item) - if analysis: - self._submit_data(item, analysis) + pool.map(self.analyze_submit, items) + + def analyze_submit(self, item): + analysis = self._get_analysis(item) + if analysis: + self._submit_data(item, analysis) def _get_analysis(self, item): mbid = item['mb_trackid'] From 6c2daa061e7250e009793eb286f0712913b0b80c Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 13 Aug 2018 23:22:40 -0300 Subject: [PATCH 06/28] Moved from Pool to ThreadPool --- beetsplug/absubmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index b8080fda5..acb80487b 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -25,7 +25,7 @@ import os import subprocess import tempfile -from multiprocessing import Pool +from multiprocessing import ThreadPool from distutils.spawn import find_executable import requests @@ -104,7 +104,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): def command(self, lib, opts, args): # Create threadpool - pool = Pool() + pool = ThreadPool() # Get items from arguments items = lib.items(ui.decargs(args)) pool.map(self.analyze_submit, items) From 31c95482d999d72e5e15eb69bc5556d40aa846f3 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 13 Aug 2018 23:23:41 -0300 Subject: [PATCH 07/28] Fix imports --- beetsplug/absubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index acb80487b..e21b1ec51 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -25,7 +25,7 @@ import os import subprocess import tempfile -from multiprocessing import ThreadPool +from multiprocessing.pool import ThreadPool from distutils.spawn import find_executable import requests From e13faf7f2473ba043d7c4bf55d140e3508026b30 Mon Sep 17 00:00:00 2001 From: Mike Mob Date: Mon, 13 Aug 2018 22:25:36 -0400 Subject: [PATCH 08/28] Update gmusic.rst --- docs/plugins/gmusic.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index e585ec742..e760f73a2 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -20,7 +20,15 @@ Then, you can enable the ``gmusic`` plugin in your configuration (see Usage ----- -**Configuration required before use.** +Configuration is required before use. Below is an example configuration:: + + gmusic: + email: user@example.com + password: seekrit + auto: yes + uploader_id: 00:11:22:33:AA:BB + device_id: F96AE4C643A5 + To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: @@ -73,12 +81,3 @@ The available options are: Refer to the `Google Play Music Help `__ page for more details on authorized devices. - -Below is an example configuration:: - - gmusic: - email: user@example.com - password: seekrit - auto: yes - uploader_id: 00:11:22:33:AA:BB - device_id: F96AE4C643A5 From d0758288d88e7a8378b15674caa67ca72d857e25 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 13 Aug 2018 23:27:57 -0300 Subject: [PATCH 09/28] Added absubmit parallelization to changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a8b9daf3..63be03825 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,9 @@ New features: :user:`jams2` * Automatically upload to Google Play Music library on track import. :user:`shuaiscott` +* The `absubmit` plugin now works in parallel. + Thanks to :user:`bemeurer` + :bug:`2442` Fixes: From 599a6c54480dc71e45e8645fa9d370de82aafd27 Mon Sep 17 00:00:00 2001 From: Mike Mob Date: Mon, 13 Aug 2018 22:36:17 -0400 Subject: [PATCH 10/28] Update changelog.rst --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a8b9daf3..05d1f626d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,8 @@ New features: :user:`jams2` * Automatically upload to Google Play Music library on track import. :user:`shuaiscott` +* New options for Google Play Music authentication in gmusic plugin. + :user:`thetarkus` Fixes: From a9f56db11eea605f1f2a5441901818800de20a1b Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 13 Aug 2018 23:50:51 -0300 Subject: [PATCH 11/28] Attempt to use ThreadPoolExecutor --- beetsplug/absubmit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index e21b1ec51..f05df4b2a 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -25,7 +25,7 @@ import os import subprocess import tempfile -from multiprocessing.pool import ThreadPool +from concurrent.futures import ThreadPoolExecutor from distutils.spawn import find_executable import requests @@ -103,11 +103,12 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): return [cmd] def command(self, lib, opts, args): - # Create threadpool - pool = ThreadPool() # Get items from arguments items = lib.items(ui.decargs(args)) - pool.map(self.analyze_submit, items) + + with ThreadPoolExecutor() as executor: + for item in items: + executor.submit(self.analyze_submit, (item)) def analyze_submit(self, item): analysis = self._get_analysis(item) From 0803ba74e2803f2bb19f8888288f4bdb8e1a6f21 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 13 Aug 2018 23:58:38 -0300 Subject: [PATCH 12/28] Attempt at Python2 and 3 compatibility --- beetsplug/absubmit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index f05df4b2a..d5cca4619 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -24,8 +24,10 @@ import json import os import subprocess import tempfile +import sys + +from concurrent import futures -from concurrent.futures import ThreadPoolExecutor from distutils.spawn import find_executable import requests @@ -106,7 +108,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): # Get items from arguments items = lib.items(ui.decargs(args)) - with ThreadPoolExecutor() as executor: + with futures.ThreadPoolExecutor() as executor: for item in items: executor.submit(self.analyze_submit, (item)) From 4eafa40ff2fa189698465a4c7df073c0a3bada7f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 14 Aug 2018 00:05:00 -0300 Subject: [PATCH 13/28] Revert to sequential behavior for Python 2 --- beetsplug/absubmit.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index d5cca4619..4487da76c 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -26,8 +26,7 @@ import subprocess import tempfile import sys -from concurrent import futures - +from multiprocessing.pool import ThreadPool from distutils.spawn import find_executable import requests @@ -107,10 +106,15 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) - - with futures.ThreadPoolExecutor() as executor: + if sys.version_info[0] < 3: for item in items: - executor.submit(self.analyze_submit, (item)) + self.analyze_submit(item) + else: + # Create threadpool + pool = ThreadPool() + pool.map(self.analyze_submit, items) + pool.close() + pool.join() def analyze_submit(self, item): analysis = self._get_analysis(item) From 69c86843021fd1ff87d85797356653addce2c1ff Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 14 Aug 2018 10:19:13 -0400 Subject: [PATCH 14/28] Revise comment --- beetsplug/absubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/absubmit.py b/beetsplug/absubmit.py index 4487da76c..5cce11bc0 100644 --- a/beetsplug/absubmit.py +++ b/beetsplug/absubmit.py @@ -110,7 +110,7 @@ class AcousticBrainzSubmitPlugin(plugins.BeetsPlugin): for item in items: self.analyze_submit(item) else: - # Create threadpool + # Analyze in parallel using a thread pool. pool = ThreadPool() pool.map(self.analyze_submit, items) pool.close() From dac38d2daf2fdcd073d0d7ccb2594fbb891a76ba Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 14 Aug 2018 10:20:10 -0400 Subject: [PATCH 15/28] Refine changelog for #3003 (fix #2442) --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8e05a5eaf..ead93cd2e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,8 +20,8 @@ New features: :user:`shuaiscott` * New options for Google Play Music authentication in gmusic plugin. :user:`thetarkus` -* The `absubmit` plugin now works in parallel. - Thanks to :user:`bemeurer` +* The `absubmit` plugin now works in parallel (on Python 3 only). + Thanks to :user:`bemeurer`. :bug:`2442` Fixes: From b7354fef23e8a08e8c68d98aa50ce0170fc3e326 Mon Sep 17 00:00:00 2001 From: thetarkus Date: Tue, 14 Aug 2018 22:35:10 -0400 Subject: [PATCH 16/28] gmusic plugin: only authenticate when needed --- beetsplug/gmusic.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index 1d5449ab9..ca2aa3b18 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -30,23 +30,13 @@ import gmusicapi.clients class Gmusic(BeetsPlugin): def __init__(self): super(Gmusic, self).__init__() + self.m = Musicmanager() self.config.add({ u'auto': False, u'uploader_id': '', u'uploader_name': '', u'device_id': '', }) - # Checks for OAuth2 credentials, - # if they don't exist - performs authorization - self.m = Musicmanager() - if os.path.isfile(gmusicapi.clients.OAUTH_FILEPATH): - uploader_id = self.config['uploader_id'] - uploader_name = self.config['uploader_name'] - self.m.login(uploader_id=uploader_id.as_str().upper() or None, - uploader_name=uploader_name.as_str() or None) - else: - self.m.perform_oauth() - if self.config['auto']: self.import_stages = [self.autoupload] @@ -67,9 +57,23 @@ class Gmusic(BeetsPlugin): search.func = self.search return [gupload, search] + def authenticate(self): + if self.m.is_authenticated(): + return + # Checks for OAuth2 credentials, + # if they don't exist - performs authorization + if os.path.isfile(gmusicapi.clients.OAUTH_FILEPATH): + uploader_id = self.config['uploader_id'] + uploader_name = self.config['uploader_name'] + self.m.login(uploader_id=uploader_id.as_str().upper() or None, + uploader_name=uploader_name.as_str() or None) + else: + self.m.perform_oauth() + def upload(self, lib, opts, args): items = lib.items(ui.decargs(args)) files = self.getpaths(items) + self.authenticate() ui.print_(u'Uploading your files...') self.m.upload(filepaths=files) ui.print_(u'Your files were successfully added to library') @@ -77,6 +81,7 @@ class Gmusic(BeetsPlugin): def autoupload(self, session, task): items = task.imported_items() files = self.getpaths(items) + self.authenticate() self._log.info(u'Uploading files to Google Play Music...', files) self.m.upload(filepaths=files) self._log.info(u'Your files were successfully added to your ' From abffb29a3fcdda8e55496a2d9e2f7ab5a38969de Mon Sep 17 00:00:00 2001 From: thetarkus Date: Tue, 14 Aug 2018 23:30:54 -0400 Subject: [PATCH 17/28] gmusic plugin addition: add oauth_filepath option, better device_id guessing --- beetsplug/gmusic.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index ca2aa3b18..76a219c61 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -36,6 +36,7 @@ class Gmusic(BeetsPlugin): u'uploader_id': '', u'uploader_name': '', u'device_id': '', + u'oauth_filepath': '', }) if self.config['auto']: self.import_stages = [self.autoupload] @@ -46,8 +47,7 @@ class Gmusic(BeetsPlugin): gupload.func = self.upload search = Subcommand('gmusic-songs', - help=u'list of songs in Google Play Music library' - ) + help=u'list of songs in Google Play Music library') search.parser.add_option('-t', '--track', dest='track', action='store_true', help='Search by track name') @@ -62,13 +62,16 @@ class Gmusic(BeetsPlugin): return # Checks for OAuth2 credentials, # if they don't exist - performs authorization - if os.path.isfile(gmusicapi.clients.OAUTH_FILEPATH): + oauth_filepath = (self.config['oauth_filepath'].as_str() + or gmusicapi.clients.OAUTH_FILEPATH) + if os.path.isfile(oauth_filepath): uploader_id = self.config['uploader_id'] uploader_name = self.config['uploader_name'] - self.m.login(uploader_id=uploader_id.as_str().upper() or None, + self.m.login(oauth_credentials=oauth_filepath, + uploader_id=uploader_id.as_str().upper() or None, uploader_name=uploader_name.as_str() or None) else: - self.m.perform_oauth() + self.m.perform_oauth(oauth_filepath) def upload(self, lib, opts, args): items = lib.items(ui.decargs(args)) @@ -93,6 +96,7 @@ class Gmusic(BeetsPlugin): def search(self, lib, opts, args): password = config['gmusic']['password'] email = config['gmusic']['email'] + uploader_id = config['gmusic']['uploader_id'] device_id = config['gmusic']['device_id'] password.redact = True email.redact = True @@ -100,8 +104,10 @@ class Gmusic(BeetsPlugin): # we need to use mobileclient interface mobile = Mobileclient() try: - mobile.login(email.as_str(), password.as_str(), - device_id.as_str() or Mobileclient.FROM_MAC_ADDRESS) + new_device_id = (device_id.as_str() + or uploader_id.as_str().replace(':', '') + or Mobileclient.FROM_MAC_ADDRESS).upper() + mobile.login(email.as_str(), password.as_str(), new_device_id) files = mobile.get_all_songs() except NotLoggedIn: ui.print_( From 971b0718c7a4707009554b71fc6a10d83115b3e6 Mon Sep 17 00:00:00 2001 From: thetarkus Date: Tue, 14 Aug 2018 23:44:03 -0400 Subject: [PATCH 18/28] Update gmusic docs --- docs/plugins/gmusic.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index e760f73a2..e43508a92 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -27,7 +27,8 @@ Configuration is required before use. Below is an example configuration:: password: seekrit auto: yes uploader_id: 00:11:22:33:AA:BB - device_id: F96AE4C643A5 + device_id: 00112233AABB + oauth_filepath: ~/.config/beets/oauth.cred To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: @@ -67,16 +68,19 @@ The available options are: - **auto**: Set to ``yes`` to automatically upload new imports to Google Play Music. Default: ``no`` -- **uploader_id**: Unique id as a MAC address, eg ``'00:11:22:33:AA:BB'``. +- **uploader_id**: Unique id as a MAC address, eg ``00:11:22:33:AA:BB``. This option should be set before the maximum number of authorized devices is reached. If provided, use the same id for all future runs on this, and other, beets installations as to not reach the maximum number of authorized devices. Default: device's MAC address. -- **device_id**: Unique device ID for authorized devices. +- **device_id**: Unique device ID for authorized devices. It is usually + the same as your MAC address with the colons removed, eg ``00112233AABB``. This option only needs to be set if you receive an `InvalidDeviceId` exception. Below the exception will be a list of valid device IDs. Default: none. +- **oauth_filepath**: Filepath for oauth credentials file. + Default: none. Refer to the `Google Play Music Help `__ From 0107262ed34c40fe5b0cdf0eee5f63dc02652486 Mon Sep 17 00:00:00 2001 From: "nath@dp7510" Date: Tue, 14 Aug 2018 12:45:59 +0200 Subject: [PATCH 19/28] fetchart: restore itunes art source Reimplement minimalistic itunes scraper from scratch --- beetsplug/fetchart.py | 74 +++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 0e106694d..f4d7ff3f8 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -35,12 +35,6 @@ from beets.util import confit from beets.util import syspath, bytestring_path, py3_path import six -try: - import itunes - HAVE_ITUNES = True -except ImportError: - HAVE_ITUNES = False - CONTENT_TYPES = { 'image/jpeg': [b'jpg', b'jpeg'], 'image/png': [b'png'] @@ -458,37 +452,59 @@ class FanartTV(RemoteArtSource): class ITunesStore(RemoteArtSource): NAME = u"iTunes Store" + API_URL = u'https://itunes.apple.com/search' def get(self, album, plugin, paths): """Return art URL from iTunes Store given an album title. """ if not (album.albumartist and album.album): return - search_string = (album.albumartist + ' ' + album.album).encode('utf-8') + + payload = { + 'term': album.albumartist + u' ' + album.album, + 'entity': u'album', + 'media': u'music', + 'limit': 200 + } try: - # Isolate bugs in the iTunes library while searching. + r = self.request(self.API_URL, params=payload) + r.raise_for_status() + except requests.RequestException as e: + self._log.debug(u'iTunes search failed: {0}', e) + return + + try: + candidates = r.json()['results'] + except ValueError as e: + self._log.debug(u'Could not decode json response: {0}', e) + return + except KeyError as e: + self._log.debug(u'No results lists in response: {0}', e) + return + + if not candidates: + self._log.debug(u'iTunes search for {:r} got no results', + payload['term']) + return + + for c in candidates: try: - results = itunes.search_album(search_string) - except Exception as exc: - self._log.debug(u'iTunes search failed: {0}', exc) - return + if (c['artistName'] == album.albumartist + and c['collectionName'] == album.album): + art_url = c['artworkUrl100'] + art_url = art_url.replace('100x100', '1200x1200') + yield self._candidate(url=art_url, + match=Candidate.MATCH_EXACT) + except KeyError as e: + self._log.debug(u'Malformed itunes candidate {0}', e) - # Get the first match. - if results: - itunes_album = results[0] - else: - self._log.debug(u'iTunes search for {:r} got no results', - search_string) - return - - if itunes_album.get_artwork()['100']: - small_url = itunes_album.get_artwork()['100'] - big_url = small_url.replace('100x100', '1200x1200') - yield self._candidate(url=big_url, match=Candidate.MATCH_EXACT) - else: - self._log.debug(u'album has no artwork in iTunes Store') - except IndexError: - self._log.debug(u'album not found in iTunes Store') + try: + fallback_art_url = candidates[0]['artworkUrl100'] + fallback_art_url = fallback_art_url.replace('100x100', '1200x1200') + yield self._candidate(url=fallback_art_url, + match=Candidate.MATCH_FALLBACK) + except KeyError as e: + self._log.debug(u'Malformed itunes candidate {0}', e) class Wikipedia(RemoteArtSource): @@ -756,8 +772,6 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.register_listener('import_task_files', self.assign_art) available_sources = list(SOURCES_ALL) - if not HAVE_ITUNES and u'itunes' in available_sources: - available_sources.remove(u'itunes') if not self.config['google_key'].get() and \ u'google' in available_sources: available_sources.remove(u'google') From 399bfb985eec01891ee7397c3307dd6f248b90fb Mon Sep 17 00:00:00 2001 From: "nath@dp7510" Date: Wed, 15 Aug 2018 13:56:39 +0200 Subject: [PATCH 20/28] fetchart: fix itunes debug info Some typos + make it more useful overall --- beetsplug/fetchart.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index f4d7ff3f8..cb62059cd 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -479,11 +479,13 @@ class ITunesStore(RemoteArtSource): self._log.debug(u'Could not decode json response: {0}', e) return except KeyError as e: - self._log.debug(u'No results lists in response: {0}', e) + self._log.debug(u'{} not found in json. Fields are {} ', + e, + list(r.json().keys())) return if not candidates: - self._log.debug(u'iTunes search for {:r} got no results', + self._log.debug(u'iTunes search for {!r} got no results', payload['term']) return @@ -496,7 +498,9 @@ class ITunesStore(RemoteArtSource): yield self._candidate(url=art_url, match=Candidate.MATCH_EXACT) except KeyError as e: - self._log.debug(u'Malformed itunes candidate {0}', e) + self._log.debug(u'Malformed itunes candidate: {} not found in {}', # NOQA E501 + e, + list(c.keys())) try: fallback_art_url = candidates[0]['artworkUrl100'] @@ -504,7 +508,9 @@ class ITunesStore(RemoteArtSource): yield self._candidate(url=fallback_art_url, match=Candidate.MATCH_FALLBACK) except KeyError as e: - self._log.debug(u'Malformed itunes candidate {0}', e) + self._log.debug(u'Malformed itunes candidate: {} not found in {}', + e, + list(c.keys())) class Wikipedia(RemoteArtSource): From ff6306b663c56ebeb9380dad90267c51ba18b1f4 Mon Sep 17 00:00:00 2001 From: "nath@home" Date: Wed, 15 Aug 2018 14:36:41 +0200 Subject: [PATCH 21/28] Changelog for #2718 Reverts 49e548bd from #2540 Fixes #2371 #2551 --- docs/changelog.rst | 7 +++++++ docs/plugins/fetchart.rst | 22 ++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a8b9daf3..1c6b6d358 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,11 @@ New features: Fixes: +* Restore iTunes Store album art source, and remove the dependency on + python-itunes_, which had gone unmaintained and was not py3 compatible. + Thanks to :user:`ocelma` for creating python-itunes_ in the first place. + Thanks to :user:`nathdwek`. + :bug:`2371` :bug:`2551` :bug:`2718` * Fix compatibility Python 3.7 and its change to a name in the ``re`` module. :bug:`2978` * R128 normalization tags are now properly deleted from files when the values @@ -41,6 +46,8 @@ Fixes: Thanks to :user:`rveachkc`. :bug:`2979`: :bug:`2980` +.. _python-itunes: https://github.com/ocelma/python-itunes + 1.4.7 (May 29, 2018) -------------------- diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index d6d9adeff..a4d959c01 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -16,12 +16,13 @@ The plugin uses `requests`_ to fetch album art from the Web. Fetching Album Art During Import -------------------------------- -When the plugin is enabled, it automatically gets album art for every album -you import. +When the plugin is enabled, it automatically tries to get album art for every +album you import. By default, beets stores album art image files alongside the music files for an album in a file called ``cover.jpg``. To customize the name of this file, use -the :ref:`art-filename` config option. +the :ref:`art-filename` config option. To embed the art into the files'tag, +use the :doc:`/plugins/embedart`. (You'll want to have both plugins enabled.) Configuration ------------- @@ -49,7 +50,7 @@ file. The available options are: (``enforce_ratio: 0.5%``). Default: ``no``. - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. - Default: ``filesystem coverart amazon albumart``, i.e., everything but + Default: ``filesystem coverart itunes amazon albumart``, i.e., everything but ``wikipedia``, ``google`` and ``fanarttv``. Enable those sources for more matches at the cost of some speed. They are searched in the given order, thus in the default config, no remote (Web) art source are queried if @@ -83,13 +84,13 @@ or `Pillow`_. .. _ImageMagick: http://www.imagemagick.org/ Here's an example that makes plugin select only images that contain *front* or -*back* keywords in their filenames and prioritizes the Amazon source over +*back* keywords in their filenames and prioritizes the iTunes source over others:: fetchart: cautious: true cover_names: front back - sources: amazon * + sources: itunes * Manually Fetching Album Art @@ -142,7 +143,7 @@ Album Art Sources ----------------- By default, this plugin searches for art in the local filesystem as well as on -the Cover Art Archive, Amazon, and AlbumArt.org, in that +the Cover Art Archive, the iTunes Store, Amazon, and AlbumArt.org, in that order. You can reorder the sources or remove some to speed up the process using the ``sources`` configuration option. @@ -222,10 +223,3 @@ album art fetch, you could do The values written to ``art_source`` are the same names used in the ``sources`` configuration value. - -Embedding Album Art -------------------- - -This plugin fetches album art but does not embed images into files' tags. To do -that, use the :doc:`/plugins/embedart`. (You'll want to have both plugins -enabled.) From c99bde4786faf6c1bb9fae51c717c1b87502702c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 15 Aug 2018 10:31:40 -0400 Subject: [PATCH 22/28] Fix a typo --- docs/plugins/fetchart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index a4d959c01..002471ec1 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -21,7 +21,7 @@ album you import. By default, beets stores album art image files alongside the music files for an album in a file called ``cover.jpg``. To customize the name of this file, use -the :ref:`art-filename` config option. To embed the art into the files'tag, +the :ref:`art-filename` config option. To embed the art into the files' tags, use the :doc:`/plugins/embedart`. (You'll want to have both plugins enabled.) Configuration From e824132137f8617397c35b6c51aed508e1ad6ba1 Mon Sep 17 00:00:00 2001 From: thetarkus Date: Wed, 15 Aug 2018 11:42:04 -0400 Subject: [PATCH 23/28] Rename oauth_filepath to oauth_file, update gmusic doc --- beetsplug/gmusic.py | 11 +++++------ docs/plugins/gmusic.rst | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index 76a219c61..c2fda19d4 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -36,7 +36,7 @@ class Gmusic(BeetsPlugin): u'uploader_id': '', u'uploader_name': '', u'device_id': '', - u'oauth_filepath': '', + u'oauth_file': gmusicapi.clients.OAUTH_FILEPATH, }) if self.config['auto']: self.import_stages = [self.autoupload] @@ -62,16 +62,15 @@ class Gmusic(BeetsPlugin): return # Checks for OAuth2 credentials, # if they don't exist - performs authorization - oauth_filepath = (self.config['oauth_filepath'].as_str() - or gmusicapi.clients.OAUTH_FILEPATH) - if os.path.isfile(oauth_filepath): + oauth_file = self.config['oauth_file'].as_str() + if os.path.isfile(oauth_file): uploader_id = self.config['uploader_id'] uploader_name = self.config['uploader_name'] - self.m.login(oauth_credentials=oauth_filepath, + self.m.login(oauth_credentials=oauth_file, uploader_id=uploader_id.as_str().upper() or None, uploader_name=uploader_name.as_str() or None) else: - self.m.perform_oauth(oauth_filepath) + self.m.perform_oauth(oauth_file) def upload(self, lib, opts, args): items = lib.items(ui.decargs(args)) diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index e43508a92..a4f4c8e05 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -28,7 +28,7 @@ Configuration is required before use. Below is an example configuration:: auto: yes uploader_id: 00:11:22:33:AA:BB device_id: 00112233AABB - oauth_filepath: ~/.config/beets/oauth.cred + oauth_file: ~/.config/beets/oauth.cred To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: @@ -79,8 +79,8 @@ The available options are: This option only needs to be set if you receive an `InvalidDeviceId` exception. Below the exception will be a list of valid device IDs. Default: none. -- **oauth_filepath**: Filepath for oauth credentials file. - Default: none. +- **oauth_file**: Filepath for oauth credentials file. + Default: `{user_data_dir} `__/gmusicapi/oauth.cred Refer to the `Google Play Music Help `__ From 1b5d0ca0e5e76e389c51e02af38866690947a120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Haa=C3=9F?= Date: Wed, 15 Aug 2018 23:24:26 +0200 Subject: [PATCH 24/28] Improve error messages in case of sqlite errors --- beets/ui/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index af2b79a19..8345e40f0 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1193,10 +1193,11 @@ def _open_library(config): get_replacements(), ) lib.get_item(0) # Test database connection. - except (sqlite3.OperationalError, sqlite3.DatabaseError): + except (sqlite3.OperationalError, sqlite3.DatabaseError) as db_error: log.debug(u'{}', traceback.format_exc()) - raise UserError(u"database file {0} could not be opened".format( - util.displayable_path(dbpath) + raise UserError(u"database file {0} could not be opened, reason: {1}".format( + util.displayable_path(dbpath), + db_error )) log.debug(u'library database: {0}\n' u'library directory: {1}', From 9f6f1987ab6d201ba447e8ef91ed6b0da6d4bc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Haa=C3=9F?= Date: Thu, 16 Aug 2018 00:18:26 +0200 Subject: [PATCH 25/28] CodeReview: change message, add changelog entry, fix line length for style check --- beets/ui/__init__.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 8345e40f0..d3c3dafc2 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1195,7 +1195,7 @@ def _open_library(config): lib.get_item(0) # Test database connection. except (sqlite3.OperationalError, sqlite3.DatabaseError) as db_error: log.debug(u'{}', traceback.format_exc()) - raise UserError(u"database file {0} could not be opened, reason: {1}".format( + raise UserError(u"database file {0} cannot not be opened: {1}".format( util.displayable_path(dbpath), db_error )) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0d4d01535..86f09a476 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -50,6 +50,8 @@ Fixes: with CORS enabled. Thanks to :user:`rveachkc`. :bug:`2979`: :bug:`2980` +* Improve error reporting: during startup if sqlite returns an error the + sqlite error message is attached to the beets message .. _python-itunes: https://github.com/ocelma/python-itunes From e90eb629c96181390cb8547e5757cf78b61a6a1e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 15 Aug 2018 15:30:53 -0700 Subject: [PATCH 26/28] Reference bug #3005 --- docs/changelog.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 86f09a476..45d8220fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -51,7 +51,8 @@ Fixes: Thanks to :user:`rveachkc`. :bug:`2979`: :bug:`2980` * Improve error reporting: during startup if sqlite returns an error the - sqlite error message is attached to the beets message + sqlite error message is attached to the beets message. + :bug:`3005` .. _python-itunes: https://github.com/ocelma/python-itunes From f58f03dbfd990dbad69e0e24fb52aeccd7a9f125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Haa=C3=9F?= Date: Mon, 20 Aug 2018 22:59:52 +0200 Subject: [PATCH 27/28] replaygain: albumpeak on large collections is calculated as average, not maximum (bug 3008) --- beetsplug/replaygain.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 35229281d..75f1714aa 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -181,7 +181,7 @@ class Bs1770gainBackend(Backend): i += 1 returnchunk = self.compute_chunk_gain(chunk, is_album) albumgaintot += returnchunk[-1].gain - albumpeaktot += returnchunk[-1].peak + albumpeaktot = max(albumpeaktot, returnchunk[-1].peak) returnchunks = returnchunks + returnchunk[0:-1] returnchunks.append(Gain(albumgaintot / i, albumpeaktot / i)) return returnchunks diff --git a/docs/changelog.rst b/docs/changelog.rst index 45d8220fe..4091f6b90 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,8 @@ New features: * The `absubmit` plugin now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. :bug:`2442` +* replaygain: albumpeak on large collections is calculated as average, not maximum + :bug:`3008` Fixes: From a3770686b4987bf8b50001caa8438a4bbca96284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Haa=C3=9F?= Date: Mon, 20 Aug 2018 23:38:12 +0200 Subject: [PATCH 28/28] to fix the peak calculation also delete the division --- beetsplug/replaygain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 75f1714aa..ac45aa4f8 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -183,7 +183,7 @@ class Bs1770gainBackend(Backend): albumgaintot += returnchunk[-1].gain albumpeaktot = max(albumpeaktot, returnchunk[-1].peak) returnchunks = returnchunks + returnchunk[0:-1] - returnchunks.append(Gain(albumgaintot / i, albumpeaktot / i)) + returnchunks.append(Gain(albumgaintot / i, albumpeaktot)) return returnchunks else: return self.compute_chunk_gain(items, is_album)