From dd25e1a82564f7375dabff2f9d9a5bef1f986d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Wed, 25 Aug 2021 22:44:48 +0200 Subject: [PATCH 001/357] Add option to save resized images w/o interlace. Add option to save images resized using PIL or ImageMagick as non-progressive. --- beets/util/artresizer.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index bf6254c81..c00ed0489 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -64,7 +64,10 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): +def pil_resize( + maxwidth, path_in, path_out=None, + quality=0, max_filesize=0, deinterlace=False +): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -83,7 +86,12 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # Use PIL's default quality. quality = -1 - im.save(util.py3_path(path_out), quality=quality) + if not deinterlace: + im.save(util.py3_path(path_out), quality=quality) + else: + im.save(util.py3_path(path_out), quality=quality, + progressive=False) + if max_filesize > 0: # If maximum filesize is set, we attempt to lower the quality of # jpeg conversion by a proportional amount, up to 3 attempts @@ -105,9 +113,12 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): if lower_qual < 10: lower_qual = 10 # Use optimize flag to improve filesize decrease - im.save( - util.py3_path(path_out), quality=lower_qual, optimize=True - ) + if not deinterlace: + im.save(util.py3_path(path_out), quality=lower_qual, + optimize=True) + else: + im.save(util.py3_path(path_out), quality=lower_qual, + optimize=True, progressive=False) log.warning(u"PIL Failed to resize file to below {0}B", max_filesize) return path_out @@ -120,7 +131,8 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): return path_in -def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): +def im_resize(maxwidth, path_in, path_out=None, + quality=0, max_filesize=0, deinterlace=False): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -146,6 +158,9 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): if max_filesize > 0: cmd += ['-define', 'jpeg:extent={0}b'.format(max_filesize)] + if deinterlace: + cmd += ['-interlace', 'none'] + cmd.append(util.syspath(path_out, prefix=False)) try: @@ -243,7 +258,8 @@ class ArtResizer(six.with_metaclass(Shareable, object)): self.im_identify_cmd = ['magick', 'identify'] def resize( - self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 + self, maxwidth, path_in, path_out=None, + quality=0, max_filesize=0, deinterlace=False ): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a @@ -253,7 +269,8 @@ class ArtResizer(six.with_metaclass(Shareable, object)): if self.local: func = BACKEND_FUNCS[self.method[0]] return func(maxwidth, path_in, path_out, - quality=quality, max_filesize=max_filesize) + quality=quality, max_filesize=max_filesize, + deinterlace=deinterlace) else: return path_in From cf31dbf6da2e999609e67d48eb537bacb355d9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Wed, 25 Aug 2021 22:54:50 +0200 Subject: [PATCH 002/357] Add function to convert images to non-progressive. --- beets/util/artresizer.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index c00ed0489..e91f31b47 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -216,6 +216,40 @@ BACKEND_GET_SIZE = { } +def pil_deinterlace(path_in, path_out=None): + path_out = path_out or temp_file_for(path_in) + from PIL import Image + + try: + im = Image.open(util.syspath(path_in)) + im.save(util.py3_path(path_out), progressive=False) + return path_out + except IOError: + return path_in + + +def im_deinterlace(path_in, path_out=None): + path_out = path_out or temp_file_for(path_in) + + cmd = ArtResizer.shared.im_convert_cmd + [ + util.syspath(path_in, prefix=False), + '-interlace', 'none', + util.syspath(path_out, prefix=False), + ] + + try: + util.command_output(cmd) + return path_out + except subprocess.CalledProcessError: + return path_in + + +DEINTERLACE_FUNCS = { + PIL: pil_deinterlace, + IMAGEMAGICK: im_deinterlace, +} + + class Shareable(type): """A pseudo-singleton metaclass that allows both shared and non-shared instances. The ``MyClass.shared`` property holds a @@ -274,6 +308,13 @@ class ArtResizer(six.with_metaclass(Shareable, object)): else: return path_in + def deinterlace(self, path_in, path_out=None): + if self.local: + func = DEINTERLACE_FUNCS[self.method[0]] + return func(path_in, path_out) + else: + return path_in + def proxy_url(self, maxwidth, url, quality=0): """Modifies an image URL according the method, returning a new URL. For WEBPROXY, a URL on the proxy server is returned. From c6e05e8ab70cb18861f2d5083fd1a0808b5e032e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Wed, 25 Aug 2021 22:55:54 +0200 Subject: [PATCH 003/357] Add tests to check deinterlace functionality. --- test/test_art_resize.py | 43 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/test/test_art_resize.py b/test/test_art_resize.py index f7090d5e7..bdde0a1ce 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -23,12 +23,15 @@ import os from test import _common from test.helper import TestHelper -from beets.util import syspath +from beets.util import command_output, syspath from beets.util.artresizer import ( pil_resize, im_resize, get_im_version, get_pil_version, + pil_deinterlace, + im_deinterlace, + ArtResizer, ) @@ -90,6 +93,18 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): self.assertLess(os.stat(syspath(im_b)).st_size, os.stat(syspath(im_75_qual)).st_size) + # check if new deinterlace parameter breaks resize + im_di = resize_func( + 225, + self.IMG_225x225, + quality=95, + max_filesize=0, + deinterlace=True, + ) + # check valid path returned - deinterlace hasn't broken resize command + self.assertExists(im_di) + + @unittest.skipUnless(get_pil_version(), "PIL not available") def test_pil_file_resize(self): """Test PIL resize function is lowering file size.""" @@ -100,6 +115,32 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): """Test IM resize function is lowering file size.""" self._test_img_resize(im_resize) + @unittest.skipUnless(get_pil_version(), "PIL not available") + def test_pil_file_deinterlace(self): + """Test PIL deinterlace function. + + Check if pil_deinterlace function returns images + that are non-progressive + """ + path = pil_deinterlace(self.IMG_225x225) + from PIL import Image + with Image.open(path) as img: + self.assertFalse('progression' in img.info) + + @unittest.skipUnless(get_im_version(), "ImageMagick not available") + def test_im_file_deinterlace(self): + """Test ImageMagick deinterlace function. + + Check if im_deinterlace function returns images + that are non-progressive. + """ + path = im_deinterlace(self.IMG_225x225) + cmd = ArtResizer.shared.im_identify_cmd + [ + '-format', '%[interlace]', syspath(path, prefix=False), + ] + out = command_output(cmd).stdout + self.assertTrue(out == b'None') + def suite(): """Run this suite of tests.""" From 0a0719f9eddac773883f2fb239f1ca247b8c53eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Wed, 25 Aug 2021 22:56:46 +0200 Subject: [PATCH 004/357] Add plugin option to store cover art non-progressive. --- beetsplug/fetchart.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 0a3254f53..d63970222 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -52,6 +52,7 @@ class Candidate(object): CANDIDATE_EXACT = 1 CANDIDATE_DOWNSCALE = 2 CANDIDATE_DOWNSIZE = 3 + CANDIDATE_DEINTERLACE = 4 MATCH_EXACT = 0 MATCH_FALLBACK = 1 @@ -80,7 +81,7 @@ class Candidate(object): return self.CANDIDATE_BAD if (not (plugin.enforce_ratio or plugin.minwidth or plugin.maxwidth - or plugin.max_filesize)): + or plugin.max_filesize or plugin.deinterlace)): return self.CANDIDATE_EXACT # get_size returns None if no local imaging backend is available @@ -147,6 +148,8 @@ class Candidate(object): return self.CANDIDATE_DOWNSCALE elif downsize: return self.CANDIDATE_DOWNSIZE + elif plugin.deinterlace: + return self.CANDIDATE_DEINTERLACE else: return self.CANDIDATE_EXACT @@ -159,13 +162,18 @@ class Candidate(object): self.path = \ ArtResizer.shared.resize(plugin.maxwidth, self.path, quality=plugin.quality, - max_filesize=plugin.max_filesize) + max_filesize=plugin.max_filesize, + deinterlace=plugin.deinterlace) elif self.check == self.CANDIDATE_DOWNSIZE: # dimensions are correct, so maxwidth is set to maximum dimension self.path = \ ArtResizer.shared.resize(max(self.size), self.path, quality=plugin.quality, - max_filesize=plugin.max_filesize) + max_filesize=plugin.max_filesize, + deinterlace=plugin.deinterlace) + elif self.check == self.CANDIDATE_DEINTERLACE: + self.path = \ + ArtResizer.shared.deinterlace(self.path) def _logged_get(log, *args, **kwargs): @@ -932,6 +940,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'lastfm_key': None, 'store_source': False, 'high_resolution': False, + 'deinterlace': False, }) self.config['google_key'].redact = True self.config['fanarttv_key'].redact = True @@ -949,6 +958,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): confuse.String(pattern=self.PAT_PERCENT)])) self.margin_px = None self.margin_percent = None + self.deinterlace = self.config['deinterlace'].get(bool) if type(self.enforce_ratio) is six.text_type: if self.enforce_ratio[-1] == u'%': self.margin_percent = float(self.enforce_ratio[:-1]) / 100 From 31d9c80e1875a3b4b64fc050f8b6deeaca225863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Wed, 25 Aug 2021 22:58:04 +0200 Subject: [PATCH 005/357] Update docs/ to reflect new option to deinterlace. Document new option to fetchart plugin to store cover art as non-progressive. --- docs/plugins/fetchart.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 6344c1562..8cefda9cd 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -86,8 +86,10 @@ file. The available options are: - **high_resolution**: If enabled, fetchart retrieves artwork in the highest resolution it can find (warning: image files can sometimes reach >20MB). Default: ``no``. +- **deinterlace**: If enabled, convert images to non-progressive/baseline. + Default: ``no``. -Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ +Note: ``maxwidth``, ``enforce_ratio`` and ``deinterlace`` options require either `ImageMagick`_ or `Pillow`_. .. note:: From 54522e6908e3c4f54b0199422412b6fc3fef62e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Thu, 26 Aug 2021 13:18:15 +0200 Subject: [PATCH 006/357] Remove extraneous empty line that failed lint. --- test/test_art_resize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_art_resize.py b/test/test_art_resize.py index bdde0a1ce..11175b5f7 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -104,7 +104,6 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): # check valid path returned - deinterlace hasn't broken resize command self.assertExists(im_di) - @unittest.skipUnless(get_pil_version(), "PIL not available") def test_pil_file_resize(self): """Test PIL resize function is lowering file size.""" From 7c1de83ef284755851cb44fbdaeffd9191658f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Thu, 26 Aug 2021 13:38:17 +0200 Subject: [PATCH 007/357] Extend fetchart documentation to cover new option. --- docs/plugins/fetchart.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 8cefda9cd..2ee7c12a8 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -87,9 +87,12 @@ file. The available options are: resolution it can find (warning: image files can sometimes reach >20MB). Default: ``no``. - **deinterlace**: If enabled, convert images to non-progressive/baseline. +- **deinterlace**: If enabled, `Pillow`_ or `ImageMagick`_ backends are + instructed to store cover art as non-progressive. This might be preferred for + DAPs that don't support progressive images. Default: ``no``. -Note: ``maxwidth``, ``enforce_ratio`` and ``deinterlace`` options require either `ImageMagick`_ +Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ or `Pillow`_. .. note:: From 1a3ecc1ef41516a9fba2828bf62dbe19382d006b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Thu, 26 Aug 2021 13:38:56 +0200 Subject: [PATCH 008/357] Add to changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0e1cc30a5..3c8d199dd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,10 @@ Changelog This release now requires Python 3.6 or later (it removes support for Python 2.7, 3.4, and 3.5). +* A new :doc:`/plugins/fetchart` option to store cover art as non-progressive + image. Useful for DAPs that support progressive images. Set ``deinterlace: + yes`` in your configuration to enable. + 1.5.0 (August 19, 2021) ----------------------- From bee4f13d8426e955e214d33a17d41f6d704e4820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Thu, 26 Aug 2021 14:43:37 +0200 Subject: [PATCH 009/357] Remove duplicate line. --- docs/plugins/fetchart.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 2ee7c12a8..3605ff27c 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -86,7 +86,6 @@ file. The available options are: - **high_resolution**: If enabled, fetchart retrieves artwork in the highest resolution it can find (warning: image files can sometimes reach >20MB). Default: ``no``. -- **deinterlace**: If enabled, convert images to non-progressive/baseline. - **deinterlace**: If enabled, `Pillow`_ or `ImageMagick`_ backends are instructed to store cover art as non-progressive. This might be preferred for DAPs that don't support progressive images. From 901377ded553989ca4b36811478b086dc6f8f58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Thu, 26 Aug 2021 15:01:37 +0200 Subject: [PATCH 010/357] Improve wording. --- docs/plugins/fetchart.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 3605ff27c..5df6c6e34 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -87,8 +87,8 @@ file. The available options are: resolution it can find (warning: image files can sometimes reach >20MB). Default: ``no``. - **deinterlace**: If enabled, `Pillow`_ or `ImageMagick`_ backends are - instructed to store cover art as non-progressive. This might be preferred for - DAPs that don't support progressive images. + instructed to store cover art as non-progressive JPEG. You might need this if + you use DAPs that don't support progressive images. Default: ``no``. Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ From d30c8b32b52b6f06e6a43ee469d453b6a349a42e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Thu, 26 Aug 2021 15:02:51 +0200 Subject: [PATCH 011/357] Remove unnecessary line break. --- beetsplug/fetchart.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index d63970222..d6fff6316 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -172,8 +172,7 @@ class Candidate(object): max_filesize=plugin.max_filesize, deinterlace=plugin.deinterlace) elif self.check == self.CANDIDATE_DEINTERLACE: - self.path = \ - ArtResizer.shared.deinterlace(self.path) + self.path = ArtResizer.shared.deinterlace(self.path) def _logged_get(log, *args, **kwargs): From 1f6f0022c527246092b516668e519ad5aab1963d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Tue, 31 Aug 2021 09:22:28 +0200 Subject: [PATCH 012/357] Remove unnecessary if-else As @ArsenArsen noted progressive defaults to false in the JPEG writer making the if-else pointless. --- beets/util/artresizer.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index e91f31b47..f244e0820 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -86,11 +86,8 @@ def pil_resize( # Use PIL's default quality. quality = -1 - if not deinterlace: - im.save(util.py3_path(path_out), quality=quality) - else: - im.save(util.py3_path(path_out), quality=quality, - progressive=False) + im.save(util.py3_path(path_out), quality=quality, + progressive=not deinterlace) if max_filesize > 0: # If maximum filesize is set, we attempt to lower the quality of @@ -113,12 +110,8 @@ def pil_resize( if lower_qual < 10: lower_qual = 10 # Use optimize flag to improve filesize decrease - if not deinterlace: - im.save(util.py3_path(path_out), quality=lower_qual, - optimize=True) - else: - im.save(util.py3_path(path_out), quality=lower_qual, - optimize=True, progressive=False) + im.save(util.py3_path(path_out), quality=lower_qual, + optimize=True, progressive=not deinterlace) log.warning(u"PIL Failed to resize file to below {0}B", max_filesize) return path_out From 68a1407c67e273934de888340ea3e20ca63ce028 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Tue, 5 Oct 2021 13:03:07 +1000 Subject: [PATCH 013/357] Update CI to the latest 3.10 release --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 137f74b72..0299afaf1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc.2] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.9' && matrix.python-version != '3.10.0-rc.2' + if: matrix.python-version != '3.9' && matrix.python-version != '3.10' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.10.0-rc.2' + if: matrix.python-version == '3.10' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 From eafef0d521c8205183082fa05de5d3710a3721e7 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Tue, 5 Oct 2021 13:10:09 +1000 Subject: [PATCH 014/357] Versioning --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0299afaf1..f3a44fc8c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.9' && matrix.python-version != '3.10' + if: matrix.python-version != '3.9' && matrix.python-version != '3.10.0' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.10.0' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 From 400aece2a5612814fe1b7fc0ff779441be998599 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Wed, 6 Oct 2021 07:35:05 +1000 Subject: [PATCH 015/357] Add quotes around versions --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f3a44fc8c..fb1d42ceb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.9' && matrix.python-version != '3.10.0' + if: matrix.python-version != '3.9' && matrix.python-version != '3.10' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.10.0' + if: matrix.python-version == '3.10' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 From d8cff030a636f0e5be31dab1d736582f23c3b895 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 26 Oct 2021 09:00:31 -0700 Subject: [PATCH 016/357] Add a basic "--album" flag to "beet info" This currently implies "--library" because I'm not sure what "album info" of the tags of individual files would mean. --- beetsplug/info.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/beetsplug/info.py b/beetsplug/info.py index 1bb29d09b..de4af9fa3 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -25,7 +25,7 @@ from beets.library import Item from beets.util import displayable_path, normpath, syspath -def tag_data(lib, args): +def tag_data(opts, lib, args): query = [] for arg in args: path = normpath(arg) @@ -69,8 +69,8 @@ def tag_data_emitter(path): return emitter -def library_data(lib, args): - for item in lib.items(args): +def library_data(opts, lib, args): + for item in lib.albums(args) if opts.album else lib.items(args): yield library_data_emitter(item) @@ -156,6 +156,10 @@ class InfoPlugin(BeetsPlugin): '-l', '--library', action='store_true', help='show library fields instead of tags', ) + cmd.parser.add_option( + '-a', '--album', action='store_true', + help='show album fields instead of tracks (implies "--library")', + ) cmd.parser.add_option( '-s', '--summarize', action='store_true', help='summarize the tags of all files', @@ -186,7 +190,7 @@ class InfoPlugin(BeetsPlugin): dictionary and only prints that. If two files have different values for the same tag, the value is set to '[various]' """ - if opts.library: + if opts.library or opts.album: data_collector = library_data else: data_collector = tag_data @@ -199,7 +203,7 @@ class InfoPlugin(BeetsPlugin): first = True summary = {} - for data_emitter in data_collector(lib, ui.decargs(args)): + for data_emitter in data_collector(opts, lib, ui.decargs(args)): try: data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, OSError) as ex: From 5c4147826319ff3d7343901ddf7d851ade8246e4 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 26 Oct 2021 09:09:44 -0700 Subject: [PATCH 017/357] Add "beet info --album" to the plugin documentation too --- docs/plugins/info.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins/info.rst b/docs/plugins/info.rst index 3950cf0aa..1ed7582af 100644 --- a/docs/plugins/info.rst +++ b/docs/plugins/info.rst @@ -31,6 +31,8 @@ Additional command-line options include: * ``--library`` or ``-l``: Show data from the library database instead of the files' tags. +* ``--album`` or ``-a``: Show data from albums instead of tracks (implies + ``--library``). * ``--summarize`` or ``-s``: Merge all the information from multiple files into a single list of values. If the tags differ across the files, print ``[various]``. From a5b5831d2876b2093c5bfd6b09a7ac9c7d32101a Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 27 Oct 2021 08:27:39 -0700 Subject: [PATCH 018/357] Switch "beet info --album" approach to use a named argument instead These functions are also used by the export plugin, so it's useful if the signature stays backwards compatible. --- beetsplug/info.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/beetsplug/info.py b/beetsplug/info.py index de4af9fa3..1e6d4b329 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -25,7 +25,7 @@ from beets.library import Item from beets.util import displayable_path, normpath, syspath -def tag_data(opts, lib, args): +def tag_data(lib, args, album=False): query = [] for arg in args: path = normpath(arg) @@ -69,8 +69,8 @@ def tag_data_emitter(path): return emitter -def library_data(opts, lib, args): - for item in lib.albums(args) if opts.album else lib.items(args): +def library_data(lib, args, album=False): + for item in lib.albums(args) if album else lib.items(args): yield library_data_emitter(item) @@ -203,7 +203,10 @@ class InfoPlugin(BeetsPlugin): first = True summary = {} - for data_emitter in data_collector(opts, lib, ui.decargs(args)): + for data_emitter in data_collector( + lib, ui.decargs(args), + album=opts.album, + ): try: data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, OSError) as ex: From 89a7cc37011b8a75f2a466d0d6850b6b8510c7b9 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 27 Oct 2021 08:30:05 -0700 Subject: [PATCH 019/357] Add "beet info --album" changelog entry --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 87dc04a83..b0234023a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,7 @@ Other new things: * Permissions plugin now sets cover art permissions to the file permissions. * :doc:`/plugins/unimported`: Support excluding specific subdirectories in library. +* :doc:`/plugins/info`: Support ``--album`` flag. Bug fixes: From 7bd36ed6ca0b4dcc5f08a23b3f45ef791475f3b7 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 27 Oct 2021 08:55:36 -0700 Subject: [PATCH 020/357] Add "beet export --album" (matching "beet info --album") --- beetsplug/export.py | 16 +++++++++++++--- docs/changelog.rst | 1 + docs/plugins/export.rst | 3 +++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 3cb8f8c4f..0d7049054 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -87,6 +87,10 @@ class ExportPlugin(BeetsPlugin): '-l', '--library', action='store_true', help='show library fields instead of tags', ) + cmd.parser.add_option( + '-a', '--album', action='store_true', + help='show album fields instead of tracks (implies "--library")', + ) cmd.parser.add_option( '--append', action='store_true', default=False, help='if should append data to the file', @@ -121,14 +125,20 @@ class ExportPlugin(BeetsPlugin): } ) - items = [] - data_collector = library_data if opts.library else tag_data + if opts.library or opts.album: + data_collector = library_data + else: + data_collector = tag_data included_keys = [] for keys in opts.included_keys: included_keys.extend(keys.split(',')) - for data_emitter in data_collector(lib, ui.decargs(args)): + items = [] + for data_emitter in data_collector( + lib, ui.decargs(args), + album=opts.album, + ): try: data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, OSError) as ex: diff --git a/docs/changelog.rst b/docs/changelog.rst index b0234023a..14908da77 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -36,6 +36,7 @@ Other new things: * :doc:`/plugins/unimported`: Support excluding specific subdirectories in library. * :doc:`/plugins/info`: Support ``--album`` flag. +* :doc:`/plugins/export`: Support ``--album`` flag. Bug fixes: diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index 284d2b8b6..bca9d1e5a 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -34,6 +34,9 @@ The ``export`` command has these command-line options: * ``--library`` or ``-l``: Show data from the library database instead of the files' tags. +* ``--album`` or ``-a``: Show data from albums instead of tracks (implies + ``--library``). + * ``--output`` or ``-o``: Path for an output file. If not informed, will print the data in the console. From 9ddc75035a1b6c534fa0337e430df7909e09c1ee Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 27 Oct 2021 09:06:27 -0700 Subject: [PATCH 021/357] Fix duplicated output in "beet export" --- beetsplug/export.py | 2 -- docs/changelog.rst | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 0d7049054..bb2c9ba28 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -149,8 +149,6 @@ class ExportPlugin(BeetsPlugin): if isinstance(value, bytes): data[key] = util.displayable_path(value) - items += [data] - if file_format_is_line_based: export_format.export(data, **format_options) else: diff --git a/docs/changelog.rst b/docs/changelog.rst index 14908da77..cab5d06dd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -49,6 +49,8 @@ Bug fixes: * :doc:`/plugins/discogs`: Remove requests ratel imit code from plugin in favor of discogs library built-in capability :bug: `4108` +* :doc:`/plugins/export`: Fix duplicated output. + 1.5.0 (August 19, 2021) ----------------------- From 86465e6437ef7f3344046defe4d4b0f6de3890a8 Mon Sep 17 00:00:00 2001 From: Christopher Larson Date: Tue, 12 Oct 2021 15:39:33 -0700 Subject: [PATCH 022/357] Allow custom replacements in Item.destination This allows for the use of differing replacements for destinations other than the library, which is useful for beets-alternatives in the case where filesystem requirements differ between the two paths. Signed-off-by: Christopher Larson --- beets/library.py | 6 ++++-- docs/changelog.rst | 5 +++++ test/test_library.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index d94468800..df1b8ffee 100644 --- a/beets/library.py +++ b/beets/library.py @@ -938,7 +938,7 @@ class Item(LibModel): # Templating. def destination(self, fragment=False, basedir=None, platform=None, - path_formats=None): + path_formats=None, replacements=None): """Returns the path in the library directory designated for the item (i.e., where the file ought to be). fragment makes this method return just the path fragment underneath the root library @@ -950,6 +950,8 @@ class Item(LibModel): platform = platform or sys.platform basedir = basedir or self._db.directory path_formats = path_formats or self._db.path_formats + if replacements is None: + replacements = self._db.replacements # Use a path format based on a query, falling back on the # default. @@ -994,7 +996,7 @@ class Item(LibModel): maxlen = util.max_filename_length(self._db.directory) subpath, fellback = util.legalize_path( - subpath, self._db.replacements, maxlen, + subpath, replacements, maxlen, os.path.splitext(self.path)[1], fragment ) if fellback: diff --git a/docs/changelog.rst b/docs/changelog.rst index 87dc04a83..1b345650d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -36,6 +36,11 @@ Other new things: * :doc:`/plugins/unimported`: Support excluding specific subdirectories in library. +For plugin developers: + +* :py:meth:`beets.library.Item.destination` now accepts a `replacements` + argument to be used in favor of the default. + Bug fixes: * :doc:`/plugins/lyrics`: Fix crash bug when beautifulsoup4 is not installed. diff --git a/test/test_library.py b/test/test_library.py index 8b04e50d0..e64f75561 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -449,6 +449,16 @@ class DestinationTest(_common.TestCase): self.assertEqual(self.i.destination(), np('base/ber/foo')) + def test_destination_with_replacements_argument(self): + self.lib.directory = b'base' + self.lib.replacements = [(re.compile(r'a'), 'f')] + self.lib.path_formats = [('default', '$album/$title')] + self.i.title = 'foo' + self.i.album = 'bar' + replacements = [(re.compile(r'a'), 'e')] + self.assertEqual(self.i.destination(replacements=replacements), + np('base/ber/foo')) + @unittest.skip('unimplemented: #359') def test_destination_with_empty_component(self): self.lib.directory = b'base' From 786236f046ae5f2b5396aa6e553eace953f8be44 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 30 Oct 2021 12:41:04 +0200 Subject: [PATCH 023/357] remove the gmusic plugin --- beetsplug/gmusic.py | 118 ++-------------------------------------- docs/changelog.rst | 3 + docs/plugins/gmusic.rst | 86 +---------------------------- docs/plugins/index.rst | 1 - setup.py | 1 - 5 files changed, 9 insertions(+), 200 deletions(-) diff --git a/beetsplug/gmusic.py b/beetsplug/gmusic.py index 1761dbb13..844234f94 100644 --- a/beetsplug/gmusic.py +++ b/beetsplug/gmusic.py @@ -1,5 +1,4 @@ # This file is part of beets. -# Copyright 2017, Tigran Kostandyan. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -12,124 +11,15 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Upload files to Google Play Music and list songs in its library.""" - -import os.path +"""Deprecation warning for the removed gmusic plugin.""" from beets.plugins import BeetsPlugin -from beets import ui -from beets import config -from beets.ui import Subcommand -from gmusicapi import Musicmanager, Mobileclient -from gmusicapi.exceptions import NotLoggedIn -import gmusicapi.clients class Gmusic(BeetsPlugin): def __init__(self): super().__init__() - self.m = Musicmanager() - # OAUTH_FILEPATH was moved in gmusicapi 12.0.0. - if hasattr(Musicmanager, 'OAUTH_FILEPATH'): - oauth_file = Musicmanager.OAUTH_FILEPATH - else: - oauth_file = gmusicapi.clients.OAUTH_FILEPATH - - self.config.add({ - 'auto': False, - 'uploader_id': '', - 'uploader_name': '', - 'device_id': '', - 'oauth_file': oauth_file, - }) - if self.config['auto']: - self.import_stages = [self.autoupload] - - def commands(self): - gupload = Subcommand('gmusic-upload', - help='upload your tracks to Google Play Music') - gupload.func = self.upload - - search = Subcommand('gmusic-songs', - help='list of songs in Google Play Music library') - search.parser.add_option('-t', '--track', dest='track', - action='store_true', - help='Search by track name') - search.parser.add_option('-a', '--artist', dest='artist', - action='store_true', - help='Search by artist') - search.func = self.search - return [gupload, search] - - def authenticate(self): - if self.m.is_authenticated(): - return - # Checks for OAuth2 credentials, - # if they don't exist - performs authorization - oauth_file = self.config['oauth_file'].as_filename() - if os.path.isfile(oauth_file): - uploader_id = self.config['uploader_id'] - uploader_name = self.config['uploader_name'] - self.m.login(oauth_credentials=oauth_file, - uploader_id=uploader_id.as_str().upper() or None, - uploader_name=uploader_name.as_str() or None) - else: - self.m.perform_oauth(oauth_file) - - def upload(self, lib, opts, args): - items = lib.items(ui.decargs(args)) - files = self.getpaths(items) - self.authenticate() - ui.print_('Uploading your files...') - self.m.upload(filepaths=files) - ui.print_('Your files were successfully added to library') - - def autoupload(self, session, task): - items = task.imported_items() - files = self.getpaths(items) - self.authenticate() - self._log.info('Uploading files to Google Play Music...', files) - self.m.upload(filepaths=files) - self._log.info('Your files were successfully added to your ' - + 'Google Play Music library') - - def getpaths(self, items): - return [x.path for x in items] - - def search(self, lib, opts, args): - password = config['gmusic']['password'] - email = config['gmusic']['email'] - uploader_id = config['gmusic']['uploader_id'] - device_id = config['gmusic']['device_id'] - password.redact = True - email.redact = True - # Since Musicmanager doesn't support library management - # we need to use mobileclient interface - mobile = Mobileclient() - try: - new_device_id = (device_id.as_str() - or uploader_id.as_str().replace(':', '') - or Mobileclient.FROM_MAC_ADDRESS).upper() - mobile.login(email.as_str(), password.as_str(), new_device_id) - files = mobile.get_all_songs() - except NotLoggedIn: - ui.print_( - 'Authentication error. Please check your email and password.' - ) - return - if not args: - for i, file in enumerate(files, start=1): - print(i, ui.colorize('blue', file['artist']), - file['title'], ui.colorize('red', file['album'])) - else: - if opts.track: - self.match(files, args, 'title') - else: - self.match(files, args, 'artist') - - @staticmethod - def match(files, args, search_by): - for file in files: - if ' '.join(ui.decargs(args)) in file[search_by]: - print(file['artist'], file['title'], file['album']) + self._log.warning("The 'gmusic' plugin has been removed following the" + " shutdown of Google Play Music. Remove the plugin" + " from your configuration to silence this warning.") diff --git a/docs/changelog.rst b/docs/changelog.rst index 72af36e3c..d43b7610f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,9 @@ For packagers: :bug:`4037` :bug:`4038` * This version of beets no longer depends on the `six`_ library. :bug:`4030` +* The `gmusic` plugin was removed since Google Play Music has been shut down. + Thus, the optional dependency on `gmusicapi` does not exist anymore. + :bug:`4089` Major new features: diff --git a/docs/plugins/gmusic.rst b/docs/plugins/gmusic.rst index 94ee2dae4..412978bd6 100644 --- a/docs/plugins/gmusic.rst +++ b/docs/plugins/gmusic.rst @@ -1,87 +1,5 @@ Gmusic Plugin ============= -The ``gmusic`` plugin lets you upload songs to Google Play Music and query -songs in your library. - - -Installation ------------- - -The plugin requires :pypi:`gmusicapi`. You can install it using ``pip``:: - - pip install gmusicapi - -.. _gmusicapi: https://github.com/simon-weber/gmusicapi/ - -Then, you can enable the ``gmusic`` plugin in your configuration (see -:ref:`using-plugins`). - - -Usage ------ -Configuration is required before use. Below is an example configuration:: - - gmusic: - email: user@example.com - password: seekrit - auto: yes - uploader_id: 00:11:22:33:AA:BB - device_id: 00112233AABB - oauth_file: ~/.config/beets/oauth.cred - - -To upload tracks to Google Play Music, use the ``gmusic-upload`` command:: - - beet gmusic-upload [QUERY] - -If you don't include a query, the plugin will upload your entire collection. - -To list your music collection, use the ``gmusic-songs`` command:: - - beet gmusic-songs [-at] [ARGS] - -Use the ``-a`` option to search by artist and ``-t`` to search by track. For -example:: - - beet gmusic-songs -a John Frusciante - beet gmusic-songs -t Black Hole Sun - -For a list of all songs in your library, run ``beet gmusic-songs`` without any -arguments. - - -Configuration -------------- -To configure the plugin, make a ``gmusic:`` section in your configuration file. -The available options are: - -- **email**: Your Google account email address. - Default: none. -- **password**: Password to your Google account. Required to query songs in - your collection. - For accounts with 2-step-verification, an - `app password `__ - will need to be generated. An app password for an account without - 2-step-verification is not required but is recommended. - Default: none. -- **auto**: Set to ``yes`` to automatically upload new imports to Google Play - Music. - Default: ``no`` -- **uploader_id**: Unique id as a MAC address, eg ``00:11:22:33:AA:BB``. - This option should be set before the maximum number of authorized devices is - reached. - If provided, use the same id for all future runs on this, and other, beets - installations as to not reach the maximum number of authorized devices. - Default: device's MAC address. -- **device_id**: Unique device ID for authorized devices. It is usually - the same as your MAC address with the colons removed, eg ``00112233AABB``. - This option only needs to be set if you receive an `InvalidDeviceId` - exception. Below the exception will be a list of valid device IDs. - Default: none. -- **oauth_file**: Filepath for oauth credentials file. - Default: `{user_data_dir} `__/gmusicapi/oauth.cred - -Refer to the `Google Play Music Help -`__ -page for more details on authorized devices. +The ``gmusic`` plugin interfaced beets to Google Play Music. It has been +removed after the shutdown of this service. diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 9c628951a..5ca8794fd 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -231,7 +231,6 @@ Miscellaneous * :doc:`filefilter`: Automatically skip files during the import process based on regular expressions. * :doc:`fuzzy`: Search albums and tracks with fuzzy string matching. -* :doc:`gmusic`: Search and upload files to Google Play Music. * :doc:`hook`: Run a command when an event is emitted by beets. * :doc:`ihate`: Automatically skip albums and tracks during the import process. * :doc:`info`: Print music files' tags to the console. diff --git a/setup.py b/setup.py index 48aede251..e6ff6a592 100755 --- a/setup.py +++ b/setup.py @@ -126,7 +126,6 @@ setup( 'embedart': ['Pillow'], 'embyupdate': ['requests'], 'chroma': ['pyacoustid'], - 'gmusic': ['gmusicapi'], 'discogs': ['python3-discogs-client>=2.3.10'], 'beatport': ['requests-oauthlib>=0.6.1'], 'kodiupdate': ['requests'], From 8d50301be58463ac860cafaf898670fccb525bbe Mon Sep 17 00:00:00 2001 From: "Kirill A. Korinsky" Date: Sat, 30 Oct 2021 20:17:11 +0200 Subject: [PATCH 024/357] Introduce atomic move and write of file The idea of this changes is simple: let move file to some temporary name inside distance folder, and after the file is already copy it renames to expected name. When someone tries to save anything it also moves file to trigger OS level notification for change FS. This commit also enforce that `beets.util.move` shouldn't be used to move directories as it described in comment. Thus, this is fixed #3849 --- beets/util/__init__.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 7ae71164e..d58bb28e4 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -19,6 +19,7 @@ import sys import errno import locale import re +import tempfile import shutil import fnmatch import functools @@ -478,6 +479,11 @@ def move(path, dest, replace=False): instead, in which case metadata will *not* be preserved. Paths are translated to system paths. """ + if os.path.isdir(path): + raise FilesystemError(u'source is directory', 'move', (path, dest)) + if os.path.isdir(dest): + raise FilesystemError(u'destination is directory', 'move', + (path, dest)) if samefile(path, dest): return path = syspath(path) @@ -487,15 +493,23 @@ def move(path, dest, replace=False): # First, try renaming the file. try: - os.rename(path, dest) + os.replace(path, dest) except OSError: - # Otherwise, copy and delete the original. + tmp = tempfile.mktemp(suffix='.beets', + prefix=py3_path(b'.' + os.path.basename(dest)), + dir=py3_path(os.path.dirname(dest))) + tmp = syspath(tmp) try: - shutil.copyfile(path, dest) + shutil.copyfile(path, tmp) + os.replace(tmp, dest) + tmp = None os.remove(path) except OSError as exc: raise FilesystemError(exc, 'move', (path, dest), traceback.format_exc()) + finally: + if tmp is not None: + os.remove(tmp) def link(path, dest, replace=False): From 6e434934d4ecb9f9d060d4b83e254b0b7b04adee Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sat, 30 Oct 2021 11:19:21 -0700 Subject: [PATCH 025/357] Remove completed TODO item --- beetsplug/export.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index bb2c9ba28..99f6d7063 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -79,8 +79,6 @@ class ExportPlugin(BeetsPlugin): }) def commands(self): - # TODO: Add option to use albums - cmd = ui.Subcommand('export', help='export data from beets') cmd.func = self.run cmd.parser.add_option( From 5886aa9247f9673ddf3c0530d0bd01f95ea27848 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 29 Oct 2021 19:43:16 -0700 Subject: [PATCH 026/357] Use "colordiff" to highlight "beet move" path differences --- beets/ui/__init__.py | 12 +++++++++--- docs/changelog.rst | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 8cf6cbbf6..5e0de77e2 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -757,15 +757,21 @@ def show_path_changes(path_changes): if max_width > col_width: # Print every change over two lines for source, dest in zip(sources, destinations): - log.info('{0} \n -> {1}', source, dest) + color_source, color_dest = colordiff(source, dest) + print_('{0} \n -> {1}'.format(color_source, color_dest)) else: # Print every change on a single line, and add a header title_pad = max_width - len('Source ') + len(' -> ') - log.info('Source {0} Destination', ' ' * title_pad) + print_('Source {0} Destination'.format(' ' * title_pad)) for source, dest in zip(sources, destinations): pad = max_width - len(source) - log.info('{0} {1} -> {2}', source, ' ' * pad, dest) + color_source, color_dest = colordiff(source, dest) + print_('{0} {1} -> {2}'.format( + color_source, + ' ' * pad, + color_dest, + )) # Helper functions for option parsing. diff --git a/docs/changelog.rst b/docs/changelog.rst index 72af36e3c..5ce90b624 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,7 @@ Other new things: subdirectories in library. * :doc:`/plugins/info`: Support ``--album`` flag. * :doc:`/plugins/export`: Support ``--album`` flag. +* ``beet move`` path differences are now highlighted in color (when enabled). For plugin developers: From 1a130059e8ef7e1ea613f3e1dd7dbedd802e33fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Mon, 1 Nov 2021 13:51:57 +0100 Subject: [PATCH 027/357] `deinterlace` option to affect when no other processing remove interlacing by default when resizing/down-scaling, the `deinterlace` option is to remove interlace when otherwise no processing would have happened. --- beets/util/artresizer.py | 26 ++++++++++---------------- beetsplug/fetchart.py | 7 +++---- test/test_art_resize.py | 13 +------------ 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index ad9f12f34..1ab689b04 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -58,10 +58,7 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize( - maxwidth, path_in, path_out=None, - quality=0, max_filesize=0, deinterlace=False -): +def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -80,8 +77,9 @@ def pil_resize( # Use PIL's default quality. quality = -1 - im.save(util.py3_path(path_out), quality=quality, - progressive=not deinterlace) + # progressive=False only affects JPEG writter and is the default + # leaving for explicitness + im.save(util.py3_path(path_out), quality=quality, progressive=False) if max_filesize > 0: # If maximum filesize is set, we attempt to lower the quality of @@ -105,7 +103,7 @@ def pil_resize( lower_qual = 10 # Use optimize flag to improve filesize decrease im.save(util.py3_path(path_out), quality=lower_qual, - optimize=True, progressive=not deinterlace) + optimize=True, progressive=False) log.warning("PIL Failed to resize file to below {0}B", max_filesize) return path_out @@ -118,8 +116,7 @@ def pil_resize( return path_in -def im_resize(maxwidth, path_in, path_out=None, - quality=0, max_filesize=0, deinterlace=False): +def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -132,9 +129,11 @@ def im_resize(maxwidth, path_in, path_out=None, # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. + # no interlace seems to be default, specify for explicitness cmd = ArtResizer.shared.im_convert_cmd + [ util.syspath(path_in, prefix=False), '-resize', f'{maxwidth}x>', + '-interlace', 'none', ] if quality > 0: @@ -145,9 +144,6 @@ def im_resize(maxwidth, path_in, path_out=None, if max_filesize > 0: cmd += ['-define', f'jpeg:extent={max_filesize}b'] - if deinterlace: - cmd += ['-interlace', 'none'] - cmd.append(util.syspath(path_out, prefix=False)) try: @@ -279,8 +275,7 @@ class ArtResizer(metaclass=Shareable): self.im_identify_cmd = ['magick', 'identify'] def resize( - self, maxwidth, path_in, path_out=None, - quality=0, max_filesize=0, deinterlace=False + self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 ): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a @@ -290,8 +285,7 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_FUNCS[self.method[0]] return func(maxwidth, path_in, path_out, - quality=quality, max_filesize=max_filesize, - deinterlace=deinterlace) + quality=quality, max_filesize=max_filesize) else: return path_in diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 0cef77210..574e8dae1 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -73,6 +73,7 @@ class Candidate: Return `CANDIDATE_DOWNSCALE` if the file must be rescaled. Return `CANDIDATE_DOWNSIZE` if the file must be resized, and possibly also rescaled. + Return `CANDIDATE_DEINTERLACE` if the file must be deinterlaced. """ if not self.path: return self.CANDIDATE_BAD @@ -159,15 +160,13 @@ class Candidate: self.path = \ ArtResizer.shared.resize(plugin.maxwidth, self.path, quality=plugin.quality, - max_filesize=plugin.max_filesize, - deinterlace=plugin.deinterlace) + max_filesize=plugin.max_filesize) elif self.check == self.CANDIDATE_DOWNSIZE: # dimensions are correct, so maxwidth is set to maximum dimension self.path = \ ArtResizer.shared.resize(max(self.size), self.path, quality=plugin.quality, - max_filesize=plugin.max_filesize, - deinterlace=plugin.deinterlace) + max_filesize=plugin.max_filesize) elif self.check == self.CANDIDATE_DEINTERLACE: self.path = ArtResizer.shared.deinterlace(self.path) diff --git a/test/test_art_resize.py b/test/test_art_resize.py index 9bc8f2eff..73847e0a6 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -90,17 +90,6 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): self.assertLess(os.stat(syspath(im_b)).st_size, os.stat(syspath(im_75_qual)).st_size) - # check if new deinterlace parameter breaks resize - im_di = resize_func( - 225, - self.IMG_225x225, - quality=95, - max_filesize=0, - deinterlace=True, - ) - # check valid path returned - deinterlace hasn't broken resize command - self.assertExists(im_di) - @unittest.skipUnless(get_pil_version(), "PIL not available") def test_pil_file_resize(self): """Test PIL resize function is lowering file size.""" @@ -135,7 +124,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): '-format', '%[interlace]', syspath(path, prefix=False), ] out = command_output(cmd).stdout - self.assertTrue(out == 'None') + self.assertTrue(out == b'None') def suite(): From fbc2862ff06a4948364afe7a366516b51a98af8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotrek=20Kocha=C5=84ski?= Date: Mon, 1 Nov 2021 18:35:45 +0100 Subject: [PATCH 028/357] Improve style and clarity of comments --- beets/util/artresizer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 1ab689b04..f9381f6c4 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -77,8 +77,8 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # Use PIL's default quality. quality = -1 - # progressive=False only affects JPEG writter and is the default - # leaving for explicitness + # progressive=False only affects JPEGs and is the default, + # but we include it here for explicitness. im.save(util.py3_path(path_out), quality=quality, progressive=False) if max_filesize > 0: @@ -129,7 +129,8 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. - # no interlace seems to be default, specify for explicitness + # ImageMagick already seems to default to no interlace, but we include it + # here for the sake of explicitness. cmd = ArtResizer.shared.im_convert_cmd + [ util.syspath(path_in, prefix=False), '-resize', f'{maxwidth}x>', From 5578d0713b3a9b60450ce3fb11c8e4a008001f7e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 1 Nov 2021 19:00:07 +0100 Subject: [PATCH 029/357] update changelog for #4060 --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b7da07673..09e4477c4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,6 +41,11 @@ Other new things: * :doc:`/plugins/info`: Support ``--album`` flag. * :doc:`/plugins/export`: Support ``--album`` flag. * ``beet move`` path differences are now highlighted in color (when enabled). +* When moving files and a direct rename of a file is not possible, beets now + copies to a temporary file in the target folder first instead of directly + using the target path. This gets us closer to always updating files + atomically. Thanks to :user:`catap`. + :bug:`4060` For plugin developers: From 0b578a3384f21dc81ea924aff5b05ef7870bd5eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arsen=20Arsenovi=C4=87?= Date: Tue, 2 Nov 2021 12:51:27 +0100 Subject: [PATCH 030/357] fetchart: add option to force cover format --- beets/util/artresizer.py | 107 ++++++++++++++++++++++++++++++++++++++- beetsplug/fetchart.py | 27 +++++++++- 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index f9381f6c4..8683e2287 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -18,6 +18,7 @@ public resizing proxy if neither is available. import subprocess import os +import os.path import re from tempfile import NamedTemporaryFile from urllib.parse import urlencode @@ -234,6 +235,72 @@ DEINTERLACE_FUNCS = { } +def im_get_format(filepath): + cmd = ArtResizer.shared.im_identify_cmd + [ + '-format', '%[magick]', + util.syspath(filepath) + ] + + try: + return util.command_output(cmd).stdout + except subprocess.CalledProcessError: + return None + + +def pil_get_format(filepath): + from PIL import Image, UnidentifiedImageError + + try: + with Image.open(util.syspath(filepath)) as im: + return im.format + except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError): + log.exception("failed to detect image format for {}", filepath) + return None + + +BACKEND_GET_FORMAT = { + PIL: pil_get_format, + IMAGEMAGICK: im_get_format, +} + + +def im_convert_format(source, target, deinterlaced): + cmd = ArtResizer.shared.im_convert_cmd + [ + util.syspath(source), + *(["-interlace", "none"] if deinterlaced else []), + util.syspath(target), + ] + + try: + subprocess.check_call( + cmd, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL + ) + return target + except subprocess.CalledProcessError: + return source + + +def pil_convert_format(source, target, deinterlaced): + from PIL import Image, UnidentifiedImageError + + try: + with Image.open(util.syspath(source)) as im: + im.save(util.py3_path(target), progressive=not deinterlaced) + return target + except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError, + OSError): + log.exception("failed to convert image {} -> {}", source, target) + return source + + +BACKEND_CONVERT_IMAGE_FORMAT = { + PIL: pil_convert_format, + IMAGEMAGICK: im_convert_format, +} + + class Shareable(type): """A pseudo-singleton metaclass that allows both shared and non-shared instances. The ``MyClass.shared`` property holds a @@ -318,12 +385,50 @@ class ArtResizer(metaclass=Shareable): """Return the size of an image file as an int couple (width, height) in pixels. - Only available locally + Only available locally. """ if self.local: func = BACKEND_GET_SIZE[self.method[0]] return func(path_in) + def get_format(self, path_in): + """Returns the format of the image as a string. + + Only available locally. + """ + if self.local: + func = BACKEND_GET_FORMAT[self.method[0]] + return func(path_in) + + def reformat(self, path_in, new_format, deinterlaced=True): + """Converts image to desired format, updating its extension, but + keeping the same filename. + + Only available locally. + """ + if not self.local: + return path_in + + new_format = new_format.lower() + # A nonexhaustive map of image "types" to extensions overrides + new_format = { + 'jpeg': 'jpg', + }.get(new_format, new_format) + + fname, ext = os.path.splitext(path_in) + path_new = fname + b'.' + new_format.encode('utf8') + func = BACKEND_CONVERT_IMAGE_FORMAT[self.method[0]] + + # allows the exception to propagate, while still making sure a changed + # file path was removed + result_path = path_in + try: + result_path = func(path_in, path_new, deinterlaced) + finally: + if result_path != path_in: + os.unlink(path_in) + return result_path + def _can_compare(self): """A boolean indicating whether image comparison is available""" diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 574e8dae1..f2c1e5a7a 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -50,6 +50,7 @@ class Candidate: CANDIDATE_DOWNSCALE = 2 CANDIDATE_DOWNSIZE = 3 CANDIDATE_DEINTERLACE = 4 + CANDIDATE_REFORMAT = 5 MATCH_EXACT = 0 MATCH_FALLBACK = 1 @@ -74,12 +75,14 @@ class Candidate: Return `CANDIDATE_DOWNSIZE` if the file must be resized, and possibly also rescaled. Return `CANDIDATE_DEINTERLACE` if the file must be deinterlaced. + Return `CANDIDATE_REFORMAT` if the file has to be converted. """ if not self.path: return self.CANDIDATE_BAD if (not (plugin.enforce_ratio or plugin.minwidth or plugin.maxwidth - or plugin.max_filesize or plugin.deinterlace)): + or plugin.max_filesize or plugin.deinterlace + or plugin.cover_format)): return self.CANDIDATE_EXACT # get_size returns None if no local imaging backend is available @@ -142,12 +145,23 @@ class Candidate: filesize, plugin.max_filesize) downsize = True + # Check image format + reformat = False + if plugin.cover_format: + fmt = ArtResizer.shared.get_format(self.path) + reformat = fmt != plugin.cover_format + if reformat: + self._log.debug('image needs reformatting: {} -> {}', + fmt, plugin.cover_format) + if downscale: return self.CANDIDATE_DOWNSCALE elif downsize: return self.CANDIDATE_DOWNSIZE elif plugin.deinterlace: return self.CANDIDATE_DEINTERLACE + elif reformat: + return self.CANDIDATE_REFORMAT else: return self.CANDIDATE_EXACT @@ -169,6 +183,12 @@ class Candidate: max_filesize=plugin.max_filesize) elif self.check == self.CANDIDATE_DEINTERLACE: self.path = ArtResizer.shared.deinterlace(self.path) + elif self.check == self.CANDIDATE_REFORMAT: + self.path = ArtResizer.shared.reformat( + self.path, + plugin.cover_format, + deinterlaced=plugin.deinterlace, + ) def _logged_get(log, *args, **kwargs): @@ -923,6 +943,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'store_source': False, 'high_resolution': False, 'deinterlace': False, + 'cover_format': None, }) self.config['google_key'].redact = True self.config['fanarttv_key'].redact = True @@ -959,6 +980,10 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.src_removed = (config['import']['delete'].get(bool) or config['import']['move'].get(bool)) + self.cover_format = self.config['cover_format'].get( + confuse.Optional(str) + ) + if self.config['auto']: # Enable two import hooks when fetching is enabled. self.import_stages = [self.fetch_art] From 96be1840e3c7f8f757520528601ddc82ab82cbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arsen=20Arsenovi=C4=87?= Date: Tue, 2 Nov 2021 13:27:25 +0100 Subject: [PATCH 031/357] docs: add fetchart cover_format option --- docs/plugins/fetchart.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 5df6c6e34..997cf2497 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -90,6 +90,10 @@ file. The available options are: instructed to store cover art as non-progressive JPEG. You might need this if you use DAPs that don't support progressive images. Default: ``no``. +- **cover_format**: If enabled, forced the cover image into the specified + format. Most often, this will be either ``JPEG`` or ``PNG`` [#imgformats]_. + Also respects ``deinterlace``. + Default: None (leave unchanged). Note: ``maxwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ or `Pillow`_. @@ -105,6 +109,12 @@ or `Pillow`_. .. _beets custom search engine: https://cse.google.com.au:443/cse/publicurl?cx=001442825323518660753:hrh5ch1gjzm .. _Pillow: https://github.com/python-pillow/Pillow .. _ImageMagick: https://www.imagemagick.org/ +.. [#imgformats] Other image formats are available, though the full list + depends on your system and what backend you are using. If you're using the + ImageMagick backend, you can use ``magick identify -list format`` to get a + full list of all supported formats, and you can use the Python function + PIL.features.pilinfo() to print a list of all supported formats in Pillow + (``python3 -c 'import PIL.features as f; f.pilinfo()'``). Here's an example that makes plugin select only images that contain ``front`` or ``back`` keywords in their filenames and prioritizes the iTunes source over From 3de657403a18c33c817bd88758da6c7b5cb4cea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arsen=20Arsenovi=C4=87?= Date: Tue, 2 Nov 2021 13:30:12 +0100 Subject: [PATCH 032/357] changelog: add entry about fetch_art cover_format --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7a6e1a4b6..1b4590f94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -49,6 +49,8 @@ Other new things: * :doc:`/plugins/fetchart`: A new option to store cover art as non-progressive image. Useful for DAPs that support progressive images. Set ``deinterlace: yes`` in your configuration to enable. +* :doc:`/plugins/fetchart`: A new option to change cover art format. Useful for + DAPs that do not support some image formats. For plugin developers: From b67c25a55db3fac4fb567c2155feaa54bcd9b705 Mon Sep 17 00:00:00 2001 From: Julien Cassette Date: Thu, 11 Nov 2021 19:09:28 +0100 Subject: [PATCH 033/357] Use slow queries for flexible attributes in aunique (fix #2678, close #3553) --- beets/library.py | 4 +++- docs/changelog.rst | 3 +++ test/test_library.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index df1b8ffee..d35a7fae6 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1690,7 +1690,9 @@ class DefaultTemplateFunctions: subqueries = [] for key in keys: value = album.get(key, '') - subqueries.append(dbcore.MatchQuery(key, value)) + # Use slow queries for flexible attributes. + fast = key in album.item_keys + subqueries.append(dbcore.MatchQuery(key, value, fast)) albums = self.lib.albums(dbcore.AndQuery(subqueries)) # If there's only one album to matching these details, then do diff --git a/docs/changelog.rst b/docs/changelog.rst index 1b4590f94..552cdc31f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -70,6 +70,9 @@ Bug fixes: * :doc:`/plugins/export`: Fix duplicated output. +* :doc:`/dev/library`: Use slow queries for flexible attributes in aunique. + :bug:`2678` :bug:`3553` + 1.5.0 (August 19, 2021) ----------------------- diff --git a/test/test_library.py b/test/test_library.py index e64f75561..da7d745e2 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -791,6 +791,16 @@ class DisambiguationTest(_common.TestCase, PathFormattingMixin): self._setf('foo%aunique{albumartist album,year,}/$title') self._assert_dest(b'/base/foo 2001/the title', self.i1) + def test_key_flexible_attribute(self): + album1 = self.lib.get_album(self.i1) + album1.flex = 'flex1' + album2 = self.lib.get_album(self.i2) + album2.flex = 'flex2' + album1.store() + album2.store() + self._setf('foo%aunique{albumartist album flex,year}/$title') + self._assert_dest(b'/base/foo/the title', self.i1) + class PluginDestinationTest(_common.TestCase): def setUp(self): From a7ef7704f858bc3d00fafd30a1c3bea3c8e8fb03 Mon Sep 17 00:00:00 2001 From: Christopher Larson Date: Mon, 15 Nov 2021 19:30:05 -0700 Subject: [PATCH 034/357] Send the pluginload event after types and queries are available Making these types and queries available is part of fully loading the plugins, so the event should not be sent until this work is done. This allows plugins to make use of those types and queries in a pluginload listener. --- beets/ui/__init__.py | 23 ++++++++++++----------- docs/changelog.rst | 1 + 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 5e0de77e2..121cb5dc0 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1129,7 +1129,6 @@ def _load_plugins(options, config): plugin_list = config['plugins'].as_str_seq() plugins.load_plugins(plugin_list) - plugins.send("pluginload") return plugins @@ -1145,16 +1144,6 @@ def _setup(options, lib=None): plugins = _load_plugins(options, config) - # Get the default subcommands. - from beets.ui.commands import default_commands - - subcommands = list(default_commands) - subcommands.extend(plugins.commands()) - - if lib is None: - lib = _open_library(config) - plugins.send("library_opened", lib=lib) - # Add types and queries defined by plugins. plugin_types_album = plugins.types(library.Album) library.Album._types.update(plugin_types_album) @@ -1166,6 +1155,18 @@ def _setup(options, lib=None): library.Item._queries.update(plugins.named_queries(library.Item)) library.Album._queries.update(plugins.named_queries(library.Album)) + plugins.send("pluginload") + + # Get the default subcommands. + from beets.ui.commands import default_commands + + subcommands = list(default_commands) + subcommands.extend(plugins.commands()) + + if lib is None: + lib = _open_library(config) + plugins.send("library_opened", lib=lib) + return subcommands, plugins, lib diff --git a/docs/changelog.rst b/docs/changelog.rst index 552cdc31f..5369eb834 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -56,6 +56,7 @@ For plugin developers: * :py:meth:`beets.library.Item.destination` now accepts a `replacements` argument to be used in favor of the default. +* Send the `pluginload` event after plugin types and queries are available, not before. Bug fixes: From 5e6be0ddb3bd6b6a51322135e91a360aa0aacbd3 Mon Sep 17 00:00:00 2001 From: Julien Cassette Date: Tue, 16 Nov 2021 21:40:34 +0100 Subject: [PATCH 035/357] Use short-circuit evaluation in AndQuery and OrQuery (fix #4145) --- beets/dbcore/query.py | 4 ++-- docs/changelog.rst | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index e8e3d1f4a..96476a5b1 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -443,7 +443,7 @@ class AndQuery(MutableCollectionQuery): return self.clause_with_joiner('and') def match(self, item): - return all([q.match(item) for q in self.subqueries]) + return all(q.match(item) for q in self.subqueries) class OrQuery(MutableCollectionQuery): @@ -453,7 +453,7 @@ class OrQuery(MutableCollectionQuery): return self.clause_with_joiner('or') def match(self, item): - return any([q.match(item) for q in self.subqueries]) + return any(q.match(item) for q in self.subqueries) class NotQuery(Query): diff --git a/docs/changelog.rst b/docs/changelog.rst index 552cdc31f..998125ad9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -73,6 +73,9 @@ Bug fixes: * :doc:`/dev/library`: Use slow queries for flexible attributes in aunique. :bug:`2678` :bug:`3553` +* :doc:`/reference/query`: Use short-circuit evaluation in AndQuery and OrQuery + :bug:`4145` + 1.5.0 (August 19, 2021) ----------------------- From d7055fac1d039bdb52ab8bd314d024bd72268673 Mon Sep 17 00:00:00 2001 From: Paldin Bet Eivaz Date: Wed, 17 Nov 2021 16:24:10 -0800 Subject: [PATCH 036/357] added test for track_for_id method --- test/test_spotify.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/test_spotify.py b/test/test_spotify.py index 41217a9fd..87186d718 100644 --- a/test/test_spotify.py +++ b/test/test_spotify.py @@ -127,6 +127,67 @@ class SpotifyPluginTest(_common.TestCase, TestHelper): self.assertIn('album:Despicable Me 2', query) self.assertEqual(params['type'], ['track']) + @responses.activate + def test_track_for_id(self): + """Tests if plugin is able to fetch a track by its Spotify ID""" + + # Mock the Spotify 'Get Track' call + json_file = os.path.join( + _common.RSRC, b'spotify', b'track_info.json' + ) + with open(json_file, 'rb') as f: + response_body = f.read() + + responses.add( + responses.GET, + spotify.SpotifyPlugin.track_url + '6NPVjNh8Jhru9xOmyQigds', + body=response_body, + status=200, + content_type='application/json', + ) + + # Mock the Spotify 'Get Album' call + json_file = os.path.join( + _common.RSRC, b'spotify', b'album_info.json' + ) + with open(json_file, 'rb') as f: + response_body = f.read() + + responses.add( + responses.GET, + spotify.SpotifyPlugin.album_url + '5l3zEmMrOhOzG8d8s83GOL', + body=response_body, + status=200, + content_type='application/json', + ) + + # Mock the Spotify 'Search' call + json_file = os.path.join( + _common.RSRC, b'spotify', b'track_request.json' + ) + with open(json_file, 'rb') as f: + response_body = f.read() + + responses.add( + responses.GET, + spotify.SpotifyPlugin.search_url, + body=response_body, + status=200, + content_type='application/json', + ) + + track_info = self.spotify.track_for_id('6NPVjNh8Jhru9xOmyQigds') + item = Item( + mb_trackid=track_info.track_id, + albumartist=track_info.artist, + title=track_info.title, + length=track_info.length + ) + item.add(self.lib) + + results = self.spotify._match_library_tracks(self.lib, "Happy") + self.assertEqual(1, len(results)) + self.assertEqual("6NPVjNh8Jhru9xOmyQigds", results[0]['id']) def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 7a30bd6d616da94dae5388bb8ab6e55cfc42ce73 Mon Sep 17 00:00:00 2001 From: Paldin Bet Eivaz Date: Wed, 17 Nov 2021 16:25:26 -0800 Subject: [PATCH 037/357] created Spotify track and album info rsrcs --- test/rsrc/spotify/album_info.json | 766 ++++++++++++++++++++++++++++++ test/rsrc/spotify/track_info.json | 77 +++ 2 files changed, 843 insertions(+) create mode 100644 test/rsrc/spotify/album_info.json create mode 100644 test/rsrc/spotify/track_info.json diff --git a/test/rsrc/spotify/album_info.json b/test/rsrc/spotify/album_info.json new file mode 100644 index 000000000..66d6890dc --- /dev/null +++ b/test/rsrc/spotify/album_info.json @@ -0,0 +1,766 @@ +{ + "album_type": "compilation", + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/0LyfQWJT6nXafLPZqxe9Of" + }, + "href": "https://api.spotify.com/v1/artists/0LyfQWJT6nXafLPZqxe9Of", + "id": "0LyfQWJT6nXafLPZqxe9Of", + "name": "Various Artists", + "type": "artist", + "uri": "spotify:artist:0LyfQWJT6nXafLPZqxe9Of" + } + ], + "available_markets": [], + "copyrights": [ + { + "text": "2013 Back Lot Music", + "type": "C" + }, + { + "text": "2013 Back Lot Music", + "type": "P" + } + ], + "external_ids": { + "upc": "857970002363" + }, + "external_urls": { + "spotify": "https://open.spotify.com/album/5l3zEmMrOhOzG8d8s83GOL" + }, + "genres": [], + "href": "https://api.spotify.com/v1/albums/5l3zEmMrOhOzG8d8s83GOL", + "id": "5l3zEmMrOhOzG8d8s83GOL", + "images": [ + { + "height": 640, + "url": "https://i.scdn.co/image/ab67616d0000b27399140a62d43aec760f6172a2", + "width": 640 + }, + { + "height": 300, + "url": "https://i.scdn.co/image/ab67616d00001e0299140a62d43aec760f6172a2", + "width": 300 + }, + { + "height": 64, + "url": "https://i.scdn.co/image/ab67616d0000485199140a62d43aec760f6172a2", + "width": 64 + } + ], + "label": "Back Lot Music", + "name": "Despicable Me 2 (Original Motion Picture Soundtrack)", + "popularity": 0, + "release_date": "2013-06-18", + "release_date_precision": "day", + "total_tracks": 24, + "tracks": { + "href": "https://api.spotify.com/v1/albums/5l3zEmMrOhOzG8d8s83GOL/tracks?offset=0&limit=50", + "items": [ + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/5nLYd9ST4Cnwy6NHaCxbj8" + }, + "href": "https://api.spotify.com/v1/artists/5nLYd9ST4Cnwy6NHaCxbj8", + "id": "5nLYd9ST4Cnwy6NHaCxbj8", + "name": "CeeLo Green", + "type": "artist", + "uri": "spotify:artist:5nLYd9ST4Cnwy6NHaCxbj8" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 221805, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/3EiEbQAR44icEkz3rsMI0N" + }, + "href": "https://api.spotify.com/v1/tracks/3EiEbQAR44icEkz3rsMI0N", + "id": "3EiEbQAR44icEkz3rsMI0N", + "is_local": false, + "name": "Scream", + "preview_url": null, + "track_number": 1, + "type": "track", + "uri": "spotify:track:3EiEbQAR44icEkz3rsMI0N" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/3NVrWkcHOtmPbMSvgHmijZ" + }, + "href": "https://api.spotify.com/v1/artists/3NVrWkcHOtmPbMSvgHmijZ", + "id": "3NVrWkcHOtmPbMSvgHmijZ", + "name": "The Minions", + "type": "artist", + "uri": "spotify:artist:3NVrWkcHOtmPbMSvgHmijZ" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 39065, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/1G4Z91vvEGTYd2ZgOD0MuN" + }, + "href": "https://api.spotify.com/v1/tracks/1G4Z91vvEGTYd2ZgOD0MuN", + "id": "1G4Z91vvEGTYd2ZgOD0MuN", + "is_local": false, + "name": "Another Irish Drinking Song", + "preview_url": null, + "track_number": 2, + "type": "track", + "uri": "spotify:track:1G4Z91vvEGTYd2ZgOD0MuN" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RdwBSPQiwcmiDo9kixcl8" + }, + "href": "https://api.spotify.com/v1/artists/2RdwBSPQiwcmiDo9kixcl8", + "id": "2RdwBSPQiwcmiDo9kixcl8", + "name": "Pharrell Williams", + "type": "artist", + "uri": "spotify:artist:2RdwBSPQiwcmiDo9kixcl8" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 176078, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/7DKqhn3Aa0NT9N9GAcagda" + }, + "href": "https://api.spotify.com/v1/tracks/7DKqhn3Aa0NT9N9GAcagda", + "id": "7DKqhn3Aa0NT9N9GAcagda", + "is_local": false, + "name": "Just a Cloud Away", + "preview_url": null, + "track_number": 3, + "type": "track", + "uri": "spotify:track:7DKqhn3Aa0NT9N9GAcagda" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RdwBSPQiwcmiDo9kixcl8" + }, + "href": "https://api.spotify.com/v1/artists/2RdwBSPQiwcmiDo9kixcl8", + "id": "2RdwBSPQiwcmiDo9kixcl8", + "name": "Pharrell Williams", + "type": "artist", + "uri": "spotify:artist:2RdwBSPQiwcmiDo9kixcl8" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 233305, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/6NPVjNh8Jhru9xOmyQigds" + }, + "href": "https://api.spotify.com/v1/tracks/6NPVjNh8Jhru9xOmyQigds", + "id": "6NPVjNh8Jhru9xOmyQigds", + "is_local": false, + "name": "Happy", + "preview_url": null, + "track_number": 4, + "type": "track", + "uri": "spotify:track:6NPVjNh8Jhru9xOmyQigds" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/3NVrWkcHOtmPbMSvgHmijZ" + }, + "href": "https://api.spotify.com/v1/artists/3NVrWkcHOtmPbMSvgHmijZ", + "id": "3NVrWkcHOtmPbMSvgHmijZ", + "name": "The Minions", + "type": "artist", + "uri": "spotify:artist:3NVrWkcHOtmPbMSvgHmijZ" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 98211, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/5HSqCeDCn2EEGR5ORwaHA0" + }, + "href": "https://api.spotify.com/v1/tracks/5HSqCeDCn2EEGR5ORwaHA0", + "id": "5HSqCeDCn2EEGR5ORwaHA0", + "is_local": false, + "name": "I Swear", + "preview_url": null, + "track_number": 5, + "type": "track", + "uri": "spotify:track:5HSqCeDCn2EEGR5ORwaHA0" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/3NVrWkcHOtmPbMSvgHmijZ" + }, + "href": "https://api.spotify.com/v1/artists/3NVrWkcHOtmPbMSvgHmijZ", + "id": "3NVrWkcHOtmPbMSvgHmijZ", + "name": "The Minions", + "type": "artist", + "uri": "spotify:artist:3NVrWkcHOtmPbMSvgHmijZ" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 175291, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/2Ls4QknWvBoGSeAlNKw0Xj" + }, + "href": "https://api.spotify.com/v1/tracks/2Ls4QknWvBoGSeAlNKw0Xj", + "id": "2Ls4QknWvBoGSeAlNKw0Xj", + "is_local": false, + "name": "Y.M.C.A.", + "preview_url": null, + "track_number": 6, + "type": "track", + "uri": "spotify:track:2Ls4QknWvBoGSeAlNKw0Xj" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RdwBSPQiwcmiDo9kixcl8" + }, + "href": "https://api.spotify.com/v1/artists/2RdwBSPQiwcmiDo9kixcl8", + "id": "2RdwBSPQiwcmiDo9kixcl8", + "name": "Pharrell Williams", + "type": "artist", + "uri": "spotify:artist:2RdwBSPQiwcmiDo9kixcl8" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 206105, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/1XkUmKLbm1tzVtrkdj2Ou8" + }, + "href": "https://api.spotify.com/v1/tracks/1XkUmKLbm1tzVtrkdj2Ou8", + "id": "1XkUmKLbm1tzVtrkdj2Ou8", + "is_local": false, + "name": "Fun, Fun, Fun", + "preview_url": null, + "track_number": 7, + "type": "track", + "uri": "spotify:track:1XkUmKLbm1tzVtrkdj2Ou8" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RdwBSPQiwcmiDo9kixcl8" + }, + "href": "https://api.spotify.com/v1/artists/2RdwBSPQiwcmiDo9kixcl8", + "id": "2RdwBSPQiwcmiDo9kixcl8", + "name": "Pharrell Williams", + "type": "artist", + "uri": "spotify:artist:2RdwBSPQiwcmiDo9kixcl8" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 254705, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/42lHGtAZd6xVLC789afLWt" + }, + "href": "https://api.spotify.com/v1/tracks/42lHGtAZd6xVLC789afLWt", + "id": "42lHGtAZd6xVLC789afLWt", + "is_local": false, + "name": "Despicable Me", + "preview_url": null, + "track_number": 8, + "type": "track", + "uri": "spotify:track:42lHGtAZd6xVLC789afLWt" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 126825, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/7uAC260NViRKyYW4st4vri" + }, + "href": "https://api.spotify.com/v1/tracks/7uAC260NViRKyYW4st4vri", + "id": "7uAC260NViRKyYW4st4vri", + "is_local": false, + "name": "PX-41 Labs", + "preview_url": null, + "track_number": 9, + "type": "track", + "uri": "spotify:track:7uAC260NViRKyYW4st4vri" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 87118, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/6YLmc6yT7OGiNwbShHuEN2" + }, + "href": "https://api.spotify.com/v1/tracks/6YLmc6yT7OGiNwbShHuEN2", + "id": "6YLmc6yT7OGiNwbShHuEN2", + "is_local": false, + "name": "The Fairy Party", + "preview_url": null, + "track_number": 10, + "type": "track", + "uri": "spotify:track:6YLmc6yT7OGiNwbShHuEN2" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 339478, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/5lwsXhSXKFoxoGOFLZdQX6" + }, + "href": "https://api.spotify.com/v1/tracks/5lwsXhSXKFoxoGOFLZdQX6", + "id": "5lwsXhSXKFoxoGOFLZdQX6", + "is_local": false, + "name": "Lucy And The AVL", + "preview_url": null, + "track_number": 11, + "type": "track", + "uri": "spotify:track:5lwsXhSXKFoxoGOFLZdQX6" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 87478, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/2FlWtPuBMGo0a0X7LGETyk" + }, + "href": "https://api.spotify.com/v1/tracks/2FlWtPuBMGo0a0X7LGETyk", + "id": "2FlWtPuBMGo0a0X7LGETyk", + "is_local": false, + "name": "Goodbye Nefario", + "preview_url": null, + "track_number": 12, + "type": "track", + "uri": "spotify:track:2FlWtPuBMGo0a0X7LGETyk" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 86998, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/3YnhGNADeUaoBTjB1uGUjh" + }, + "href": "https://api.spotify.com/v1/tracks/3YnhGNADeUaoBTjB1uGUjh", + "id": "3YnhGNADeUaoBTjB1uGUjh", + "is_local": false, + "name": "Time for Bed", + "preview_url": null, + "track_number": 13, + "type": "track", + "uri": "spotify:track:3YnhGNADeUaoBTjB1uGUjh" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 180265, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/6npUKThV4XI20VLW5ryr5O" + }, + "href": "https://api.spotify.com/v1/tracks/6npUKThV4XI20VLW5ryr5O", + "id": "6npUKThV4XI20VLW5ryr5O", + "is_local": false, + "name": "Break-In", + "preview_url": null, + "track_number": 14, + "type": "track", + "uri": "spotify:track:6npUKThV4XI20VLW5ryr5O" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 95011, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/1qyFlqVfbgyiM7tQ2Jy9vC" + }, + "href": "https://api.spotify.com/v1/tracks/1qyFlqVfbgyiM7tQ2Jy9vC", + "id": "1qyFlqVfbgyiM7tQ2Jy9vC", + "is_local": false, + "name": "Stalking Floyd Eaglesan", + "preview_url": null, + "track_number": 15, + "type": "track", + "uri": "spotify:track:1qyFlqVfbgyiM7tQ2Jy9vC" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 189771, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/4DRQctGiqjJkbFa7iTK4pb" + }, + "href": "https://api.spotify.com/v1/tracks/4DRQctGiqjJkbFa7iTK4pb", + "id": "4DRQctGiqjJkbFa7iTK4pb", + "is_local": false, + "name": "Moving to Australia", + "preview_url": null, + "track_number": 16, + "type": "track", + "uri": "spotify:track:4DRQctGiqjJkbFa7iTK4pb" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 85878, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/1TSjM9GY2oN6RO6aYGN25n" + }, + "href": "https://api.spotify.com/v1/tracks/1TSjM9GY2oN6RO6aYGN25n", + "id": "1TSjM9GY2oN6RO6aYGN25n", + "is_local": false, + "name": "Going to Save the World", + "preview_url": null, + "track_number": 17, + "type": "track", + "uri": "spotify:track:1TSjM9GY2oN6RO6aYGN25n" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 87158, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/3AEMuoglM1myQ8ouIyh8LG" + }, + "href": "https://api.spotify.com/v1/tracks/3AEMuoglM1myQ8ouIyh8LG", + "id": "3AEMuoglM1myQ8ouIyh8LG", + "is_local": false, + "name": "El Macho", + "preview_url": null, + "track_number": 18, + "type": "track", + "uri": "spotify:track:3AEMuoglM1myQ8ouIyh8LG" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 47438, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/2d7fEVYdZnjlya3MPEma21" + }, + "href": "https://api.spotify.com/v1/tracks/2d7fEVYdZnjlya3MPEma21", + "id": "2d7fEVYdZnjlya3MPEma21", + "is_local": false, + "name": "Jillian", + "preview_url": null, + "track_number": 19, + "type": "track", + "uri": "spotify:track:2d7fEVYdZnjlya3MPEma21" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 89398, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/7h8WnOo4Fh6NvfTUnR7nOa" + }, + "href": "https://api.spotify.com/v1/tracks/7h8WnOo4Fh6NvfTUnR7nOa", + "id": "7h8WnOo4Fh6NvfTUnR7nOa", + "is_local": false, + "name": "Take Her Home", + "preview_url": null, + "track_number": 20, + "type": "track", + "uri": "spotify:track:7h8WnOo4Fh6NvfTUnR7nOa" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 212691, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/25A9ZlegjJ0z2fI1PgTqy2" + }, + "href": "https://api.spotify.com/v1/tracks/25A9ZlegjJ0z2fI1PgTqy2", + "id": "25A9ZlegjJ0z2fI1PgTqy2", + "is_local": false, + "name": "El Macho's Lair", + "preview_url": null, + "track_number": 21, + "type": "track", + "uri": "spotify:track:25A9ZlegjJ0z2fI1PgTqy2" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 117745, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/48GwOCuPhWKDktq3efmfRg" + }, + "href": "https://api.spotify.com/v1/tracks/48GwOCuPhWKDktq3efmfRg", + "id": "48GwOCuPhWKDktq3efmfRg", + "is_local": false, + "name": "Home Invasion", + "preview_url": null, + "track_number": 22, + "type": "track", + "uri": "spotify:track:48GwOCuPhWKDktq3efmfRg" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RaHCHhZWBXn460JpMaicz" + }, + "href": "https://api.spotify.com/v1/artists/2RaHCHhZWBXn460JpMaicz", + "id": "2RaHCHhZWBXn460JpMaicz", + "name": "Heitor Pereira", + "type": "artist", + "uri": "spotify:artist:2RaHCHhZWBXn460JpMaicz" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 443251, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/6dZkl2egcKVm8rO9W7pPWa" + }, + "href": "https://api.spotify.com/v1/tracks/6dZkl2egcKVm8rO9W7pPWa", + "id": "6dZkl2egcKVm8rO9W7pPWa", + "is_local": false, + "name": "The Big Battle", + "preview_url": null, + "track_number": 23, + "type": "track", + "uri": "spotify:track:6dZkl2egcKVm8rO9W7pPWa" + }, + { + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/3NVrWkcHOtmPbMSvgHmijZ" + }, + "href": "https://api.spotify.com/v1/artists/3NVrWkcHOtmPbMSvgHmijZ", + "id": "3NVrWkcHOtmPbMSvgHmijZ", + "name": "The Minions", + "type": "artist", + "uri": "spotify:artist:3NVrWkcHOtmPbMSvgHmijZ" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 13886, + "explicit": false, + "external_urls": { + "spotify": "https://open.spotify.com/track/2L0OyiAepqAbKvUZfWovOJ" + }, + "href": "https://api.spotify.com/v1/tracks/2L0OyiAepqAbKvUZfWovOJ", + "id": "2L0OyiAepqAbKvUZfWovOJ", + "is_local": false, + "name": "Ba Do Bleep", + "preview_url": null, + "track_number": 24, + "type": "track", + "uri": "spotify:track:2L0OyiAepqAbKvUZfWovOJ" + } + ], + "limit": 50, + "next": null, + "offset": 0, + "previous": null, + "total": 24 + }, + "type": "album", + "uri": "spotify:album:5l3zEmMrOhOzG8d8s83GOL" +} \ No newline at end of file diff --git a/test/rsrc/spotify/track_info.json b/test/rsrc/spotify/track_info.json new file mode 100644 index 000000000..eb252ee6e --- /dev/null +++ b/test/rsrc/spotify/track_info.json @@ -0,0 +1,77 @@ +{ + "album": { + "album_type": "compilation", + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/0LyfQWJT6nXafLPZqxe9Of" + }, + "href": "https://api.spotify.com/v1/artists/0LyfQWJT6nXafLPZqxe9Of", + "id": "0LyfQWJT6nXafLPZqxe9Of", + "name": "Various Artists", + "type": "artist", + "uri": "spotify:artist:0LyfQWJT6nXafLPZqxe9Of" + } + ], + "available_markets": [], + "external_urls": { + "spotify": "https://open.spotify.com/album/5l3zEmMrOhOzG8d8s83GOL" + }, + "href": "https://api.spotify.com/v1/albums/5l3zEmMrOhOzG8d8s83GOL", + "id": "5l3zEmMrOhOzG8d8s83GOL", + "images": [ + { + "height": 640, + "url": "https://i.scdn.co/image/ab67616d0000b27399140a62d43aec760f6172a2", + "width": 640 + }, + { + "height": 300, + "url": "https://i.scdn.co/image/ab67616d00001e0299140a62d43aec760f6172a2", + "width": 300 + }, + { + "height": 64, + "url": "https://i.scdn.co/image/ab67616d0000485199140a62d43aec760f6172a2", + "width": 64 + } + ], + "name": "Despicable Me 2 (Original Motion Picture Soundtrack)", + "release_date": "2013-06-18", + "release_date_precision": "day", + "total_tracks": 24, + "type": "album", + "uri": "spotify:album:5l3zEmMrOhOzG8d8s83GOL" + }, + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/2RdwBSPQiwcmiDo9kixcl8" + }, + "href": "https://api.spotify.com/v1/artists/2RdwBSPQiwcmiDo9kixcl8", + "id": "2RdwBSPQiwcmiDo9kixcl8", + "name": "Pharrell Williams", + "type": "artist", + "uri": "spotify:artist:2RdwBSPQiwcmiDo9kixcl8" + } + ], + "available_markets": [], + "disc_number": 1, + "duration_ms": 233305, + "explicit": false, + "external_ids": { + "isrc": "USQ4E1300686" + }, + "external_urls": { + "spotify": "https://open.spotify.com/track/6NPVjNh8Jhru9xOmyQigds" + }, + "href": "https://api.spotify.com/v1/tracks/6NPVjNh8Jhru9xOmyQigds", + "id": "6NPVjNh8Jhru9xOmyQigds", + "is_local": false, + "name": "Happy", + "popularity": 1, + "preview_url": null, + "track_number": 4, + "type": "track", + "uri": "spotify:track:6NPVjNh8Jhru9xOmyQigds" +} \ No newline at end of file From a1fe106dc5e22bb2e605554f3d1f6b8f2ed2ad2e Mon Sep 17 00:00:00 2001 From: Paldin Bet Eivaz Date: Thu, 18 Nov 2021 09:37:15 -0800 Subject: [PATCH 038/357] fixed linting error --- test/test_spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_spotify.py b/test/test_spotify.py index 87186d718..f90ecd907 100644 --- a/test/test_spotify.py +++ b/test/test_spotify.py @@ -189,6 +189,7 @@ class SpotifyPluginTest(_common.TestCase, TestHelper): self.assertEqual(1, len(results)) self.assertEqual("6NPVjNh8Jhru9xOmyQigds", results[0]['id']) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From c459ff26fbba063f2b4a9fd4832dc460a552b402 Mon Sep 17 00:00:00 2001 From: "Kirill A. Korinsky" Date: Fri, 19 Nov 2021 16:54:28 +0100 Subject: [PATCH 039/357] Prevent fails of tests when path contains dots Unit test may fails when path to temprorary library contains `.`; to garantue that bug wasn't here, it forces to use one more `.` inside path. Fixes: https://github.com/beetbox/beets/issues/4151 --- test/test_library.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_library.py b/test/test_library.py index da7d745e2..6981b87f9 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -148,7 +148,10 @@ class GetSetTest(_common.TestCase): class DestinationTest(_common.TestCase): def setUp(self): super().setUp() - self.lib = beets.library.Library(':memory:') + # default directory is ~/Music and the only reason why it was switched + # to ~/.Music is to confirm that tests works well when path to + # temporary directory contains . + self.lib = beets.library.Library(':memory:', '~/.Music') self.i = item(self.lib) def tearDown(self): @@ -224,7 +227,7 @@ class DestinationTest(_common.TestCase): self.i.album = '.something' dest = self.i.destination() self.assertTrue(b'something' in dest) - self.assertFalse(b'/.' in dest) + self.assertFalse(b'/.something' in dest) def test_destination_preserves_legitimate_slashes(self): self.i.artist = 'one' From c78bd2972c9ab036e1b5aefcacb2249473dc3e94 Mon Sep 17 00:00:00 2001 From: "Kirill A. Korinsky" Date: Sun, 21 Nov 2021 20:10:30 +0100 Subject: [PATCH 040/357] Add missed `py7zr` dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6ff6a592..02306d6ec 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ setup( 'mpdstats': ['python-mpd2>=0.4.2'], 'plexupdate': ['requests'], 'web': ['flask', 'flask-cors'], - 'import': ['rarfile'], + 'import': ['rarfile', 'py7zr'], 'thumbnails': ['pyxdg', 'Pillow'], 'metasync': ['dbus-python'], 'sonosupdate': ['soco'], From 73c7cc86fe248c9bfca267b0b2f5f4c24fcc5c12 Mon Sep 17 00:00:00 2001 From: David Logie Date: Mon, 22 Nov 2021 20:56:50 +0000 Subject: [PATCH 041/357] Add an 'album_removed' event. This works similarly to the existing 'item_removed' event but is called with an `Album` object. --- beets/library.py | 1 + docs/changelog.rst | 2 ++ docs/dev/plugins.rst | 3 +++ 3 files changed, 6 insertions(+) diff --git a/beets/library.py b/beets/library.py index d35a7fae6..4fc70987a 100644 --- a/beets/library.py +++ b/beets/library.py @@ -878,6 +878,7 @@ class Item(LibModel): album = self.get_album() if album and not album.items(): album.remove(delete, False) + plugins.send('album_removed', album=album) # Send a 'item_removed' signal to plugins plugins.send('item_removed', item=self) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ed497d3b..34fc8338e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -51,6 +51,8 @@ Other new things: yes`` in your configuration to enable. * :doc:`/plugins/fetchart`: A new option to change cover art format. Useful for DAPs that do not support some image formats. +* New plugin event: ``album_removed``. Called when an album is removed from the + library (even when its file is not deleted from disk). For plugin developers: diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index b32955b61..3956aa760 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -143,6 +143,9 @@ The events currently available are: command finishes adding an album to the library. Parameters: ``lib``, ``album`` +* `album_removed`: called with an ``Album`` object every time an album is + removed from the library (even when its file is not deleted from disk). + * `item_copied`: called with an ``Item`` object whenever its file is copied. Parameters: ``item``, ``source`` path, ``destination`` path From 9c9f7eb1ed839fd22f53f1b29e382d3cd67c4eee Mon Sep 17 00:00:00 2001 From: David Logie Date: Mon, 22 Nov 2021 22:00:36 +0000 Subject: [PATCH 042/357] Move the `plugins.send()` call into the `Album.remove()` method. --- beets/library.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 4fc70987a..888836cd9 100644 --- a/beets/library.py +++ b/beets/library.py @@ -878,7 +878,6 @@ class Item(LibModel): album = self.get_album() if album and not album.items(): album.remove(delete, False) - plugins.send('album_removed', album=album) # Send a 'item_removed' signal to plugins plugins.send('item_removed', item=self) @@ -1143,6 +1142,9 @@ class Album(LibModel): """ super().remove() + # Send a 'album_removed' signal to plugins + plugins.send('album_removed', album=self) + # Delete art file. if delete: artpath = self.artpath From 1fad3d01aea4627af42a9b7190d6869d2b007cc4 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 26 Nov 2021 15:35:07 -0500 Subject: [PATCH 043/357] aura: Sanitize filenames in image IDs When constructing paths to image files to serve, we previously spliced strings from URL requests directly into the path to be opened. This is theoretically worrisome because it could allow clients to read other files that they are not supposed to read. I'm not actually sure this is a real security problem because Flask's URL parsing should probably rule out IDs that have `/` in them anyway. But out of an abundance of caution, this now prevents paths from showing up in IDs at all---and also prevents `.` and `..` from being valid names. --- beetsplug/aura.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 3799e0df4..f4ae5527a 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -17,6 +17,7 @@ from mimetypes import guess_type import re +import os.path from os.path import isfile, getsize from beets.plugins import BeetsPlugin @@ -595,6 +596,24 @@ class ArtistDocument(AURADocument): return self.single_resource_document(artist_resource) +def safe_filename(fn): + """Check whether a string is a simple (non-path) filename. + + For example, `foo.txt` is safe because it is a "plain" filename. But + `foo/bar.txt` and `../foo.txt` and `.` are all non-safe because they + can traverse to other directories other than the current one. + """ + # Rule out any directories. + if os.path.basename(fn) != fn: + return False + + # In single names, rule out Unix directory traversal names. + if fn in ('.', '..'): + return False + + return True + + class ImageDocument(AURADocument): """Class for building documents for /images/(id) endpoints.""" @@ -616,6 +635,8 @@ class ImageDocument(AURADocument): parent_type = id_split[0] parent_id = id_split[1] img_filename = "-".join(id_split[2:]) + if not safe_filename(img_filename): + return None # Get the path to the directory parent's images are in if parent_type == "album": @@ -631,7 +652,7 @@ class ImageDocument(AURADocument): # Images for other resource types are not supported return None - img_path = dir_path + "/" + img_filename + img_path = os.path.join(dir_path, img_filename) # Check the image actually exists if isfile(img_path): return img_path From 4e692095eb392ad392db246d1726ac5b8058655c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 26 Nov 2021 15:39:30 -0500 Subject: [PATCH 044/357] Changelog for #4160 --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3bb52baa8..ee8060b2e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -34,6 +34,11 @@ Other new things: * Permissions plugin now sets cover art permissions to the file permissions. +Fixes: + +* :doc:`/plugins/aura`: Fix a potential security hole when serving image + files. :bug:`4160` + 1.5.0 (August 19, 2021) ----------------------- From 8eee0bbd8a23346fcd76488d5ea2d9d6dae6925d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 26 Nov 2021 15:52:39 -0500 Subject: [PATCH 045/357] Slight changelog reordering --- docs/changelog.rst | 63 ++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3d287116c..c48c3e3a5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,29 +7,14 @@ Changelog This release now requires Python 3.6 or later (it removes support for Python 2.7, 3.4, and 3.5). -For packagers: - -* As noted above, the minimum Python version is now 3.6. -* We fixed a flaky test, named `test_album_art` in the `test_zero.py` file, - that some distributions had disabled. Disabling this test should no longer - be necessary. - :bug:`4037` :bug:`4038` -* This version of beets no longer depends on the `six`_ library. - :bug:`4030` -* The `gmusic` plugin was removed since Google Play Music has been shut down. - Thus, the optional dependency on `gmusicapi` does not exist anymore. - :bug:`4089` - Major new features: * Include the genre tags from the release group when the musicbrainz genre option is set, and sort them by the number of votes. Thanks to :user:`aereaux`. - * Primary and secondary release types from MusicBrainz are now stored in ``albumtypes`` field. Thanks to :user:`edgars-supe`. :bug:`2200` - * :doc:`/plugins/albumtypes`: An accompanying plugin for formatting ``albumtypes``. Thanks to :user:`edgars-supe`. @@ -54,35 +39,41 @@ Other new things: * New plugin event: ``album_removed``. Called when an album is removed from the library (even when its file is not deleted from disk). +Bug fixes: + +* :doc:`/plugins/lyrics`: Fix crash bug when beautifulsoup4 is not installed. + :bug:`4027` +* :doc:`/plugins/discogs`: Adapt regex to new URL format . + :bug: `4080` +* :doc:`/plugins/discogs`: Remove requests ratel imit code from plugin in favor of discogs library built-in capability + :bug: `4108` +* :doc:`/plugins/export`: Fix duplicated output. +* :doc:`/dev/library`: Use slow queries for flexible attributes in aunique. + :bug:`2678` :bug:`3553` +* :doc:`/reference/query`: Use short-circuit evaluation in AndQuery and OrQuery + :bug:`4145` +* :doc:`/plugins/aura`: Fix a potential security hole when serving image + files. :bug:`4160` + For plugin developers: * :py:meth:`beets.library.Item.destination` now accepts a `replacements` argument to be used in favor of the default. * Send the `pluginload` event after plugin types and queries are available, not before. -Bug fixes: +For packagers: -* :doc:`/plugins/lyrics`: Fix crash bug when beautifulsoup4 is not installed. - :bug:`4027` +* As noted above, the minimum Python version is now 3.6. +* We fixed a flaky test, named `test_album_art` in the `test_zero.py` file, + that some distributions had disabled. Disabling this test should no longer + be necessary. + :bug:`4037` :bug:`4038` +* This version of beets no longer depends on the `six`_ library. + :bug:`4030` +* The `gmusic` plugin was removed since Google Play Music has been shut down. + Thus, the optional dependency on `gmusicapi` does not exist anymore. + :bug:`4089` -* :doc:`/plugins/discogs`: Adapt regex to new URL format . - :bug: `4080` - -* :doc:`/plugins/discogs`: Remove requests ratel imit code from plugin in favor of discogs library built-in capability - :bug: `4108` - -* :doc:`/plugins/export`: Fix duplicated output. - -* :doc:`/dev/library`: Use slow queries for flexible attributes in aunique. - :bug:`2678` :bug:`3553` - -* :doc:`/reference/query`: Use short-circuit evaluation in AndQuery and OrQuery - :bug:`4145` - -Fixes: - -* :doc:`/plugins/aura`: Fix a potential security hole when serving image - files. :bug:`4160` 1.5.0 (August 19, 2021) ----------------------- From f33606c87aa0d5af3e6c680d40c6b10dddef8f18 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Nov 2021 11:17:14 -0500 Subject: [PATCH 046/357] Switch version to 1.6.0 Especially with the Python version changes, it seems like this justifies more than a 0.0.1 bump. --- beets/__init__.py | 2 +- docs/changelog.rst | 2 +- docs/conf.py | 4 ++-- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/beets/__init__.py b/beets/__init__.py index 285417773..9642a6f3c 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -16,7 +16,7 @@ import confuse from sys import stderr -__version__ = '1.5.1' +__version__ = '1.6.0' __author__ = 'Adrian Sampson ' diff --git a/docs/changelog.rst b/docs/changelog.rst index c48c3e3a5..56d21838d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -1.5.1 (in development) +1.6.0 (in development) ---------------------- This release now requires Python 3.6 or later (it removes support for Python diff --git a/docs/conf.py b/docs/conf.py index 09ee22080..f8fac35aa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,8 +11,8 @@ master_doc = 'index' project = 'beets' copyright = '2016, Adrian Sampson' -version = '1.5' -release = '1.5.1' +version = '1.6' +release = '1.6.0' pygments_style = 'sphinx' diff --git a/setup.py b/setup.py index 02306d6ec..264bb2e7a 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ if 'sdist' in sys.argv: setup( name='beets', - version='1.5.1', + version='1.6.0', description='music tagger and library organizer', author='Adrian Sampson', author_email='adrian@radbox.org', From 080d577206ed2e2f163e1d68d01a873fe4436b91 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Nov 2021 11:33:08 -0500 Subject: [PATCH 047/357] Clean up changelog --- docs/changelog.rst | 93 +++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 56d21838d..1e85c8a9d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,64 +4,81 @@ Changelog 1.6.0 (in development) ---------------------- -This release now requires Python 3.6 or later (it removes support for Python -2.7, 3.4, and 3.5). +This release is our first experiment with time-based releases! We are aiming +to publish a new release of beets every 3 months. We therefore have a healthy +but not dizzyingly long list of new features and fixes. + +With this release, beets now requires Python 3.6 or later (it removes support +for Python 2.7, 3.4, and 3.5). There are also a few other dependency +changes---if you're a maintainer of a beets package for a package manager, +thank you for your ongoing efforts, and please see the list of notes below. Major new features: -* Include the genre tags from the release group when the musicbrainz genre - option is set, and sort them by the number of votes. Thanks to - :user:`aereaux`. -* Primary and secondary release types from MusicBrainz are now stored in - ``albumtypes`` field. Thanks to :user:`edgars-supe`. +* When fetching genres from MusicBrainz, we now include genres from the + release group (in addition to the release). We also prioritize genres based + on the number of votes. + Thanks to :user:`aereaux`. +* Primary and secondary release types from MusicBrainz are now stored in a new + ``albumtypes`` field. + Thanks to :user:`edgars-supe`. :bug:`2200` -* :doc:`/plugins/albumtypes`: An accompanying plugin for formatting - ``albumtypes``. Thanks to :user:`edgars-supe`. +* An accompanying new :doc:`/plugins/albumtypes` includes some options for + formatting this new ``albumtypes`` field. + Thanks to :user:`edgars-supe`. Other new things: -* Permissions plugin now sets cover art permissions to the file permissions. -* :doc:`/plugins/unimported`: Support excluding specific - subdirectories in library. -* :doc:`/plugins/info`: Support ``--album`` flag. -* :doc:`/plugins/export`: Support ``--album`` flag. -* ``beet move`` path differences are now highlighted in color (when enabled). -* When moving files and a direct rename of a file is not possible, beets now - copies to a temporary file in the target folder first instead of directly - using the target path. This gets us closer to always updating files - atomically. Thanks to :user:`catap`. +* :doc:`/plugins/permissions`: The plugin now sets cover art permissions to + match the audio file permissions. +* :doc:`/plugins/unimported`: A new configuration option supports excluding + specific subdirectories in library. +* :doc:`/plugins/info`: Add support for an ``--album`` flag. +* :doc:`/plugins/export`: Similarly add support for an ``--album`` flag. +* ``beet move`` now highlights path differences in color (when enabled). +* When moving files and a direct rename of a file is not possible (for + example, when crossing filesystems), beets now copies to a temporary file in + the target folder first and then moves to the destination instead of + directly copying the target path. This gets us closer to always updating + files atomically. + Thanks to :user:`catap`. :bug:`4060` -* :doc:`/plugins/fetchart`: A new option to store cover art as non-progressive - image. Useful for DAPs that support progressive images. Set ``deinterlace: - yes`` in your configuration to enable. -* :doc:`/plugins/fetchart`: A new option to change cover art format. Useful for - DAPs that do not support some image formats. -* New plugin event: ``album_removed``. Called when an album is removed from the - library (even when its file is not deleted from disk). +* :doc:`/plugins/fetchart`: Add a new option to store cover art as + non-progressive image. This is useful for DAPs that do not support + progressive images. Set ``deinterlace: yes`` in your configuration to enable + this conversion. +* :doc:`/plugins/fetchart`: Add a new option to change the file format of + cover art images. This may also be useful for DAPs that only support some + image formats. +* Support flexible attributes in ``%aunique``. + :bug:`2678` :bug:`3553` +* Make ``%aunique`` faster, especially when using inline fields. + :bug:`4145` Bug fixes: -* :doc:`/plugins/lyrics`: Fix crash bug when beautifulsoup4 is not installed. +* :doc:`/plugins/lyrics`: Fix a crash when Beautiful Soup is not installed. :bug:`4027` -* :doc:`/plugins/discogs`: Adapt regex to new URL format . - :bug: `4080` -* :doc:`/plugins/discogs`: Remove requests ratel imit code from plugin in favor of discogs library built-in capability +* :doc:`/plugins/discogs`: Support a new Discogs URL format for IDs. + :bug:`4080` +* :doc:`/plugins/discogs`: Remove built-in rate-limiting because the Discogs + Python library we use now has its own rate-limiting. :bug: `4108` -* :doc:`/plugins/export`: Fix duplicated output. -* :doc:`/dev/library`: Use slow queries for flexible attributes in aunique. - :bug:`2678` :bug:`3553` -* :doc:`/reference/query`: Use short-circuit evaluation in AndQuery and OrQuery - :bug:`4145` +* :doc:`/plugins/export`: Fix some duplicated output. * :doc:`/plugins/aura`: Fix a potential security hole when serving image - files. :bug:`4160` + files. + :bug:`4160` For plugin developers: * :py:meth:`beets.library.Item.destination` now accepts a `replacements` argument to be used in favor of the default. -* Send the `pluginload` event after plugin types and queries are available, not before. +* The `pluginload` event is now sent after plugin types and queries are + available, not before. +* A new plugin event, `album_removed`, is called when an album is removed from + the library (even when its file is not deleted from disk). -For packagers: +Here are some notes for packagers: * As noted above, the minimum Python version is now 3.6. * We fixed a flaky test, named `test_album_art` in the `test_zero.py` file, From 19371805e70c34e06ddd1ee3bd2e23a23827f845 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Nov 2021 11:35:57 -0500 Subject: [PATCH 048/357] Add date to changelog --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1e85c8a9d..37f5756fa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,8 +1,8 @@ Changelog ========= -1.6.0 (in development) ----------------------- +1.6.0 (November 27, 2021) +------------------------- This release is our first experiment with time-based releases! We are aiming to publish a new release of beets every 3 months. We therefore have a healthy From e3f4e19298cb9296dc636b7a15ab77d98087b635 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 27 Nov 2021 11:38:48 -0500 Subject: [PATCH 049/357] Version bump: v1.6.1 --- docs/changelog.rst | 6 ++++++ docs/conf.py | 2 +- extra/release.py | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 37f5756fa..06fbe3c53 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +1.6.1 (in development) +---------------------- + +Changelog goes here! + + 1.6.0 (November 27, 2021) ------------------------- diff --git a/docs/conf.py b/docs/conf.py index f8fac35aa..f8ed63f9d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ project = 'beets' copyright = '2016, Adrian Sampson' version = '1.6' -release = '1.6.0' +release = '1.6.1' pygments_style = 'sphinx' diff --git a/extra/release.py b/extra/release.py index 2a98e06e8..7904cd414 100755 --- a/extra/release.py +++ b/extra/release.py @@ -276,7 +276,7 @@ def prep(): cur_version = get_version() # Tag. - subprocess.check_output(['git', 'tag', f'v{cur_version}']) + subprocess.check_call(['git', 'tag', f'v{cur_version}']) # Build. with chdir(BASE): diff --git a/setup.py b/setup.py index 264bb2e7a..aefa0b18f 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ if 'sdist' in sys.argv: setup( name='beets', - version='1.6.0', + version='1.6.1', description='music tagger and library organizer', author='Adrian Sampson', author_email='adrian@radbox.org', From 3c7853712f83aafb3a9aa7b1c2943b45858e660f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 28 Nov 2021 08:43:23 -0500 Subject: [PATCH 050/357] Require confuse >= 1.5.0 Fixes #4167. --- docs/changelog.rst | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 06fbe3c53..b6eb5ca3a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,11 @@ Changelog Changelog goes here! +For packagers: + +* We fixed a version for the dependency on the `Confuse`_ library. + :bug:`4167` + 1.6.0 (November 27, 2021) ------------------------- @@ -69,7 +74,7 @@ Bug fixes: :bug:`4080` * :doc:`/plugins/discogs`: Remove built-in rate-limiting because the Discogs Python library we use now has its own rate-limiting. - :bug: `4108` + :bug:`4108` * :doc:`/plugins/export`: Fix some duplicated output. * :doc:`/plugins/aura`: Fix a potential security hole when serving image files. diff --git a/setup.py b/setup.py index aefa0b18f..fa92448a2 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ setup( 'musicbrainzngs>=0.4', 'pyyaml', 'mediafile>=0.2.0', - 'confuse>=1.0.0', + 'confuse>=1.5.0', 'munkres>=1.0.0', 'jellyfish', ] + ( From 4e30699aa7fb14f7bb24b15e3fc30fbd22bfa733 Mon Sep 17 00:00:00 2001 From: Stephen Michel Date: Tue, 30 Nov 2021 09:44:24 -0500 Subject: [PATCH 051/357] Exclude lib64 from git whether folder or symlink Sometimes lib64 is a symlink to lib/; the previous `lib64/' entry didn't catch this. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 370776197..dc193ff2c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,7 @@ downloads/ eggs/ .eggs/ lib/ -lib64/ +lib64 parts/ sdist/ var/ From 69fe1d1bafc02112912fa1bf8b68fa04f8d9691a Mon Sep 17 00:00:00 2001 From: Ramon Boss Date: Tue, 30 Nov 2021 20:10:56 +0100 Subject: [PATCH 052/357] fix: use query param for genius search --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 2cb50ca5e..7d026def1 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -377,7 +377,7 @@ class Genius(Backend): data = {'q': title + " " + artist.lower()} try: response = requests.get( - search_url, data=data, headers=self.headers) + search_url, params=data, headers=self.headers) except requests.RequestException as exc: self._log.debug('Genius API request failed: {0}', exc) return None From 78c95413dde3d3fda75265da43ea33f0864935d1 Mon Sep 17 00:00:00 2001 From: Ramon Boss Date: Tue, 30 Nov 2021 20:17:46 +0100 Subject: [PATCH 053/357] doc: add changelog entry for genius lyrics fix --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b6eb5ca3a..8dc14aa97 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,10 @@ Changelog Changelog goes here! +Bug fixes: + +* :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. + For packagers: * We fixed a version for the dependency on the `Confuse`_ library. From 74522b41a92da1e3615eccec6baae51ff798190a Mon Sep 17 00:00:00 2001 From: tummychow Date: Tue, 30 Nov 2021 20:51:09 -0800 Subject: [PATCH 054/357] Add default for unimported.ignore_subdirectories --- beetsplug/unimported.py | 3 ++- docs/changelog.rst | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/unimported.py b/beetsplug/unimported.py index 7714ec833..4a238531d 100644 --- a/beetsplug/unimported.py +++ b/beetsplug/unimported.py @@ -32,7 +32,8 @@ class Unimported(BeetsPlugin): super().__init__() self.config.add( { - 'ignore_extensions': [] + 'ignore_extensions': [], + 'ignore_subdirectories': [] } ) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dc14aa97..6f66f61f1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Changelog goes here! Bug fixes: * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. +* :doc:`/plugins/unimported`: The new configuration option added in 1.6.0 now has + a default value if it hasn't been set. For packagers: From e876c2722be9836c6718f43b61726804fdbaff43 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 2 Dec 2021 07:36:16 -0500 Subject: [PATCH 055/357] Clarify changelog entry --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f66f61f1..8d95b7fb0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,8 +9,8 @@ Changelog goes here! Bug fixes: * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. -* :doc:`/plugins/unimported`: The new configuration option added in 1.6.0 now has - a default value if it hasn't been set. +* :doc:`/plugins/unimported`: The new ``ignore_subdirectories`` configuration + option added in 1.6.0 now has a default value if it hasn't been set. For packagers: From e39dcfc002557b87cc0e54a56b3022e20ded8828 Mon Sep 17 00:00:00 2001 From: Dominik Schrempf Date: Fri, 3 Dec 2021 22:36:48 +0100 Subject: [PATCH 056/357] clarify `-a` option for `beet modify` See https://github.com/beetbox/beets/discussions/4172. I think the confusion arises because the documentation refers to the query. That is, when `-a` is given, albums are queried, not tracks. This is especially clear when using `beet list`, because then it truly lists "albums instead of items". However, for other commands, the distinction between what is queried and what is acted on should be made more clear. This PR fixes the section for `modify`, but there are more questions: - `remove` command: The documentation states that it acts on albums instead of individual tracks. I guess we should also amend that? I think the complete album including the tracks is deleted, or is that not true? - `move` command: I think the same is true for this command. If `-a` is given, the queried albums including all tracks are moved. - `update` command: The `-a` flag is not explained here. --- docs/reference/cli.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 9b0e6f482..214956873 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -263,11 +263,11 @@ artist="Tom Tom Club"`` will change the artist for the track "Genius of Love." To remove fields (which is only possible for flexible attributes), follow a field name with an exclamation point: ``field!``. -The ``-a`` switch operates on albums instead of individual tracks. Without -this flag, the command will only change *track-level* data, even if all the -tracks belong to the same album. If you want to change an *album-level* field, -such as ``year`` or ``albumartist``, you'll want to use the ``-a`` flag to -avoid a confusing situation where the data for individual tracks conflicts +The ``-a`` switch also operates on albums in addition to the individual tracks. +Without this flag, the command will only change *track-level* data, even if all +the tracks belong to the same album. If you want to change an *album-level* +field, such as ``year`` or ``albumartist``, you'll want to use the ``-a`` flag +to avoid a confusing situation where the data for individual tracks conflicts with the data for the whole album. Items will automatically be moved around when necessary if they're in your From 8c5ced3ee11a353546034189736c6001115135a4 Mon Sep 17 00:00:00 2001 From: Katelyn Lindsey Date: Mon, 6 Dec 2021 21:14:22 -0800 Subject: [PATCH 057/357] Updates docstrings in library.py. Alters existing docstrings to follow google's docstring format and adds docstring to Items class. Also updates some typos and updates some block comments to follow PEP 8 style guide. --- beets/library.py | 299 ++++++++++++++++++++++++++--------------------- 1 file changed, 164 insertions(+), 135 deletions(-) diff --git a/beets/library.py b/beets/library.py index 888836cd9..c350d018e 100644 --- a/beets/library.py +++ b/beets/library.py @@ -53,8 +53,9 @@ class PathQuery(dbcore.FieldQuery): """ def __init__(self, field, pattern, fast=True, case_sensitive=None): - """Create a path query. `pattern` must be a path, either to a - file or a directory. + """Creates a path query. + + `pattern` must be a path, either to a file or a directory. `case_sensitive` can be a bool or `None`, indicating that the behavior should depend on the filesystem. @@ -79,7 +80,7 @@ class PathQuery(dbcore.FieldQuery): @classmethod def is_path_query(cls, query_part): - """Try to guess whether a unicode query part is a path query. + """Tries to guess whether a unicode query part is a path query. Condition: separator precedes colon and the file exists. """ @@ -140,8 +141,10 @@ class DateType(types.Float): class PathType(types.Type): - """A dbcore type for filesystem paths. These are represented as - `bytes` objects, in keeping with the Unix filesystem abstraction. + """A dbcore type for filesystem paths. + + These are represented as `bytes` objects, in keeping with + the Unix filesystem abstraction. """ sql = 'BLOB' @@ -149,8 +152,9 @@ class PathType(types.Type): model_type = bytes def __init__(self, nullable=False): - """Create a path type object. `nullable` controls whether the - type may be missing, i.e., None. + """Creates a path type object. + + `nullable` controls whether the type may be missing, i.e., None. """ self.nullable = nullable @@ -243,7 +247,7 @@ class DurationType(types.Float): # Library-specific sort types. class SmartArtistSort(dbcore.query.Sort): - """Sort by artist (either album artist or track artist), + """Sorts by artist (either album artist or track artist), prioritizing the sort field over the raw field. """ @@ -283,12 +287,13 @@ PF_KEY_DEFAULT = 'default' # Exceptions. class FileOperationError(Exception): """Indicates an error when interacting with a file on disk. + Possibilities include an unsupported media type, a permissions error, and an unhandled Mutagen exception. """ def __init__(self, path, reason): - """Create an exception describing an operation on the file at + """Creates an exception describing an operation on the file at `path` with the underlying (chained) exception `reason`. """ super().__init__(path, reason) @@ -296,8 +301,10 @@ class FileOperationError(Exception): self.reason = reason def text(self): - """Get a string representing the error. Describes both the - underlying reason and the file path in question. + """Gets a string representing the error. + + Describes both the underlying reason and the file path + in question. """ return '{}: {}'.format( util.displayable_path(self.path), @@ -310,16 +317,14 @@ class FileOperationError(Exception): class ReadError(FileOperationError): - """An error while reading a file (i.e. in `Item.read`). - """ + """An error while reading a file (i.e. in `Item.read`).""" def __str__(self): return 'error reading ' + super().text() class WriteError(FileOperationError): - """An error while writing a file (i.e. in `Item.write`). - """ + """An error while writing a file (i.e. in `Item.write`).""" def __str__(self): return 'error writing ' + super().text() @@ -328,12 +333,10 @@ class WriteError(FileOperationError): # Item and Album model classes. class LibModel(dbcore.Model): - """Shared concrete functionality for Items and Albums. - """ + """Shared concrete functionality for Items and Albums.""" + # Config key that specifies how an instance should be formatted. _format_config_key = None - """Config key that specifies how an instance should be formatted. - """ def _template_funcs(self): funcs = DefaultTemplateFunctions(self, self._db).functions() @@ -366,7 +369,7 @@ class LibModel(dbcore.Model): class FormattedItemMapping(dbcore.db.FormattedMapping): - """Add lookup for album-level fields. + """Adds lookup for album-level fields. Album-level fields take precedence if `for_path` is true. """ @@ -409,8 +412,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): return self.item._cached_album def _get(self, key): - """Get the value for a key, either from the album or the item. - Raise a KeyError for invalid keys. + """Gets the value for a key, either from the album or the item. + + Raises a KeyError for invalid keys. """ if self.for_path and key in self.album_keys: return self._get_formatted(self.album, key) @@ -422,8 +426,10 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): raise KeyError(key) def __getitem__(self, key): - """Get the value for a key. `artist` and `albumartist` - are fallback values for each other when not set. + """Gets the value for a key. + + `artist` and `albumartist` are fallback values for each other + when not set. """ value = self._get(key) @@ -448,6 +454,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): class Item(LibModel): + """Represents a song or track.""" _table = 'items' _flex_table = 'item_attributes' _fields = { @@ -539,22 +546,18 @@ class Item(LibModel): 'data_source': types.STRING, } + # Set of item fields that are backed by `MediaFile` fields. + # Any kind of field (fixed, flexible, and computed) may be a media + # field. Only these fields are read from disk in `read` and written in + # `write`. _media_fields = set(MediaFile.readable_fields()) \ .intersection(_fields.keys()) - """Set of item fields that are backed by `MediaFile` fields. - - Any kind of field (fixed, flexible, and computed) may be a media - field. Only these fields are read from disk in `read` and written in - `write`. - """ + # Set of item fields that are backed by *writable* `MediaFile` tag + # fields. + # This excludes fields that represent audio data, such as `bitrate` or + # `length`. _media_tag_fields = set(MediaFile.fields()).intersection(_fields.keys()) - """Set of item fields that are backed by *writable* `MediaFile` tag - fields. - - This excludes fields that represent audio data, such as `bitrate` or - `length`. - """ _formatter = FormattedItemMapping @@ -562,8 +565,8 @@ class Item(LibModel): _format_config_key = 'format_item' + # Cached album object. Read-only. __album = None - """Cached album object. Read-only.""" @property def _cached_album(self): @@ -594,8 +597,7 @@ class Item(LibModel): @classmethod def from_path(cls, path): - """Creates a new item from the media file at the specified path. - """ + """Creates a new item from the media file at the specified path.""" # Initiate with values that aren't read from files. i = cls(album_id=None) i.read(path) @@ -603,8 +605,7 @@ class Item(LibModel): return i def __setitem__(self, key, value): - """Set the item's value for a standard field or a flexattr. - """ + """Sets the item's value for a standard field or a flexattr.""" # Encode unicode paths and read buffers. if key == 'path': if isinstance(value, str): @@ -620,8 +621,10 @@ class Item(LibModel): self.mtime = 0 # Reset mtime on dirty. def __getitem__(self, key): - """Get the value for a field, falling back to the album if - necessary. Raise a KeyError if the field is not available. + """Gets the value for a field, falling back to the album if + necessary. + + Raises a KeyError if the field is not available. """ try: return super().__getitem__(key) @@ -641,8 +644,9 @@ class Item(LibModel): ) def keys(self, computed=False, with_album=True): - """Get a list of available field names. `with_album` - controls whether the album's fields are included. + """Gets a list of available field names. + + `with_album` controls whether the album's fields are included. """ keys = super().keys(computed=computed) if with_album and self._cached_album: @@ -652,8 +656,10 @@ class Item(LibModel): return keys def get(self, key, default=None, with_album=True): - """Get the value for a given key or `default` if it does not - exist. Set `with_album` to false to skip album fallback. + """Gets the value for a given key or `default` if it does not + exist. + + Set `with_album` to false to skip album fallback. """ try: return self._get(key, default, raise_=with_album) @@ -663,8 +669,9 @@ class Item(LibModel): return default def update(self, values): - """Set all key/value pairs in the mapping. If mtime is - specified, it is not reset (as it might otherwise be). + """Sets all key/value pairs in the mapping. + + If mtime is specified, it is not reset (as it might otherwise be). """ super().update(values) if self.mtime == 0 and 'mtime' in values: @@ -676,7 +683,7 @@ class Item(LibModel): setattr(self, key, None) def get_album(self): - """Get the Album object that this item belongs to, if any, or + """Gets the Album object that this item belongs to, if any, or None if the item is a singleton or is not associated with a library. """ @@ -687,9 +694,9 @@ class Item(LibModel): # Interaction with file metadata. def read(self, read_path=None): - """Read the metadata from the associated file. + """Reads the metadata from the associated file. - If `read_path` is specified, read metadata from that file + If `read_path` is specified, reads metadata from that file instead. Updates all the properties in `_media_fields` from the media file. @@ -718,7 +725,7 @@ class Item(LibModel): self.path = read_path def write(self, path=None, tags=None, id3v23=None): - """Write the item's metadata to a media file. + """Writes the item's metadata to a media file. All fields in `_media_fields` are written to disk according to the values on this object. @@ -782,7 +789,7 @@ class Item(LibModel): return False def try_sync(self, write, move, with_album=True): - """Synchronize the item with the database and, possibly, updates its + """Synchronizes the item with the database and, possibly, updates its tags on disk and its path (by moving the file). `write` indicates whether to write new tags into the file. Similarly, @@ -806,7 +813,7 @@ class Item(LibModel): # Files themselves. def move_file(self, dest, operation=MoveOperation.MOVE): - """Move, copy, link or hardlink the item's depending on `operation`, + """Moves, copies, links or hardlinks the item depending on `operation`, updating the path value if the move succeeds. If a file exists at `dest`, then it is slightly modified to be unique. @@ -854,9 +861,9 @@ class Item(LibModel): return int(os.path.getmtime(syspath(self.path))) def try_filesize(self): - """Get the size of the underlying file in bytes. + """Gets the size of the underlying file in bytes. - If the file is missing, return 0 (and log a warning). + If the file is missing, returns 0 (and logs a warning). """ try: return os.path.getsize(syspath(self.path)) @@ -867,9 +874,12 @@ class Item(LibModel): # Model methods. def remove(self, delete=False, with_album=True): - """Removes the item. If `delete`, then the associated file is - removed from disk. If `with_album`, then the item's album (if - any) is removed if it the item was the last in the album. + """Removes the item. + + If `delete`, then the associated file is removed from disk. + + If `with_album`, then the item's album (if any) is removed + if the item was the last in the album. """ super().remove() @@ -891,10 +901,11 @@ class Item(LibModel): def move(self, operation=MoveOperation.MOVE, basedir=None, with_album=True, store=True): - """Move the item to its designated location within the library - directory (provided by destination()). Subdirectories are - created as needed. If the operation succeeds, the item's path - field is updated to reflect the new location. + """Moves the item to its designated location within the library + directory (provided by destination()). + + Subdirectories are created as needed. If the operation succeeds, + the item's path field is updated to reflect the new location. Instead of moving the item it can also be copied, linked or hardlinked depending on `operation` which should be an instance of @@ -940,10 +951,11 @@ class Item(LibModel): def destination(self, fragment=False, basedir=None, platform=None, path_formats=None, replacements=None): """Returns the path in the library directory designated for the - item (i.e., where the file ought to be). fragment makes this - method return just the path fragment underneath the root library - directory; the path is also returned as Unicode instead of - encoded as a bytestring. basedir can override the library's base + item (i.e., where the file ought to be). + + fragment makes this method return just the path fragment underneath + the root library directory; the path is also returned as Unicode instead + of encoded as a bytestring. basedir can override the library's base directory for the destination. """ self._check_db() @@ -1017,8 +1029,9 @@ class Item(LibModel): class Album(LibModel): """Provides access to information about albums stored in a - library. Reflects the library's "albums" table, including album - art. + library. + + Reflects the library's "albums" table, including album art. """ _table = 'albums' _flex_table = 'album_attributes' @@ -1076,6 +1089,7 @@ class Album(LibModel): 'artist': SmartArtistSort, } + # List of keys that are set on an album's items. item_keys = [ 'added', 'albumartist', @@ -1113,8 +1127,6 @@ class Album(LibModel): 'original_month', 'original_day', ] - """List of keys that are set on an album's items. - """ _format_config_key = 'format_album' @@ -1135,9 +1147,12 @@ class Album(LibModel): def remove(self, delete=False, with_items=True): """Removes this album and all its associated items from the - library. If delete, then the items' files are also deleted - from disk, along with any album art. The directories - containing the album are also removed (recursively) if empty. + library. + + If delete, then the items' files are also deleted from disk, + along with any album art. The directories containing the album are + also removed (recursively) if empty. + Set with_items to False to avoid removing the album's items. """ super().remove() @@ -1157,7 +1172,7 @@ class Album(LibModel): item.remove(delete, False) def move_art(self, operation=MoveOperation.MOVE): - """Move, copy, link or hardlink (depending on `operation`) any + """Moves, copies, links or hardlinks (depending on `operation`) any existing album art so that it remains in the same directory as the items. @@ -1199,7 +1214,7 @@ class Album(LibModel): self.artpath = new_art def move(self, operation=MoveOperation.MOVE, basedir=None, store=True): - """Move, copy, link or hardlink (depending on `operation`) + """Moves, copies, links or hardlinks (depending on `operation`) all items to their destination. Any album art moves along with them. `basedir` overrides the library base directory for the destination. @@ -1208,8 +1223,8 @@ class Album(LibModel): By default, the album is stored to the database, persisting any modifications to its metadata. If `store` is `False` however, - the album is not stored automatically, and you'll have to manually - store it after invoking this method. + the album is not stored automatically, and it will have to be manually + stored after invoking this method. """ basedir = basedir or self._db.directory @@ -1239,8 +1254,7 @@ class Album(LibModel): return os.path.dirname(item.path) def _albumtotal(self): - """Return the total number of tracks on all discs on the album - """ + """Returns the total number of tracks on all discs on the album.""" if self.disctotal == 1 or not beets.config['per_disc_numbering']: return self.items()[0].tracktotal @@ -1261,7 +1275,9 @@ class Album(LibModel): def art_destination(self, image, item_dir=None): """Returns a path to the destination for the album art image - for the album. `image` is the path of the image that will be + for the album. + + `image` is the path of the image that will be moved there (used for its extension). The path construction uses the existing path of the album's @@ -1290,6 +1306,7 @@ class Album(LibModel): def set_art(self, path, copy=True): """Sets the album's cover art to the image at the given path. + The image is copied (or moved) into place, replacing any existing art. @@ -1320,10 +1337,12 @@ class Album(LibModel): plugins.send('art_set', album=self) def store(self, fields=None): - """Update the database with the album information. The album's - tracks are also updated. - :param fields: The fields to be stored. If not specified, all fields - will be. + """Updates the database with the album information. + + The album's tracks are also updated. + + `fields` represents the fields to be stored. If not specified, + all fields will be. """ # Get modified track fields. track_updates = {} @@ -1340,8 +1359,8 @@ class Album(LibModel): item.store() def try_sync(self, write, move): - """Synchronize the album and its items with the database. - Optionally, also write any new tags into the files and update + """Synchronizes the album and its items with the database. + Optionally, also writes any new tags into the files and update their paths. `write` indicates whether to write tags to the item files, and @@ -1356,7 +1375,7 @@ class Album(LibModel): # Query construction helpers. def parse_query_parts(parts, model_cls): - """Given a beets query string as a list of components, return the + """Given a beets query string as a list of components, returns the `Query` and `Sort` they represent. Like `dbcore.parse_sorted_query`, with beets query prefixes and @@ -1392,7 +1411,7 @@ def parse_query_parts(parts, model_cls): def parse_query_string(s, model_cls): - """Given a beets query string, return the `Query` and `Sort` they + """Given a beets query string, returns the `Query` and `Sort` they represent. The string is split into components using shell-like syntax. @@ -1419,8 +1438,7 @@ def _sqlite_bytelower(bytestring): # The Library: interface to the database. class Library(dbcore.Database): - """A database of music containing songs and albums. - """ + """A database of music containing songs and albums.""" _models = (Item, Album) def __init__(self, path='library.blb', @@ -1445,15 +1463,17 @@ class Library(dbcore.Database): # Adding objects to the database. def add(self, obj): - """Add the :class:`Item` or :class:`Album` object to the library - database. Return the object's new id. + """Adds the :class:`Item` or :class:`Album` object to the library + database. + + Returns the object's new id. """ obj.add(self) self._memotable = {} return obj.id def add_album(self, items): - """Create a new album consisting of a list of items. + """Creates a new album consisting of a list of items. The items are added to the database if they don't yet have an ID. Return a new :class:`Album` object. The list items must not @@ -1506,39 +1526,39 @@ class Library(dbcore.Database): @staticmethod def get_default_album_sort(): - """Get a :class:`Sort` object for albums from the config option. + """Gets a :class:`Sort` object for albums from the config option. """ return dbcore.sort_from_strings( Album, beets.config['sort_album'].as_str_seq()) @staticmethod def get_default_item_sort(): - """Get a :class:`Sort` object for items from the config option. + """Gets a :class:`Sort` object for items from the config option. """ return dbcore.sort_from_strings( Item, beets.config['sort_item'].as_str_seq()) def albums(self, query=None, sort=None): - """Get :class:`Album` objects matching the query. + """Gets :class:`Album` objects matching the query. """ return self._fetch(Album, query, sort or self.get_default_album_sort()) def items(self, query=None, sort=None): - """Get :class:`Item` objects matching the query. + """Gets :class:`Item` objects matching the query. """ return self._fetch(Item, query, sort or self.get_default_item_sort()) # Convenience accessors. def get_item(self, id): - """Fetch an :class:`Item` by its ID. Returns `None` if no match is + """Fetches a :class:`Item` by its ID. Returns `None` if no match is found. """ return self._get(Item, id) def get_album(self, item_or_id): - """Given an album ID or an item associated with an album, return - an :class:`Album` object for the album. If no such album exists, + """Given an album ID or an item associated with an album, returns + a :class:`Album` object for the album. If no such album exists, returns `None`. """ if isinstance(item_or_id, int): @@ -1553,8 +1573,10 @@ class Library(dbcore.Database): # Default path template resources. def _int_arg(s): - """Convert a string argument to an integer for use in a template - function. May raise a ValueError. + """Converts a string argument to an integer for use in a template + function. + + May raise a ValueError. """ return int(s.strip()) @@ -1568,16 +1590,19 @@ class DefaultTemplateFunctions: _prefix = 'tmpl_' def __init__(self, item=None, lib=None): - """Parametrize the functions. If `item` or `lib` is None, then - some functions (namely, ``aunique``) will always evaluate to the - empty string. + """Parametrizes the functions. + + If `item` or `lib` is None, then some functions (namely, ``aunique``) + will always evaluate to the empty string. """ self.item = item self.lib = lib def functions(self): """Returns a dictionary containing the functions defined in this - object. The keys are function names (as exposed in templates) + object. + + The keys are function names (as exposed in templates) and the values are Python functions. """ out = {} @@ -1587,33 +1612,33 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_lower(s): - """Convert a string to lower case.""" + """Converts a string to lower case.""" return s.lower() @staticmethod def tmpl_upper(s): - """Covert a string to upper case.""" + """Converts a string to upper case.""" return s.upper() @staticmethod def tmpl_title(s): - """Convert a string to title case.""" + """Converts a string to title case.""" return string.capwords(s) @staticmethod def tmpl_left(s, chars): - """Get the leftmost characters of a string.""" + """Gets the leftmost characters of a string.""" return s[0:_int_arg(chars)] @staticmethod def tmpl_right(s, chars): - """Get the rightmost characters of a string.""" + """Gets the rightmost characters of a string.""" return s[-_int_arg(chars):] @staticmethod def tmpl_if(condition, trueval, falseval=''): - """If ``condition`` is nonempty and nonzero, emit ``trueval``; - otherwise, emit ``falseval`` (if provided). + """If ``condition`` is nonempty and nonzero, emits ``trueval``; + otherwise, emits ``falseval`` (if provided). """ try: int_condition = _int_arg(condition) @@ -1630,21 +1655,21 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_asciify(s): - """Translate non-ASCII characters to their ASCII equivalents. + """Translates non-ASCII characters to their ASCII equivalents. """ return util.asciify_path(s, beets.config['path_sep_replace'].as_str()) @staticmethod def tmpl_time(s, fmt): - """Format a time value using `strftime`. - """ + """Formats a time value using `strftime`.""" cur_fmt = beets.config['time_format'].as_str() return time.strftime(fmt, time.strptime(s, cur_fmt)) def tmpl_aunique(self, keys=None, disam=None, bracket=None): - """Generate a string that is guaranteed to be unique among all - albums in the library who share the same set of keys. A fields - from "disam" is used in the string if one is sufficient to + """Generates a string that is guaranteed to be unique among all + albums in the library who share the same set of keys. + + A fields from "disam" is used in the string if one is sufficient to disambiguate the albums. Otherwise, a fallback opaque value is used. Both "keys" and "disam" should be given as whitespace-separated lists of field names, while "bracket" is a @@ -1736,26 +1761,30 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_first(s, count=1, skip=0, sep='; ', join_str='; '): """ Gets the item(s) from x to y in a string separated by something - and join then with something - - :param s: the string - :param count: The number of items included - :param skip: The number of items skipped - :param sep: the separator. Usually is '; ' (default) or '/ ' - :param join_str: the string which will join the items, default '; '. + and joins then with something. + + Args: + s: the string + count: The number of items included + skip: The number of items skipped + sep: the separator. Usually is '; ' (default) or '/ ' + join_str: the string which will join the items, default '; '. """ skip = int(skip) count = skip + int(count) return join_str.join(s.split(sep)[skip:count]) def tmpl_ifdef(self, field, trueval='', falseval=''): - """ If field exists return trueval or the field (default) - otherwise, emit return falseval (if provided). - - :param field: The name of the field - :param trueval: The string if the condition is true - :param falseval: The string if the condition is false - :return: The string, based on condition + """ If field exists returns trueval or the field (default) + otherwise, emits return falseval (if provided). + + Args: + field: The name of the field + trueval: The string if the condition is true + falseval: The string if the condition is false + + Returns: + The string, based on condition. """ if field in self.item: return trueval if trueval else self.item.formatted().get(field) From cdb6b21f1af0e72082e6ad8dee559a2c9a87b73f Mon Sep 17 00:00:00 2001 From: Patrick Nicholson Date: Tue, 7 Dec 2021 18:47:55 -0500 Subject: [PATCH 058/357] Adding limit and its documentation --- beetsplug/limit.py | 95 ++++++++++++++++++++++++++++++++++++++++++ docs/plugins/limit.rst | 52 +++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 beetsplug/limit.py create mode 100644 docs/plugins/limit.rst diff --git a/beetsplug/limit.py b/beetsplug/limit.py new file mode 100644 index 000000000..20b1ff263 --- /dev/null +++ b/beetsplug/limit.py @@ -0,0 +1,95 @@ +# This file is part of beets. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +from beets.dbcore import FieldQuery +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand, decargs, print_ +from collections import deque +from itertools import islice + + +def lslimit(lib, opts, args): + """Query command with head/tail""" + + head = opts.head + tail = opts.tail + + if head and tail: + raise RuntimeError("Only use one of --head and --tail") + + query = decargs(args) + if opts.album: + objs = lib.albums(query) + else: + objs = lib.items(query) + + if head: + objs = islice(objs, head) + elif tail: + objs = deque(objs, tail) + + for obj in objs: + print_(format(obj)) + + +lslimit_cmd = Subcommand( + "lslimit", + help="query with optional head or tail" +) + +lslimit_cmd.parser.add_option( + '--head', + action='store', + type="int", + default=None +) + +lslimit_cmd.parser.add_option( + '--tail', + action='store', + type="int", + default=None +) + +lslimit_cmd.parser.add_all_common_options() +lslimit_cmd.func = lslimit + + +class LsLimitPlugin(BeetsPlugin): + def commands(self): + return [lslimit_cmd] + + +class HeadPlugin(BeetsPlugin): + """Head of an arbitrary query. + + This allows a user to limit the results of any query to the first + `pattern` rows. Example usage: return first 10 tracks `beet ls '<10'`. + """ + + def queries(self): + + class HeadQuery(FieldQuery): + """Singleton query implementation that tracks result count.""" + n = 0 + include = True + @classmethod + def value_match(cls, pattern, value): + cls.n += 1 + if cls.include: + cls.include = cls.n <= int(pattern) + return cls.include + + return { + "<": HeadQuery + } \ No newline at end of file diff --git a/docs/plugins/limit.rst b/docs/plugins/limit.rst new file mode 100644 index 000000000..7265ac34d --- /dev/null +++ b/docs/plugins/limit.rst @@ -0,0 +1,52 @@ +Limit Query Plugin +================== + +``limit`` is a plugin to limit a query to the first or last set of +results. We also provide a query prefix ``' Date: Tue, 7 Dec 2021 21:27:57 -0500 Subject: [PATCH 059/357] Tests for limit plugin --- test/test_limit.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/test_limit.py diff --git a/test/test_limit.py b/test/test_limit.py new file mode 100644 index 000000000..1dbe415f4 --- /dev/null +++ b/test/test_limit.py @@ -0,0 +1,80 @@ +# This file is part of beets. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Tests for the 'limit' plugin.""" + +import unittest + +from test import _common +from beetsplug import limit +from beets import config + +from test.helper import TestHelper + + +class LsLimitPluginTest(unittest.TestCase, TestHelper): + + def setUp(self): + self.setup_beets() + self.load_plugins("limit") + self.num_test_items = 10 + assert self.num_test_items % 2 == 0 + self.num_limit = self.num_test_items / 2 + self.num_limit_prefix = "'<" + str(self.num_limit) + "'" + self.track_head_range = "track:.." + str(self.num_limit) + self.track_tail_range = "track:" + str(self.num_limit + 1) + ".." + for item_no, item in enumerate(self.add_item_fixtures(count=self.num_test_items)): + item.track = item_no + 1 + item.store() + + def tearDown(self): + self.teardown_beets() + + def test_no_limit(self): + result = self.run_with_output("lslimit") + self.assertEqual(result.count("\n"), self.num_test_items) + + def test_lslimit_head(self): + result = self.run_with_output("lslimit", "--head", str(self.num_limit)) + self.assertEqual(result.count("\n"), self.num_limit) + + def test_lslimit_tail(self): + result = self.run_with_output("lslimit", "--tail", str(self.num_limit)) + self.assertEqual(result.count("\n"), self.num_limit) + + def test_lslimit_head_invariant(self): + result = self.run_with_output("lslimit", "--head", str(self.num_limit), self.track_tail_range) + self.assertEqual(result.count("\n"), self.num_limit) + + def test_lslimit_tail_invariant(self): + result = self.run_with_output("lslimit", "--tail", str(self.num_limit), self.track_head_range) + self.assertEqual(result.count("\n"), self.num_limit) + + def test_prefix(self): + result = self.run_with_output("ls", self.num_limit_prefix) + self.assertEqual(result.count("\n"), self.num_limit) + + def test_prefix_when_correctly_ordered(self): + result = self.run_with_output("ls", self.track_tail_range, self.num_limit_prefix) + self.assertEqual(result.count("\n"), self.num_limit) + + def test_prefix_when_incorrectly_ordred(self): + result = self.run_with_output("ls", self.num_limit_prefix, self.track_tail_range) + self.assertEqual(result.count("\n"), 0) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 9838369f02a5623eed593b75d517f3657322748f Mon Sep 17 00:00:00 2001 From: Patrick Nicholson Date: Tue, 7 Dec 2021 21:32:39 -0500 Subject: [PATCH 060/357] limit added to changelog --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8d95b7fb0..c76291571 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,10 @@ For packagers: * We fixed a version for the dependency on the `Confuse`_ library. :bug:`4167` +Other new things: + +* :doc:`/plugins/limit`: Limit query results to head or tail (``lslimit`` + command only) 1.6.0 (November 27, 2021) ------------------------- From bbd32639b4c469fe3d6668f1e3bb17d8ba7a70ce Mon Sep 17 00:00:00 2001 From: Katelyn Lindsey Date: Wed, 8 Dec 2021 01:31:16 -0800 Subject: [PATCH 061/357] Fixes inconsistencies in ending quote placements for single-line docstrings. --- beets/library.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/beets/library.py b/beets/library.py index c350d018e..45c1eaf3f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1526,40 +1526,38 @@ class Library(dbcore.Database): @staticmethod def get_default_album_sort(): - """Gets a :class:`Sort` object for albums from the config option. - """ + """Gets a :class:`Sort` object for albums from the config option.""" return dbcore.sort_from_strings( Album, beets.config['sort_album'].as_str_seq()) @staticmethod def get_default_item_sort(): - """Gets a :class:`Sort` object for items from the config option. - """ + """Gets a :class:`Sort` object for items from the config option.""" return dbcore.sort_from_strings( Item, beets.config['sort_item'].as_str_seq()) def albums(self, query=None, sort=None): - """Gets :class:`Album` objects matching the query. - """ + """Gets :class:`Album` objects matching the query.""" return self._fetch(Album, query, sort or self.get_default_album_sort()) def items(self, query=None, sort=None): - """Gets :class:`Item` objects matching the query. - """ + """Gets :class:`Item` objects matching the query.""" return self._fetch(Item, query, sort or self.get_default_item_sort()) # Convenience accessors. def get_item(self, id): - """Fetches a :class:`Item` by its ID. Returns `None` if no match is - found. + """Fetches a :class:`Item` by its ID. + + Returns `None` if no match is found. """ return self._get(Item, id) def get_album(self, item_or_id): """Given an album ID or an item associated with an album, returns - a :class:`Album` object for the album. If no such album exists, - returns `None`. + a :class:`Album` object for the album. + + If no such album exists, returns `None`. """ if isinstance(item_or_id, int): album_id = item_or_id @@ -1583,7 +1581,9 @@ def _int_arg(s): class DefaultTemplateFunctions: """A container class for the default functions provided to path - templates. These functions are contained in an object to provide + templates. + + These functions are contained in an object to provide additional context to the functions -- specifically, the Item being evaluated. """ @@ -1655,8 +1655,7 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_asciify(s): - """Translates non-ASCII characters to their ASCII equivalents. - """ + """Translates non-ASCII characters to their ASCII equivalents.""" return util.asciify_path(s, beets.config['path_sep_replace'].as_str()) @staticmethod From acf576c455e59e8197359d4517f8c0a5a9f362bb Mon Sep 17 00:00:00 2001 From: Katelyn Lindsey Date: Wed, 8 Dec 2021 03:48:55 -0800 Subject: [PATCH 062/357] Fixes linting errors by removing trailing whitespaces. --- beets/library.py | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/beets/library.py b/beets/library.py index 45c1eaf3f..6cfbecbb0 100644 --- a/beets/library.py +++ b/beets/library.py @@ -53,7 +53,7 @@ class PathQuery(dbcore.FieldQuery): """ def __init__(self, field, pattern, fast=True, case_sensitive=None): - """Creates a path query. + """Creates a path query. `pattern` must be a path, either to a file or a directory. @@ -141,9 +141,9 @@ class DateType(types.Float): class PathType(types.Type): - """A dbcore type for filesystem paths. + """A dbcore type for filesystem paths. - These are represented as `bytes` objects, in keeping with + These are represented as `bytes` objects, in keeping with the Unix filesystem abstraction. """ @@ -152,7 +152,7 @@ class PathType(types.Type): model_type = bytes def __init__(self, nullable=False): - """Creates a path type object. + """Creates a path type object. `nullable` controls whether the type may be missing, i.e., None. """ @@ -301,9 +301,9 @@ class FileOperationError(Exception): self.reason = reason def text(self): - """Gets a string representing the error. + """Gets a string representing the error. - Describes both the underlying reason and the file path + Describes both the underlying reason and the file path in question. """ return '{}: {}'.format( @@ -426,9 +426,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): raise KeyError(key) def __getitem__(self, key): - """Gets the value for a key. + """Gets the value for a key. - `artist` and `albumartist` are fallback values for each other + `artist` and `albumartist` are fallback values for each other when not set. """ value = self._get(key) @@ -622,7 +622,7 @@ class Item(LibModel): def __getitem__(self, key): """Gets the value for a field, falling back to the album if - necessary. + necessary. Raises a KeyError if the field is not available. """ @@ -644,7 +644,7 @@ class Item(LibModel): ) def keys(self, computed=False, with_album=True): - """Gets a list of available field names. + """Gets a list of available field names. `with_album` controls whether the album's fields are included. """ @@ -657,7 +657,7 @@ class Item(LibModel): def get(self, key, default=None, with_album=True): """Gets the value for a given key or `default` if it does not - exist. + exist. Set `with_album` to false to skip album fallback. """ @@ -669,7 +669,7 @@ class Item(LibModel): return default def update(self, values): - """Sets all key/value pairs in the mapping. + """Sets all key/value pairs in the mapping. If mtime is specified, it is not reset (as it might otherwise be). """ @@ -874,11 +874,11 @@ class Item(LibModel): # Model methods. def remove(self, delete=False, with_album=True): - """Removes the item. + """Removes the item. - If `delete`, then the associated file is removed from disk. + If `delete`, then the associated file is removed from disk. - If `with_album`, then the item's album (if any) is removed + If `with_album`, then the item's album (if any) is removed if the item was the last in the album. """ super().remove() @@ -902,9 +902,9 @@ class Item(LibModel): def move(self, operation=MoveOperation.MOVE, basedir=None, with_album=True, store=True): """Moves the item to its designated location within the library - directory (provided by destination()). + directory (provided by destination()). - Subdirectories are created as needed. If the operation succeeds, + Subdirectories are created as needed. If the operation succeeds, the item's path field is updated to reflect the new location. Instead of moving the item it can also be copied, linked or hardlinked @@ -951,12 +951,12 @@ class Item(LibModel): def destination(self, fragment=False, basedir=None, platform=None, path_formats=None, replacements=None): """Returns the path in the library directory designated for the - item (i.e., where the file ought to be). + item (i.e., where the file ought to be). - fragment makes this method return just the path fragment underneath - the root library directory; the path is also returned as Unicode instead - of encoded as a bytestring. basedir can override the library's base - directory for the destination. + fragment makes this method return just the path fragment underneath + the root library directory; the path is also returned as Unicode + instead of encoded as a bytestring. basedir can override the library's + base directory for the destination. """ self._check_db() platform = platform or sys.platform @@ -1029,7 +1029,7 @@ class Item(LibModel): class Album(LibModel): """Provides access to information about albums stored in a - library. + library. Reflects the library's "albums" table, including album art. """ @@ -1147,10 +1147,10 @@ class Album(LibModel): def remove(self, delete=False, with_items=True): """Removes this album and all its associated items from the - library. + library. - If delete, then the items' files are also deleted from disk, - along with any album art. The directories containing the album are + If delete, then the items' files are also deleted from disk, + along with any album art. The directories containing the album are also removed (recursively) if empty. Set with_items to False to avoid removing the album's items. @@ -1275,7 +1275,7 @@ class Album(LibModel): def art_destination(self, image, item_dir=None): """Returns a path to the destination for the album art image - for the album. + for the album. `image` is the path of the image that will be moved there (used for its extension). @@ -1306,7 +1306,7 @@ class Album(LibModel): def set_art(self, path, copy=True): """Sets the album's cover art to the image at the given path. - + The image is copied (or moved) into place, replacing any existing art. @@ -1337,11 +1337,11 @@ class Album(LibModel): plugins.send('art_set', album=self) def store(self, fields=None): - """Updates the database with the album information. + """Updates the database with the album information. The album's tracks are also updated. - `fields` represents the fields to be stored. If not specified, + `fields` represents the fields to be stored. If not specified, all fields will be. """ # Get modified track fields. @@ -1464,7 +1464,7 @@ class Library(dbcore.Database): def add(self, obj): """Adds the :class:`Item` or :class:`Album` object to the library - database. + database. Returns the object's new id. """ @@ -1547,7 +1547,7 @@ class Library(dbcore.Database): # Convenience accessors. def get_item(self, id): - """Fetches a :class:`Item` by its ID. + """Fetches a :class:`Item` by its ID. Returns `None` if no match is found. """ @@ -1555,7 +1555,7 @@ class Library(dbcore.Database): def get_album(self, item_or_id): """Given an album ID or an item associated with an album, returns - a :class:`Album` object for the album. + a :class:`Album` object for the album. If no such album exists, returns `None`. """ @@ -1572,7 +1572,7 @@ class Library(dbcore.Database): def _int_arg(s): """Converts a string argument to an integer for use in a template - function. + function. May raise a ValueError. """ @@ -1581,7 +1581,7 @@ def _int_arg(s): class DefaultTemplateFunctions: """A container class for the default functions provided to path - templates. + templates. These functions are contained in an object to provide additional context to the functions -- specifically, the Item being @@ -1590,9 +1590,9 @@ class DefaultTemplateFunctions: _prefix = 'tmpl_' def __init__(self, item=None, lib=None): - """Parametrizes the functions. + """Parametrizes the functions. - If `item` or `lib` is None, then some functions (namely, ``aunique``) + If `item` or `lib` is None, then some functions (namely, ``aunique``) will always evaluate to the empty string. """ self.item = item @@ -1600,7 +1600,7 @@ class DefaultTemplateFunctions: def functions(self): """Returns a dictionary containing the functions defined in this - object. + object. The keys are function names (as exposed in templates) and the values are Python functions. @@ -1666,7 +1666,7 @@ class DefaultTemplateFunctions: def tmpl_aunique(self, keys=None, disam=None, bracket=None): """Generates a string that is guaranteed to be unique among all - albums in the library who share the same set of keys. + albums in the library who share the same set of keys. A fields from "disam" is used in the string if one is sufficient to disambiguate the albums. Otherwise, a fallback opaque value is @@ -1761,7 +1761,7 @@ class DefaultTemplateFunctions: def tmpl_first(s, count=1, skip=0, sep='; ', join_str='; '): """ Gets the item(s) from x to y in a string separated by something and joins then with something. - + Args: s: the string count: The number of items included @@ -1776,12 +1776,12 @@ class DefaultTemplateFunctions: def tmpl_ifdef(self, field, trueval='', falseval=''): """ If field exists returns trueval or the field (default) otherwise, emits return falseval (if provided). - + Args: field: The name of the field trueval: The string if the condition is true falseval: The string if the condition is false - + Returns: The string, based on condition. """ From 2f42c8b1c019a90448d33d940b609c18ba644cbc Mon Sep 17 00:00:00 2001 From: Katelyn Lindsey Date: Thu, 9 Dec 2021 18:02:23 -0800 Subject: [PATCH 063/357] Alters docstrings in library.py to be imperative-style. --- beets/library.py | 177 ++++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 87 deletions(-) diff --git a/beets/library.py b/beets/library.py index 6cfbecbb0..3cdc713bb 100644 --- a/beets/library.py +++ b/beets/library.py @@ -53,7 +53,7 @@ class PathQuery(dbcore.FieldQuery): """ def __init__(self, field, pattern, fast=True, case_sensitive=None): - """Creates a path query. + """Create a path query. `pattern` must be a path, either to a file or a directory. @@ -80,7 +80,7 @@ class PathQuery(dbcore.FieldQuery): @classmethod def is_path_query(cls, query_part): - """Tries to guess whether a unicode query part is a path query. + """Try to guess whether a unicode query part is a path query. Condition: separator precedes colon and the file exists. """ @@ -152,7 +152,7 @@ class PathType(types.Type): model_type = bytes def __init__(self, nullable=False): - """Creates a path type object. + """Create a path type object. `nullable` controls whether the type may be missing, i.e., None. """ @@ -247,7 +247,7 @@ class DurationType(types.Float): # Library-specific sort types. class SmartArtistSort(dbcore.query.Sort): - """Sorts by artist (either album artist or track artist), + """Sort by artist (either album artist or track artist), prioritizing the sort field over the raw field. """ @@ -286,14 +286,14 @@ PF_KEY_DEFAULT = 'default' # Exceptions. class FileOperationError(Exception): - """Indicates an error when interacting with a file on disk. + """Indicate an error when interacting with a file on disk. Possibilities include an unsupported media type, a permissions error, and an unhandled Mutagen exception. """ def __init__(self, path, reason): - """Creates an exception describing an operation on the file at + """Create an exception describing an operation on the file at `path` with the underlying (chained) exception `reason`. """ super().__init__(path, reason) @@ -301,9 +301,9 @@ class FileOperationError(Exception): self.reason = reason def text(self): - """Gets a string representing the error. + """Get a string representing the error. - Describes both the underlying reason and the file path + Describe both the underlying reason and the file path in question. """ return '{}: {}'.format( @@ -369,7 +369,7 @@ class LibModel(dbcore.Model): class FormattedItemMapping(dbcore.db.FormattedMapping): - """Adds lookup for album-level fields. + """Add lookup for album-level fields. Album-level fields take precedence if `for_path` is true. """ @@ -412,9 +412,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): return self.item._cached_album def _get(self, key): - """Gets the value for a key, either from the album or the item. + """Get the value for a key, either from the album or the item. - Raises a KeyError for invalid keys. + Raise a KeyError for invalid keys. """ if self.for_path and key in self.album_keys: return self._get_formatted(self.album, key) @@ -426,7 +426,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): raise KeyError(key) def __getitem__(self, key): - """Gets the value for a key. + """Get the value for a key. `artist` and `albumartist` are fallback values for each other when not set. @@ -454,7 +454,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): class Item(LibModel): - """Represents a song or track.""" + """Represent a song or track.""" _table = 'items' _flex_table = 'item_attributes' _fields = { @@ -597,7 +597,7 @@ class Item(LibModel): @classmethod def from_path(cls, path): - """Creates a new item from the media file at the specified path.""" + """Create a new item from the media file at the specified path.""" # Initiate with values that aren't read from files. i = cls(album_id=None) i.read(path) @@ -605,7 +605,7 @@ class Item(LibModel): return i def __setitem__(self, key, value): - """Sets the item's value for a standard field or a flexattr.""" + """Set the item's value for a standard field or a flexattr.""" # Encode unicode paths and read buffers. if key == 'path': if isinstance(value, str): @@ -621,10 +621,10 @@ class Item(LibModel): self.mtime = 0 # Reset mtime on dirty. def __getitem__(self, key): - """Gets the value for a field, falling back to the album if + """Get the value for a field, falling back to the album if necessary. - Raises a KeyError if the field is not available. + Raise a KeyError if the field is not available. """ try: return super().__getitem__(key) @@ -644,7 +644,7 @@ class Item(LibModel): ) def keys(self, computed=False, with_album=True): - """Gets a list of available field names. + """Get a list of available field names. `with_album` controls whether the album's fields are included. """ @@ -656,7 +656,7 @@ class Item(LibModel): return keys def get(self, key, default=None, with_album=True): - """Gets the value for a given key or `default` if it does not + """Get the value for a given key or `default` if it does not exist. Set `with_album` to false to skip album fallback. @@ -669,7 +669,7 @@ class Item(LibModel): return default def update(self, values): - """Sets all key/value pairs in the mapping. + """Set all key/value pairs in the mapping. If mtime is specified, it is not reset (as it might otherwise be). """ @@ -683,7 +683,7 @@ class Item(LibModel): setattr(self, key, None) def get_album(self): - """Gets the Album object that this item belongs to, if any, or + """Get the Album object that this item belongs to, if any, or None if the item is a singleton or is not associated with a library. """ @@ -694,13 +694,13 @@ class Item(LibModel): # Interaction with file metadata. def read(self, read_path=None): - """Reads the metadata from the associated file. + """Read the metadata from the associated file. - If `read_path` is specified, reads metadata from that file - instead. Updates all the properties in `_media_fields` + If `read_path` is specified, read metadata from that file + instead. Update all the properties in `_media_fields` from the media file. - Raises a `ReadError` if the file could not be read. + Raise a `ReadError` if the file could not be read. """ if read_path is None: read_path = self.path @@ -725,7 +725,7 @@ class Item(LibModel): self.path = read_path def write(self, path=None, tags=None, id3v23=None): - """Writes the item's metadata to a media file. + """Write the item's metadata to a media file. All fields in `_media_fields` are written to disk according to the values on this object. @@ -776,10 +776,10 @@ class Item(LibModel): plugins.send('after_write', item=self, path=path) def try_write(self, *args, **kwargs): - """Calls `write()` but catches and logs `FileOperationError` + """Call `write()` but catch and log `FileOperationError` exceptions. - Returns `False` an exception was caught and `True` otherwise. + Return `False` an exception was caught and `True` otherwise. """ try: self.write(*args, **kwargs) @@ -789,7 +789,7 @@ class Item(LibModel): return False def try_sync(self, write, move, with_album=True): - """Synchronizes the item with the database and, possibly, updates its + """Synchronize the item with the database and, possibly, update its tags on disk and its path (by moving the file). `write` indicates whether to write new tags into the file. Similarly, @@ -813,7 +813,7 @@ class Item(LibModel): # Files themselves. def move_file(self, dest, operation=MoveOperation.MOVE): - """Moves, copies, links or hardlinks the item depending on `operation`, + """Move, copy, link or hardlink the item depending on `operation`, updating the path value if the move succeeds. If a file exists at `dest`, then it is slightly modified to be unique. @@ -855,15 +855,15 @@ class Item(LibModel): self.path = dest def current_mtime(self): - """Returns the current mtime of the file, rounded to the nearest + """Return the current mtime of the file, rounded to the nearest integer. """ return int(os.path.getmtime(syspath(self.path))) def try_filesize(self): - """Gets the size of the underlying file in bytes. + """Get the size of the underlying file in bytes. - If the file is missing, returns 0 (and logs a warning). + If the file is missing, return 0 (and log a warning). """ try: return os.path.getsize(syspath(self.path)) @@ -874,7 +874,7 @@ class Item(LibModel): # Model methods. def remove(self, delete=False, with_album=True): - """Removes the item. + """Remove the item. If `delete`, then the associated file is removed from disk. @@ -901,7 +901,7 @@ class Item(LibModel): def move(self, operation=MoveOperation.MOVE, basedir=None, with_album=True, store=True): - """Moves the item to its designated location within the library + """Move the item to its designated location within the library directory (provided by destination()). Subdirectories are created as needed. If the operation succeeds, @@ -919,8 +919,8 @@ class Item(LibModel): By default, the item is stored to the database if it is in the database, so any dirty fields prior to the move() call will be written as a side effect. - If `store` is `False` however, the item won't be stored and you'll - have to manually store it after invoking this method. + If `store` is `False` however, the item won't be stored and it will + have to be manually stored after invoking this method. """ self._check_db() dest = self.destination(basedir=basedir) @@ -950,7 +950,7 @@ class Item(LibModel): def destination(self, fragment=False, basedir=None, platform=None, path_formats=None, replacements=None): - """Returns the path in the library directory designated for the + """Return the path in the library directory designated for the item (i.e., where the file ought to be). fragment makes this method return just the path fragment underneath @@ -1028,7 +1028,7 @@ class Item(LibModel): class Album(LibModel): - """Provides access to information about albums stored in a + """Provide access to information about albums stored in a library. Reflects the library's "albums" table, including album art. @@ -1140,13 +1140,13 @@ class Album(LibModel): return getters def items(self): - """Returns an iterable over the items associated with this + """Return an iterable over the items associated with this album. """ return self._db.items(dbcore.MatchQuery('album_id', self.id)) def remove(self, delete=False, with_items=True): - """Removes this album and all its associated items from the + """Remove this album and all its associated items from the library. If delete, then the items' files are also deleted from disk, @@ -1172,7 +1172,7 @@ class Album(LibModel): item.remove(delete, False) def move_art(self, operation=MoveOperation.MOVE): - """Moves, copies, links or hardlinks (depending on `operation`) any + """Move, copy, link or hardlink (depending on `operation`) any existing album art so that it remains in the same directory as the items. @@ -1214,7 +1214,7 @@ class Album(LibModel): self.artpath = new_art def move(self, operation=MoveOperation.MOVE, basedir=None, store=True): - """Moves, copies, links or hardlinks (depending on `operation`) + """Move, copy, link or hardlink (depending on `operation`) all items to their destination. Any album art moves along with them. `basedir` overrides the library base directory for the destination. @@ -1245,7 +1245,7 @@ class Album(LibModel): self.store() def item_dir(self): - """Returns the directory containing the album's first item, + """Return the directory containing the album's first item, provided that such an item exists. """ item = self.items().get() @@ -1254,7 +1254,7 @@ class Album(LibModel): return os.path.dirname(item.path) def _albumtotal(self): - """Returns the total number of tracks on all discs on the album.""" + """Return the total number of tracks on all discs on the album.""" if self.disctotal == 1 or not beets.config['per_disc_numbering']: return self.items()[0].tracktotal @@ -1274,7 +1274,7 @@ class Album(LibModel): return total def art_destination(self, image, item_dir=None): - """Returns a path to the destination for the album art image + """Return a path to the destination for the album art image for the album. `image` is the path of the image that will be @@ -1305,12 +1305,12 @@ class Album(LibModel): return bytestring_path(dest) def set_art(self, path, copy=True): - """Sets the album's cover art to the image at the given path. + """Set the album's cover art to the image at the given path. The image is copied (or moved) into place, replacing any existing art. - Sends an 'art_set' event with `self` as the sole argument. + Send an 'art_set' event with `self` as the sole argument. """ path = bytestring_path(path) oldart = self.artpath @@ -1337,7 +1337,7 @@ class Album(LibModel): plugins.send('art_set', album=self) def store(self, fields=None): - """Updates the database with the album information. + """Update the database with the album information. The album's tracks are also updated. @@ -1359,8 +1359,8 @@ class Album(LibModel): item.store() def try_sync(self, write, move): - """Synchronizes the album and its items with the database. - Optionally, also writes any new tags into the files and update + """Synchronize the album and its items with the database. + Optionally, also write any new tags into the files and update their paths. `write` indicates whether to write tags to the item files, and @@ -1375,7 +1375,7 @@ class Album(LibModel): # Query construction helpers. def parse_query_parts(parts, model_cls): - """Given a beets query string as a list of components, returns the + """Given a beets query string as a list of components, return the `Query` and `Sort` they represent. Like `dbcore.parse_sorted_query`, with beets query prefixes and @@ -1411,7 +1411,7 @@ def parse_query_parts(parts, model_cls): def parse_query_string(s, model_cls): - """Given a beets query string, returns the `Query` and `Sort` they + """Given a beets query string, return the `Query` and `Sort` they represent. The string is split into components using shell-like syntax. @@ -1427,10 +1427,11 @@ def parse_query_string(s, model_cls): def _sqlite_bytelower(bytestring): """ A custom ``bytelower`` sqlite function so we can compare - bytestrings in a semi case insensitive fashion. This is to work - around sqlite builds are that compiled with - ``-DSQLITE_LIKE_DOESNT_MATCH_BLOBS``. See - ``https://github.com/beetbox/beets/issues/2172`` for details. + bytestrings in a semi case insensitive fashion. + + This is to work around sqlite builds are that compiled with + ``-DSQLITE_LIKE_DOESNT_MATCH_BLOBS``. See + ``https://github.com/beetbox/beets/issues/2172`` for details. """ return bytestring.lower() @@ -1463,17 +1464,17 @@ class Library(dbcore.Database): # Adding objects to the database. def add(self, obj): - """Adds the :class:`Item` or :class:`Album` object to the library + """Add the :class:`Item` or :class:`Album` object to the library database. - Returns the object's new id. + Return the object's new id. """ obj.add(self) self._memotable = {} return obj.id def add_album(self, items): - """Creates a new album consisting of a list of items. + """Create a new album consisting of a list of items. The items are added to the database if they don't yet have an ID. Return a new :class:`Album` object. The list items must not @@ -1502,8 +1503,10 @@ class Library(dbcore.Database): # Querying. def _fetch(self, model_cls, query, sort=None): - """Parse a query and fetch. If a order specification is present - in the query string the `sort` argument is ignored. + """Parse a query and fetch. + + If an order specification is present in the query string + the `sort` argument is ignored. """ # Parse the query, if necessary. try: @@ -1526,38 +1529,38 @@ class Library(dbcore.Database): @staticmethod def get_default_album_sort(): - """Gets a :class:`Sort` object for albums from the config option.""" + """Get a :class:`Sort` object for albums from the config option.""" return dbcore.sort_from_strings( Album, beets.config['sort_album'].as_str_seq()) @staticmethod def get_default_item_sort(): - """Gets a :class:`Sort` object for items from the config option.""" + """Get a :class:`Sort` object for items from the config option.""" return dbcore.sort_from_strings( Item, beets.config['sort_item'].as_str_seq()) def albums(self, query=None, sort=None): - """Gets :class:`Album` objects matching the query.""" + """Get :class:`Album` objects matching the query.""" return self._fetch(Album, query, sort or self.get_default_album_sort()) def items(self, query=None, sort=None): - """Gets :class:`Item` objects matching the query.""" + """Get :class:`Item` objects matching the query.""" return self._fetch(Item, query, sort or self.get_default_item_sort()) # Convenience accessors. def get_item(self, id): - """Fetches a :class:`Item` by its ID. + """Fetch a :class:`Item` by its ID. - Returns `None` if no match is found. + Return `None` if no match is found. """ return self._get(Item, id) def get_album(self, item_or_id): - """Given an album ID or an item associated with an album, returns + """Given an album ID or an item associated with an album, return a :class:`Album` object for the album. - If no such album exists, returns `None`. + If no such album exists, return `None`. """ if isinstance(item_or_id, int): album_id = item_or_id @@ -1571,7 +1574,7 @@ class Library(dbcore.Database): # Default path template resources. def _int_arg(s): - """Converts a string argument to an integer for use in a template + """Convert a string argument to an integer for use in a template function. May raise a ValueError. @@ -1590,7 +1593,7 @@ class DefaultTemplateFunctions: _prefix = 'tmpl_' def __init__(self, item=None, lib=None): - """Parametrizes the functions. + """Parametrize the functions. If `item` or `lib` is None, then some functions (namely, ``aunique``) will always evaluate to the empty string. @@ -1599,7 +1602,7 @@ class DefaultTemplateFunctions: self.lib = lib def functions(self): - """Returns a dictionary containing the functions defined in this + """Return a dictionary containing the functions defined in this object. The keys are function names (as exposed in templates) @@ -1612,33 +1615,33 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_lower(s): - """Converts a string to lower case.""" + """Convert a string to lower case.""" return s.lower() @staticmethod def tmpl_upper(s): - """Converts a string to upper case.""" + """Convert a string to upper case.""" return s.upper() @staticmethod def tmpl_title(s): - """Converts a string to title case.""" + """Convert a string to title case.""" return string.capwords(s) @staticmethod def tmpl_left(s, chars): - """Gets the leftmost characters of a string.""" + """Get the leftmost characters of a string.""" return s[0:_int_arg(chars)] @staticmethod def tmpl_right(s, chars): - """Gets the rightmost characters of a string.""" + """Get the rightmost characters of a string.""" return s[-_int_arg(chars):] @staticmethod def tmpl_if(condition, trueval, falseval=''): - """If ``condition`` is nonempty and nonzero, emits ``trueval``; - otherwise, emits ``falseval`` (if provided). + """If ``condition`` is nonempty and nonzero, emit ``trueval``; + otherwise, emit ``falseval`` (if provided). """ try: int_condition = _int_arg(condition) @@ -1655,17 +1658,17 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_asciify(s): - """Translates non-ASCII characters to their ASCII equivalents.""" + """Translate non-ASCII characters to their ASCII equivalents.""" return util.asciify_path(s, beets.config['path_sep_replace'].as_str()) @staticmethod def tmpl_time(s, fmt): - """Formats a time value using `strftime`.""" + """Format a time value using `strftime`.""" cur_fmt = beets.config['time_format'].as_str() return time.strftime(fmt, time.strptime(s, cur_fmt)) def tmpl_aunique(self, keys=None, disam=None, bracket=None): - """Generates a string that is guaranteed to be unique among all + """Generate a string that is guaranteed to be unique among all albums in the library who share the same set of keys. A fields from "disam" is used in the string if one is sufficient to @@ -1759,8 +1762,8 @@ class DefaultTemplateFunctions: @staticmethod def tmpl_first(s, count=1, skip=0, sep='; ', join_str='; '): - """ Gets the item(s) from x to y in a string separated by something - and joins then with something. + """Get the item(s) from x to y in a string separated by something + and join then with something. Args: s: the string @@ -1774,8 +1777,8 @@ class DefaultTemplateFunctions: return join_str.join(s.split(sep)[skip:count]) def tmpl_ifdef(self, field, trueval='', falseval=''): - """ If field exists returns trueval or the field (default) - otherwise, emits return falseval (if provided). + """ If field exists return trueval or the field (default) + otherwise, emit return falseval (if provided). Args: field: The name of the field From 9f13eaeb15d12c32f92563d8183afb0997d23175 Mon Sep 17 00:00:00 2001 From: ybnd Date: Fri, 10 Dec 2021 18:33:24 +0100 Subject: [PATCH 064/357] Make r128 fields floats --- beets/library.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/library.py b/beets/library.py index 888836cd9..56aff6351 100644 --- a/beets/library.py +++ b/beets/library.py @@ -515,8 +515,8 @@ class Item(LibModel): 'rg_track_peak': types.NULL_FLOAT, 'rg_album_gain': types.NULL_FLOAT, 'rg_album_peak': types.NULL_FLOAT, - 'r128_track_gain': types.NullPaddedInt(6), - 'r128_album_gain': types.NullPaddedInt(6), + 'r128_track_gain': types.NULL_FLOAT, + 'r128_album_gain': types.NULL_FLOAT, 'original_year': types.PaddedInt(4), 'original_month': types.PaddedInt(2), 'original_day': types.PaddedInt(2), @@ -1058,7 +1058,7 @@ class Album(LibModel): 'releasegroupdisambig': types.STRING, 'rg_album_gain': types.NULL_FLOAT, 'rg_album_peak': types.NULL_FLOAT, - 'r128_album_gain': types.NullPaddedInt(6), + 'r128_album_gain': types.NULL_FLOAT, 'original_year': types.PaddedInt(4), 'original_month': types.PaddedInt(2), 'original_day': types.PaddedInt(2), From 38fc1d453248545d8fd44edebdf90dd57d1bc907 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 12 Dec 2021 12:10:50 +0100 Subject: [PATCH 065/357] Add changelog entry for r128 type change --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8d95b7fb0..3dd225864 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,10 @@ Bug fixes: * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. * :doc:`/plugins/unimported`: The new ``ignore_subdirectories`` configuration option added in 1.6.0 now has a default value if it hasn't been set. +* :doc:`/plugins/replaygain`: The type of the internal ``r128_track_gain`` and + ``r128_album_gain`` fields was changed from integer to float to fix loss of + precision due to truncation. + :bug:`4169` For packagers: From ca37c94337c0f1a99ce7bd4e14ce9ce829370750 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 12 Dec 2021 13:26:37 +0100 Subject: [PATCH 066/357] kodiupdate: Support multiple instances --- beetsplug/kodiupdate.py | 51 ++++++++++++++++++++++--------------- docs/changelog.rst | 5 ++++ docs/plugins/kodiupdate.rst | 13 ++++++++++ 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/beetsplug/kodiupdate.py b/beetsplug/kodiupdate.py index 2a885d2c2..ce6fb80ee 100644 --- a/beetsplug/kodiupdate.py +++ b/beetsplug/kodiupdate.py @@ -54,11 +54,12 @@ class KodiUpdate(BeetsPlugin): super().__init__() # Adding defaults. - config['kodi'].add({ + config['kodi'].add([{ 'host': 'localhost', 'port': 8080, 'user': 'kodi', - 'pwd': 'kodi'}) + 'pwd': 'kodi' + }]) config['kodi']['pwd'].redact = True self.register_listener('database_change', self.listen_for_db_change) @@ -72,24 +73,34 @@ class KodiUpdate(BeetsPlugin): """ self._log.info('Requesting a Kodi library update...') - # Try to send update request. - try: - r = update_kodi( - config['kodi']['host'].get(), - config['kodi']['port'].get(), - config['kodi']['user'].get(), - config['kodi']['pwd'].get()) - r.raise_for_status() + kodi = config['kodi'].get() - except requests.exceptions.RequestException as e: - self._log.warning('Kodi update failed: {0}', - str(e)) - return + # Backwards compatibility in case not configured as an array + if not isinstance(kodi, list): + kodi = [kodi] - json = r.json() - if json.get('result') != 'OK': - self._log.warning('Kodi update failed: JSON response was {0!r}', - json) - return + for instance in kodi: + # Try to send update request. + try: + r = update_kodi( + instance['host'], + instance['port'], + instance['user'], + instance['pwd'] + ) + r.raise_for_status() - self._log.info('Kodi update triggered') + json = r.json() + if json.get('result') != 'OK': + self._log.warning( + 'Kodi update failed: JSON response was {0!r}', json + ) + continue + + self._log.info( + 'Kodi update triggered for {0}:{1}', + instance['host'], instance['port'] + ) + except requests.exceptions.RequestException as e: + self._log.warning('Kodi update failed: {0}', str(e)) + continue diff --git a/docs/changelog.rst b/docs/changelog.rst index 8d95b7fb0..2daadd6bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,11 @@ Changelog Changelog goes here! +New features: + +* :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances + :bug:`4101` + Bug fixes: * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. diff --git a/docs/plugins/kodiupdate.rst b/docs/plugins/kodiupdate.rst index f521a8000..6713f0506 100644 --- a/docs/plugins/kodiupdate.rst +++ b/docs/plugins/kodiupdate.rst @@ -16,6 +16,19 @@ which looks like this:: user: kodi pwd: kodi +To update multiple Kodi instances, specify them as an array:: + + kodi: + - host: x.x.x.x + port: 8080 + user: kodi + pwd: kodi + - host: y.y.y.y + port: 8081 + user: kodi2 + pwd: kodi2 + + To use the ``kodiupdate`` plugin you need to install the `requests`_ library with:: pip install requests From 82a2a223655fb615be367bf3dccb8d963986ea5b Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 12 Dec 2021 12:53:16 +0100 Subject: [PATCH 067/357] deezer: Tolerate missing fields when searching for singletons --- beetsplug/deezer.py | 6 +++--- docs/changelog.rst | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index 5f158f936..d21b1e9ed 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -128,9 +128,9 @@ class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): artist=artist, artist_id=artist_id, length=track_data['duration'], - index=track_data['track_position'], - medium=track_data['disk_number'], - medium_index=track_data['track_position'], + index=track_data.get('track_position'), + medium=track_data.get('disk_number'), + medium_index=track_data.get('track_position'), data_source=self.data_source, data_url=track_data['link'], ) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8d95b7fb0..538a9d655 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,9 @@ Bug fixes: * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. * :doc:`/plugins/unimported`: The new ``ignore_subdirectories`` configuration option added in 1.6.0 now has a default value if it hasn't been set. +* :doc:`/plugins/deezer`: Tolerate missing fields when searching for singleton + tracks + :bug:`4116` For packagers: From 4f83b2d8a6f3c5c0dacbd269d96cc2a41d5ba3c6 Mon Sep 17 00:00:00 2001 From: Julien Cassette Date: Sat, 20 Nov 2021 18:07:21 +0100 Subject: [PATCH 068/357] Add the item fields bitrate_mode, encoder_info and encoder_settings --- beets/library.py | 3 +++ docs/changelog.rst | 2 ++ docs/reference/pathformat.rst | 3 +++ setup.py | 2 +- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index a56575a52..c8993f85b 100644 --- a/beets/library.py +++ b/beets/library.py @@ -531,6 +531,9 @@ class Item(LibModel): 'length': DurationType(), 'bitrate': types.ScaledInt(1000, 'kbps'), + 'bitrate_mode': types.STRING, + 'encoder_info': types.STRING, + 'encoder_settings': types.STRING, 'format': types.STRING, 'samplerate': types.ScaledInt(1000, 'kHz'), 'bitdepth': types.INTEGER, diff --git a/docs/changelog.rst b/docs/changelog.rst index 4d27107ab..49ed43b93 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ New features: * :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances :bug:`4101` +* Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. Bug fixes: @@ -28,6 +29,7 @@ For packagers: * We fixed a version for the dependency on the `Confuse`_ library. :bug:`4167` +* The minimum required version of :pypi:`mediafile` is now 0.9.0. 1.6.0 (November 27, 2021) diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index 9213cae4b..8f27027ac 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -235,6 +235,9 @@ Audio information: * length (in seconds) * bitrate (in kilobits per second, with units: e.g., "192kbps") +* bitrate_mode (eg. "CBR", "VBR" or "ABR", only available for the MP3 format) +* encoder_info (eg. "LAME 3.97.0", only available for some formats) +* encoder_settings (eg. "-V2", only available for the MP3 format) * format (e.g., "MP3" or "FLAC") * channels * bitdepth (only available for some formats) diff --git a/setup.py b/setup.py index fa92448a2..4c4f7d629 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ setup( 'unidecode', 'musicbrainzngs>=0.4', 'pyyaml', - 'mediafile>=0.2.0', + 'mediafile>=0.9.0', 'confuse>=1.5.0', 'munkres>=1.0.0', 'jellyfish', From 3b9382d8084408d8d344e30b062d51871b910dcc Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 18 Dec 2021 13:39:12 -0800 Subject: [PATCH 069/357] Syntax fix: e.g. --- docs/reference/pathformat.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index 8f27027ac..f6f2e06cc 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -235,9 +235,9 @@ Audio information: * length (in seconds) * bitrate (in kilobits per second, with units: e.g., "192kbps") -* bitrate_mode (eg. "CBR", "VBR" or "ABR", only available for the MP3 format) -* encoder_info (eg. "LAME 3.97.0", only available for some formats) -* encoder_settings (eg. "-V2", only available for the MP3 format) +* bitrate_mode (e.g., "CBR", "VBR" or "ABR", only available for the MP3 format) +* encoder_info (e.g., "LAME 3.97.0", only available for some formats) +* encoder_settings (e.g., "-V2", only available for the MP3 format) * format (e.g., "MP3" or "FLAC") * channels * bitdepth (only available for some formats) From fd761cb1e6c3d0f4d9c52495c7ff7f7bc13498a4 Mon Sep 17 00:00:00 2001 From: Dominik Schrempf Date: Sat, 18 Dec 2021 16:21:25 +0100 Subject: [PATCH 070/357] fix spotify pagination Basically, keep fetching tracks until there are no more available for the specified album. Fixes #4180. --- beetsplug/spotify.py | 9 ++++++++- docs/changelog.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 2529160dd..931078d28 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -194,9 +194,16 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): ) ) + tracks_data = album_data['tracks'] + tracks_items = tracks_data['items'] + while tracks_data['next']: + tracks_data = self._handle_response(requests.get, + tracks_data['next']) + tracks_items.extend(tracks_data['items']) + tracks = [] medium_totals = collections.defaultdict(int) - for i, track_data in enumerate(album_data['tracks']['items'], start=1): + for i, track_data in enumerate(tracks_items, start=1): track = self._get_track(track_data) track.index = i medium_totals[track.medium] += 1 diff --git a/docs/changelog.rst b/docs/changelog.rst index 4d27107ab..3283fc712 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,8 @@ New features: Bug fixes: +* :doc:`/plugins/spotify`: Fix auto tagger pagination issues (fetch beyond the + first 50 tracks of a release). * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. * :doc:`/plugins/unimported`: The new ``ignore_subdirectories`` configuration option added in 1.6.0 now has a default value if it hasn't been set. From 969f045610da16a0e2ce0a211d32ea11150ae63f Mon Sep 17 00:00:00 2001 From: Dominik Schrempf Date: Sun, 19 Dec 2021 00:53:39 +0100 Subject: [PATCH 071/357] fix deezer pagination See #4180, and #4198. --- beetsplug/deezer.py | 9 +++++++-- docs/changelog.rst | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/beetsplug/deezer.py b/beetsplug/deezer.py index d21b1e9ed..221673b50 100644 --- a/beetsplug/deezer.py +++ b/beetsplug/deezer.py @@ -77,11 +77,16 @@ class DeezerPlugin(MetadataSourcePlugin, BeetsPlugin): "by {} API: '{}'".format(self.data_source, release_date) ) - tracks_data = requests.get( + tracks_obj = requests.get( self.album_url + deezer_id + '/tracks' - ).json()['data'] + ).json() + tracks_data = tracks_obj['data'] if not tracks_data: return None + while "next" in tracks_obj: + tracks_obj = requests.get(tracks_obj['next']).json() + tracks_data.extend(tracks_obj['data']) + tracks = [] medium_totals = collections.defaultdict(int) for i, track_data in enumerate(tracks_data, start=1): diff --git a/docs/changelog.rst b/docs/changelog.rst index 4d27107ab..6b33e884d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,7 +12,8 @@ New features: :bug:`4101` Bug fixes: - +* :doc:`/plugins/deezer`: Fix auto tagger pagination issues (fetch beyond the + first 25 tracks of a release). * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. * :doc:`/plugins/unimported`: The new ``ignore_subdirectories`` configuration option added in 1.6.0 now has a default value if it hasn't been set. From ba3569afa121abaa6830887824b4f2dbeb9a71b0 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 19 Dec 2021 07:31:29 -0800 Subject: [PATCH 072/357] Add a paragraph space --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6b33e884d..8307ff7b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ New features: :bug:`4101` Bug fixes: + * :doc:`/plugins/deezer`: Fix auto tagger pagination issues (fetch beyond the first 25 tracks of a release). * :doc:`/plugins/lyrics`: Fix Genius search by using query params instead of body. From fcb73ad095f1d2851f9819df69079ceec0a9e6c7 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sun, 19 Dec 2021 15:41:31 +0000 Subject: [PATCH 073/357] Clarify docs for writing plugins If the beetsplug directory itself is added to the python path then python does not see the plugin as part of the beetsplug namespace. --- docs/dev/plugins.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 3956aa760..49123b8a0 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -36,7 +36,8 @@ found therein. Here's a skeleton of a plugin file:: Once you have your ``BeetsPlugin`` subclass, there's a variety of things your plugin can do. (Read on!) -To use your new plugin, make sure your ``beetsplug`` directory is in the Python +To use your new plugin, make sure the directory that contains your +``beetsplug`` directory is in the Python path (using ``PYTHONPATH`` or by installing in a `virtualenv`_, for example). Then, as described above, edit your ``config.yaml`` to include ``plugins: myawesomeplugin`` (substituting the name of the Python module From 1dc5163cb4845a06e808d80edd96b747a242b1b5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 22 Dec 2021 09:34:41 -0800 Subject: [PATCH 074/357] Create security policy --- SECURITY.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..4d7a3b9fc --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,11 @@ +# Security Policy + +## Supported Versions + +We currently support only the latest release of beets. + +## Reporting a Vulnerability + +To report a security vulnerability, please send email to [our Zulip team][z]. + +[z]: mailto:email.218c36e48d78cf125c0a6219a6c2a417.show-sender@streams.zulipchat.com From 4bb695bcdbada9c8153442688e8494199f015f04 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 26 Dec 2021 18:01:17 -0800 Subject: [PATCH 075/357] Fix copying for atomic file moves Fixes #4168. Also closes #4192, which it supersedes. The original problem is that this implementation used bytestrings incorrectly to invoke `mktemp`. However, `mktemp` is deprecated, so this PR just avoids it altogether. Fortunately, the non-deprecated APIs in `tempfile` support all-bytes arguments. --- beets/util/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index d58bb28e4..9f96d4561 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -495,13 +495,20 @@ def move(path, dest, replace=False): try: os.replace(path, dest) except OSError: - tmp = tempfile.mktemp(suffix='.beets', - prefix=py3_path(b'.' + os.path.basename(dest)), - dir=py3_path(os.path.dirname(dest))) - tmp = syspath(tmp) + # Copy the file to a temporary destination. + tmp = tempfile.NamedTemporaryFile(suffix=b'.beets', + prefix=b'.' + os.path.basename(dest), + dir=os.path.dirname(dest), + delete=False) try: - shutil.copyfile(path, tmp) - os.replace(tmp, dest) + with open(path, 'rb') as f: + shutil.copyfileobj(f, tmp) + finally: + tmp.close() + + # Move the copied file into place. + try: + os.replace(tmp.name, dest) tmp = None os.remove(path) except OSError as exc: From 592c3fa3561edfa0921b37edbd1cec910e47336f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 26 Dec 2021 18:05:56 -0800 Subject: [PATCH 076/357] Changelog for #4168 fix --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3fbe5f1fc..51fbadb5e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,9 @@ Bug fixes: ``r128_album_gain`` fields was changed from integer to float to fix loss of precision due to truncation. :bug:`4169` +* Fix a regression in the previous release that caused a `TypeError` when + moving files across filesystems. + :bug:`4168` For packagers: From 24bc4e77e2a41be7243e8703b301e24fccb520e0 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Sun, 26 Dec 2021 21:56:57 -0500 Subject: [PATCH 077/357] Being more careful about truthiness and catching negative values (they could be supported, but it's probably not intuitive). Moving command into single plugin as suggested. Fixing linter objections. --- beetsplug/limit.py | 62 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/beetsplug/limit.py b/beetsplug/limit.py index 20b1ff263..6dfb2c40d 100644 --- a/beetsplug/limit.py +++ b/beetsplug/limit.py @@ -20,43 +20,42 @@ from itertools import islice def lslimit(lib, opts, args): """Query command with head/tail""" - - head = opts.head - tail = opts.tail - if head and tail: - raise RuntimeError("Only use one of --head and --tail") - + if (opts.head is not None) and (opts.tail is not None): + raise ValueError("Only use one of --head and --tail") + if (opts.head or opts.tail or 0) < 0: + raise ValueError("Limit value must be non-negative") + query = decargs(args) if opts.album: objs = lib.albums(query) else: objs = lib.items(query) - - if head: - objs = islice(objs, head) - elif tail: - objs = deque(objs, tail) + + if opts.head is not None: + objs = islice(objs, opts.head) + elif opts.tail is not None: + objs = deque(objs, opts.tail) for obj in objs: print_(format(obj)) lslimit_cmd = Subcommand( - "lslimit", + "lslimit", help="query with optional head or tail" ) lslimit_cmd.parser.add_option( - '--head', - action='store', + '--head', + action='store', type="int", default=None ) lslimit_cmd.parser.add_option( '--tail', - action='store', + action='store', type="int", default=None ) @@ -65,31 +64,32 @@ lslimit_cmd.parser.add_all_common_options() lslimit_cmd.func = lslimit -class LsLimitPlugin(BeetsPlugin): +class LimitPlugin(BeetsPlugin): + """Query limit functionality via command and query prefix + """ + def commands(self): return [lslimit_cmd] - -class HeadPlugin(BeetsPlugin): - """Head of an arbitrary query. - - This allows a user to limit the results of any query to the first - `pattern` rows. Example usage: return first 10 tracks `beet ls '<10'`. - """ - def queries(self): class HeadQuery(FieldQuery): - """Singleton query implementation that tracks result count.""" + """This inner class pattern allows the query to track state + """ n = 0 - include = True + N = None + @classmethod def value_match(cls, pattern, value): + + if cls.N is None: + cls.N = int(pattern) + if cls.N < 0: + raise ValueError("Limit value must be non-negative") + cls.n += 1 - if cls.include: - cls.include = cls.n <= int(pattern) - return cls.include - + return cls.n <= cls.N + return { "<": HeadQuery - } \ No newline at end of file + } From 126b4e94cfdd4e72b4db54664dcf8e4b8f6e0c26 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Sun, 26 Dec 2021 21:57:29 -0500 Subject: [PATCH 078/357] Fixing doc typo and adding limit to toctree. --- docs/plugins/index.rst | 1 + docs/plugins/limit.rst | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 5ca8794fd..3d8b97606 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -98,6 +98,7 @@ following to your configuration:: kodiupdate lastgenre lastimport + limit loadext lyrics mbcollection diff --git a/docs/plugins/limit.rst b/docs/plugins/limit.rst index 7265ac34d..74bd47cb6 100644 --- a/docs/plugins/limit.rst +++ b/docs/plugins/limit.rst @@ -3,7 +3,7 @@ Limit Query Plugin ``limit`` is a plugin to limit a query to the first or last set of results. We also provide a query prefix ``' Date: Sun, 26 Dec 2021 21:58:45 -0500 Subject: [PATCH 079/357] Fixed truediv typo (switched to intdiv). Fixed linter objections. --- test/test_limit.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/test_limit.py b/test/test_limit.py index 1dbe415f4..d5a3d60af 100644 --- a/test/test_limit.py +++ b/test/test_limit.py @@ -15,10 +15,6 @@ import unittest -from test import _common -from beetsplug import limit -from beets import config - from test.helper import TestHelper @@ -29,11 +25,12 @@ class LsLimitPluginTest(unittest.TestCase, TestHelper): self.load_plugins("limit") self.num_test_items = 10 assert self.num_test_items % 2 == 0 - self.num_limit = self.num_test_items / 2 + self.num_limit = self.num_test_items // 2 self.num_limit_prefix = "'<" + str(self.num_limit) + "'" self.track_head_range = "track:.." + str(self.num_limit) self.track_tail_range = "track:" + str(self.num_limit + 1) + ".." - for item_no, item in enumerate(self.add_item_fixtures(count=self.num_test_items)): + for item_no, item in \ + enumerate(self.add_item_fixtures(count=self.num_test_items)): item.track = item_no + 1 item.store() @@ -43,21 +40,23 @@ class LsLimitPluginTest(unittest.TestCase, TestHelper): def test_no_limit(self): result = self.run_with_output("lslimit") self.assertEqual(result.count("\n"), self.num_test_items) - + def test_lslimit_head(self): result = self.run_with_output("lslimit", "--head", str(self.num_limit)) self.assertEqual(result.count("\n"), self.num_limit) - + def test_lslimit_tail(self): result = self.run_with_output("lslimit", "--tail", str(self.num_limit)) self.assertEqual(result.count("\n"), self.num_limit) - + def test_lslimit_head_invariant(self): - result = self.run_with_output("lslimit", "--head", str(self.num_limit), self.track_tail_range) + result = self.run_with_output( + "lslimit", "--head", str(self.num_limit), self.track_tail_range) self.assertEqual(result.count("\n"), self.num_limit) - + def test_lslimit_tail_invariant(self): - result = self.run_with_output("lslimit", "--tail", str(self.num_limit), self.track_head_range) + result = self.run_with_output( + "lslimit", "--tail", str(self.num_limit), self.track_head_range) self.assertEqual(result.count("\n"), self.num_limit) def test_prefix(self): @@ -65,16 +64,19 @@ class LsLimitPluginTest(unittest.TestCase, TestHelper): self.assertEqual(result.count("\n"), self.num_limit) def test_prefix_when_correctly_ordered(self): - result = self.run_with_output("ls", self.track_tail_range, self.num_limit_prefix) + result = self.run_with_output( + "ls", self.track_tail_range, self.num_limit_prefix) self.assertEqual(result.count("\n"), self.num_limit) def test_prefix_when_incorrectly_ordred(self): - result = self.run_with_output("ls", self.num_limit_prefix, self.track_tail_range) + result = self.run_with_output( + "ls", self.num_limit_prefix, self.track_tail_range) self.assertEqual(result.count("\n"), 0) def suite(): return unittest.TestLoader().loadTestsFromName(__name__) + if __name__ == '__main__': unittest.main(defaultTest='suite') From 6c64ab6df157a6f335d0a072d30b167eb4bc5c34 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Mon, 27 Dec 2021 12:11:18 -0500 Subject: [PATCH 080/357] Noticed GitHub linter wanted a docstring. --- beetsplug/limit.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/beetsplug/limit.py b/beetsplug/limit.py index 6dfb2c40d..5e5ee16be 100644 --- a/beetsplug/limit.py +++ b/beetsplug/limit.py @@ -11,6 +11,15 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. +"""Adds head/tail functionality to list/ls. + +1. Implemented as `lslimit` command with `--head` and `--tail` options. This is + the idiomatic way to use this plugin. +2. Implemented as query prefix `<` for head functionality only. This is the + composable way to use the plugin (plays nicely with anything that uses the + query language). +""" + from beets.dbcore import FieldQuery from beets.plugins import BeetsPlugin from beets.ui import Subcommand, decargs, print_ From a3754f7592794f4f7afa8915d7f12f99cafbb5d3 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Mon, 27 Dec 2021 13:26:38 -0500 Subject: [PATCH 081/357] Test failing due to unidentified argument processing issue; replacing with API calls gives expected results. Fixed some linting issues. --- beetsplug/limit.py | 19 ++++++++---------- setup.cfg | 1 + test/test_limit.py | 49 ++++++++++++++++++++++++++++++++++------------ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/beetsplug/limit.py b/beetsplug/limit.py index 5e5ee16be..3942ced0f 100644 --- a/beetsplug/limit.py +++ b/beetsplug/limit.py @@ -28,7 +28,7 @@ from itertools import islice def lslimit(lib, opts, args): - """Query command with head/tail""" + """Query command with head/tail.""" if (opts.head is not None) and (opts.tail is not None): raise ValueError("Only use one of --head and --tail") @@ -56,15 +56,15 @@ lslimit_cmd = Subcommand( ) lslimit_cmd.parser.add_option( - '--head', - action='store', + "--head", + action="store", type="int", default=None ) lslimit_cmd.parser.add_option( - '--tail', - action='store', + "--tail", + action="store", type="int", default=None ) @@ -74,28 +74,25 @@ lslimit_cmd.func = lslimit class LimitPlugin(BeetsPlugin): - """Query limit functionality via command and query prefix - """ + """Query limit functionality via command and query prefix.""" def commands(self): + """Expose `lslimit` subcommand.""" return [lslimit_cmd] def queries(self): class HeadQuery(FieldQuery): - """This inner class pattern allows the query to track state - """ + """This inner class pattern allows the query to track state.""" n = 0 N = None @classmethod def value_match(cls, pattern, value): - if cls.N is None: cls.N = int(pattern) if cls.N < 0: raise ValueError("Limit value must be non-negative") - cls.n += 1 return cls.n <= cls.N diff --git a/setup.cfg b/setup.cfg index 6aab6b7e6..31deba4b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,6 +69,7 @@ per-file-ignores = ./beetsplug/permissions.py:D ./beetsplug/spotify.py:D ./beetsplug/lastgenre/__init__.py:D + ./beetsplug/limit.py:D ./beetsplug/mbcollection.py:D ./beetsplug/metasync/amarok.py:D ./beetsplug/metasync/itunes.py:D diff --git a/test/test_limit.py b/test/test_limit.py index d5a3d60af..35c01c41a 100644 --- a/test/test_limit.py +++ b/test/test_limit.py @@ -18,60 +18,83 @@ import unittest from test.helper import TestHelper -class LsLimitPluginTest(unittest.TestCase, TestHelper): +class LimitPluginTest(unittest.TestCase, TestHelper): + """Unit tests for LimitPlugin + + Note: query prefix tests do not work correctly with `run_with_output`. + """ def setUp(self): + self.setup_beets() self.load_plugins("limit") + + # we'll create an even number of tracks in the library self.num_test_items = 10 assert self.num_test_items % 2 == 0 - self.num_limit = self.num_test_items // 2 - self.num_limit_prefix = "'<" + str(self.num_limit) + "'" - self.track_head_range = "track:.." + str(self.num_limit) - self.track_tail_range = "track:" + str(self.num_limit + 1) + ".." for item_no, item in \ enumerate(self.add_item_fixtures(count=self.num_test_items)): item.track = item_no + 1 item.store() + # our limit tests will use half of this number + self.num_limit = self.num_test_items // 2 + self.num_limit_prefix = "".join(["'", "<", str(self.num_limit), "'"]) + + # a subset of tests has only `num_limit` results, identified by a + # range filter on the track number + self.track_head_range = "track:.." + str(self.num_limit) + self.track_tail_range = "track:" + str(self.num_limit + 1) + ".." + def tearDown(self): + self.unload_plugins() self.teardown_beets() def test_no_limit(self): + """Returns all when there is no limit or filter.""" result = self.run_with_output("lslimit") self.assertEqual(result.count("\n"), self.num_test_items) def test_lslimit_head(self): + """Returns the expected number with `lslimit --head`.""" result = self.run_with_output("lslimit", "--head", str(self.num_limit)) self.assertEqual(result.count("\n"), self.num_limit) def test_lslimit_tail(self): + """Returns the expected number with `lslimit --tail`.""" result = self.run_with_output("lslimit", "--tail", str(self.num_limit)) self.assertEqual(result.count("\n"), self.num_limit) def test_lslimit_head_invariant(self): + """Returns the expected number with `lslimit --head` and a filter.""" result = self.run_with_output( "lslimit", "--head", str(self.num_limit), self.track_tail_range) self.assertEqual(result.count("\n"), self.num_limit) def test_lslimit_tail_invariant(self): + """Returns the expected number with `lslimit --tail` and a filter.""" result = self.run_with_output( "lslimit", "--tail", str(self.num_limit), self.track_head_range) self.assertEqual(result.count("\n"), self.num_limit) def test_prefix(self): - result = self.run_with_output("ls", self.num_limit_prefix) - self.assertEqual(result.count("\n"), self.num_limit) + """Returns the expected number with the query prefix.""" + result = self.lib.items(self.num_limit_prefix) + self.assertEqual(len(result), self.num_limit) def test_prefix_when_correctly_ordered(self): - result = self.run_with_output( - "ls", self.track_tail_range, self.num_limit_prefix) - self.assertEqual(result.count("\n"), self.num_limit) + """Returns the expected number with the query prefix and filter when + the prefix portion (correctly) appears last.""" + correct_order = self.track_tail_range + " " + self.num_limit_prefix + result = self.lib.items(correct_order) + self.assertEqual(len(result), self.num_limit) def test_prefix_when_incorrectly_ordred(self): - result = self.run_with_output( - "ls", self.num_limit_prefix, self.track_tail_range) - self.assertEqual(result.count("\n"), 0) + """Returns no results with the query prefix and filter when the prefix + portion (incorrectly) appears first.""" + incorrect_order = self.num_limit_prefix + " " + self.track_tail_range + result = self.lib.items(incorrect_order) + self.assertEqual(len(result), 0) def suite(): From c3829c52c8f462fb43e861b1294a5e2657774375 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Mon, 27 Dec 2021 13:38:20 -0500 Subject: [PATCH 082/357] Lint ignore. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 31deba4b1..a3d4a866a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -162,6 +162,7 @@ per-file-ignores = ./test/test_library.py:D ./test/test_ui_commands.py:D ./test/test_lyrics.py:D + ./test/test_limit.py:D ./test/test_beatport.py:D ./test/test_random.py:D ./test/test_embyupdate.py:D From 27c2b79f019ec5d2cb41976b68b57dd5bac02003 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Mon, 27 Dec 2021 13:39:35 -0500 Subject: [PATCH 083/357] Finally learning to read GitHub. --- docs/plugins/limit.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/limit.rst b/docs/plugins/limit.rst index 74bd47cb6..35d2b0428 100644 --- a/docs/plugins/limit.rst +++ b/docs/plugins/limit.rst @@ -27,9 +27,9 @@ singleton-based implementation. So why does the query prefix exist? Because it composes with any other query-based API or plugin (see :doc:`/reference/query`). For example, -you can use the query prefix in ``smartplaylists`` (see :doc:`/plugins/ -smartplaylists`) to limit the number of tracks in a smart playlist for -applications like most played and recently added. +you can use the query prefix in ``smartplaylists`` +(see :doc:`/plugins/smartplaylists`) to limit the number of tracks in a smart +playlist for applications like most played and recently added. Configuration ============= From 5b3479705629a50079870e2201d4f4f06e7c5108 Mon Sep 17 00:00:00 2001 From: patrick-nicholson Date: Mon, 27 Dec 2021 14:51:04 -0500 Subject: [PATCH 084/357] sigh --- docs/plugins/limit.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/limit.rst b/docs/plugins/limit.rst index 35d2b0428..ac8cc72c0 100644 --- a/docs/plugins/limit.rst +++ b/docs/plugins/limit.rst @@ -27,8 +27,8 @@ singleton-based implementation. So why does the query prefix exist? Because it composes with any other query-based API or plugin (see :doc:`/reference/query`). For example, -you can use the query prefix in ``smartplaylists`` -(see :doc:`/plugins/smartplaylists`) to limit the number of tracks in a smart +you can use the query prefix in ``smartplaylist`` +(see :doc:`/plugins/smartplaylist`) to limit the number of tracks in a smart playlist for applications like most played and recently added. Configuration From de3eedc033ed07be000bba446cd3c3c5dfcd282f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 27 Dec 2021 13:51:42 -0800 Subject: [PATCH 085/357] Use bytes for destination base name This is mostly "defensive programming": clients *should* only call this on bytestring paths, but just in case this gets called on a Unicode string path, we should now not crash. --- beets/util/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 9f96d4561..8fd196359 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -496,8 +496,9 @@ def move(path, dest, replace=False): os.replace(path, dest) except OSError: # Copy the file to a temporary destination. + base = os.path.basename(bytestring_path(dest)) tmp = tempfile.NamedTemporaryFile(suffix=b'.beets', - prefix=b'.' + os.path.basename(dest), + prefix=b'.' + base, dir=os.path.dirname(dest), delete=False) try: From bb13f37e59ff4dde134f511f1999bb5b59829a6c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 3 Jan 2022 10:16:39 -0800 Subject: [PATCH 086/357] Provide consistent types to NamedTemporaryFile --- beets/util/__init__.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index 8fd196359..dc7edd0ff 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -486,32 +486,33 @@ def move(path, dest, replace=False): (path, dest)) if samefile(path, dest): return - path = syspath(path) - dest = syspath(dest) - if os.path.exists(dest) and not replace: + if os.path.exists(syspath(dest)) and not replace: raise FilesystemError('file exists', 'rename', (path, dest)) # First, try renaming the file. try: - os.replace(path, dest) + os.replace(syspath(path), syspath(dest)) except OSError: # Copy the file to a temporary destination. - base = os.path.basename(bytestring_path(dest)) - tmp = tempfile.NamedTemporaryFile(suffix=b'.beets', - prefix=b'.' + base, - dir=os.path.dirname(dest), - delete=False) + basename = os.path.basename(bytestring_path(dest)) + dirname = os.path.dirname(bytestring_path(dest)) + tmp = tempfile.NamedTemporaryFile( + suffix=syspath(b'.beets', prefix=False), + prefix=syspath(b'.' + basename, prefix=False), + dir=syspath(dirname), + delete=False, + ) try: - with open(path, 'rb') as f: + with open(syspath(path), 'rb') as f: shutil.copyfileobj(f, tmp) finally: tmp.close() # Move the copied file into place. try: - os.replace(tmp.name, dest) + os.replace(tmp.name, syspath(dest)) tmp = None - os.remove(path) + os.remove(syspath(path)) except OSError as exc: raise FilesystemError(exc, 'move', (path, dest), traceback.format_exc()) From 686838856a756cb1f5684adc9a385c3cedbc3f36 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 5 Jan 2022 16:15:39 -0800 Subject: [PATCH 087/357] Two more syspath calls --- beets/util/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index dc7edd0ff..720ca311a 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -479,9 +479,9 @@ def move(path, dest, replace=False): instead, in which case metadata will *not* be preserved. Paths are translated to system paths. """ - if os.path.isdir(path): + if os.path.isdir(syspath(path)): raise FilesystemError(u'source is directory', 'move', (path, dest)) - if os.path.isdir(dest): + if os.path.isdir(syspath(dest)): raise FilesystemError(u'destination is directory', 'move', (path, dest)) if samefile(path, dest): From a09c80447a85c8718e2cfd81139e1bf4b07a263b Mon Sep 17 00:00:00 2001 From: Lars Kruse Date: Fri, 3 Dec 2021 18:20:45 +0100 Subject: [PATCH 088/357] beetsplug/web: fix translation of query path The routing map translator `QueryConverter` was misconfigured: * decoding (parsing a path): splitting with "/" as tokenizer * encoding (translating back to a path): joining items with "," as separator This caused queries containing more than one condition (separated by a slash) to return an empty result. Queries with only a single condition were not affected. Instead the encoding should have used the same delimiter (the slash) for the backward conversion. How to reproduce: * query: `/album/query/albumartist::%5Efoo%24/original_year%2B/year%2B/album%2B` * resulting content in parsed argument `queries` in the `album_query` function: * previous (wrong): `['albumartist::^foo$,original_year+,year+,album+']` * new (correct): `['albumartist::^foo$', 'original_year+', 'year+', 'album+']` --- beetsplug/web/__init__.py | 2 +- docs/changelog.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 240126e95..63f7f92ad 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -261,7 +261,7 @@ class QueryConverter(PathConverter): for query in queries] def to_url(self, value): - return ','.join([v.replace(os.sep, '\\') for v in value]) + return '/'.join([v.replace(os.sep, '\\') for v in value]) class EverythingConverter(PathConverter): diff --git a/docs/changelog.rst b/docs/changelog.rst index 190ed2558..5370ab0f7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,9 @@ Bug fixes: ``r128_album_gain`` fields was changed from integer to float to fix loss of precision due to truncation. :bug:`4169` +* :doc:`plugins/web`: Fix handling of "query" requests. Previously queries + consisting of more than one token (separated by a slash) always returned an + empty result. For packagers: From ec066940976555335770431449b9d022b390004d Mon Sep 17 00:00:00 2001 From: mousecloak Date: Fri, 7 Jan 2022 19:12:05 -0800 Subject: [PATCH 089/357] Makes the import converter respect the quiet and pretend flags. When the delete_originals was set, beets would print the following, regardless of the presence of the quiet parameter: convert: Removing original file /path/to/file.ext This commit ensures that the log is only printed when quiet is not present. --- beetsplug/convert.py | 25 +++++++++++++++---------- docs/changelog.rst | 4 ++++ test/test_convert.py | 27 +++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 6bc07c287..16b6e7075 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -514,17 +514,22 @@ class ConvertPlugin(BeetsPlugin): except subprocess.CalledProcessError: return - # Change the newly-imported database entry to point to the - # converted file. - source_path = item.path - item.path = dest - item.write() - item.read() # Load new audio information data. - item.store() + pretend = self.config['pretend'].get(bool) + quiet = self.config['quiet'].get(bool) - if self.config['delete_originals']: - self._log.info('Removing original file {0}', source_path) - util.remove(source_path, False) + if not pretend: + # Change the newly-imported database entry to point to the + # converted file. + source_path = item.path + item.path = dest + item.write() + item.read() # Load new audio information data. + item.store() + + if self.config['delete_originals']: + if not quiet: + self._log.info('Removing original file {0}', source_path) + util.remove(source_path, False) def _cleanup(self, task, session): for path in task.old_paths: diff --git a/docs/changelog.rst b/docs/changelog.rst index 190ed2558..d0b59245f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,10 @@ Bug fixes: ``r128_album_gain`` fields was changed from integer to float to fix loss of precision due to truncation. :bug:`4169` +* :doc:`/plugins/convert`: Files are no longer converted when running import in + ``--pretend`` mode. +* :doc:`/plugins/convert`: Deleting the original files during conversion no + longer logs output when the ``quiet`` flag is enabled. For packagers: diff --git a/test/test_convert.py b/test/test_convert.py index ce0750119..53c1563b8 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -122,9 +122,28 @@ class ImportConvertTest(unittest.TestCase, TestHelper): self.importer.run() for path in self.importer.paths: for root, dirnames, filenames in os.walk(path): - self.assertTrue(len(fnmatch.filter(filenames, '*.mp3')) == 0, - 'Non-empty import directory {}' - .format(util.displayable_path(path))) + self.assertEqual(len(fnmatch.filter(filenames, '*.mp3')), 0, + 'Non-empty import directory {}' + .format(util.displayable_path(path))) + + def test_delete_originals_keeps_originals_when_pretend_enabled(self): + import_file_count = self.get_count_of_import_files() + + self.config['convert']['delete_originals'] = True + self.config['convert']['pretend'] = True + self.importer.run() + + self.assertEqual(self.get_count_of_import_files(), import_file_count, + 'Count of files differs after running import') + + def get_count_of_import_files(self): + import_file_count = 0 + + for path in self.importer.paths: + for root, _, filenames in os.walk(path): + import_file_count += len(filenames) + + return import_file_count class ConvertCommand: @@ -264,7 +283,7 @@ class NeverConvertLossyFilesTest(unittest.TestCase, TestHelper, self.unload_plugins() self.teardown_beets() - def test_transcode_from_lossles(self): + def test_transcode_from_lossless(self): [item] = self.add_item_fixtures(ext='flac') with control_stdin('y'): self.run_convert_path(item.path) From 0132067a29eb970a4cbd54df1fab6cf439e29b44 Mon Sep 17 00:00:00 2001 From: mousecloak Date: Fri, 7 Jan 2022 19:35:40 -0800 Subject: [PATCH 090/357] Fix @unittest.skipIf annotations to ignore only win32 --- test/test_convert.py | 3 ++- test/test_hook.py | 15 ++++++++++----- test/test_play.py | 3 ++- test/test_query.py | 6 ++++-- test/test_ui.py | 3 ++- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/test/test_convert.py b/test/test_convert.py index 53c1563b8..493d4ecca 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -107,7 +107,8 @@ class ImportConvertTest(unittest.TestCase, TestHelper): item = self.lib.items().get() self.assertFileTag(item.path, 'convert') - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_import_original_on_convert_error(self): # `false` exits with non-zero code self.config['convert']['command'] = 'false' diff --git a/test/test_hook.py b/test/test_hook.py index 6ade06349..5049b5d24 100644 --- a/test/test_hook.py +++ b/test/test_hook.py @@ -63,7 +63,8 @@ class HookTest(_common.TestCase, TestHelper): self.assertIn('hook: invalid command ""', logs) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_hook_non_zero_exit(self): self._add_hook('test_event', 'sh -c "exit 1"') @@ -86,7 +87,8 @@ class HookTest(_common.TestCase, TestHelper): message.startswith("hook: hook for test_event failed: ") for message in logs)) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_hook_no_arguments(self): temporary_paths = [ get_temporary_path() for i in range(self.TEST_HOOK_COUNT) @@ -105,7 +107,8 @@ class HookTest(_common.TestCase, TestHelper): self.assertTrue(os.path.isfile(path)) os.remove(path) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_hook_event_substitution(self): temporary_directory = tempfile._get_default_tempdir() event_names = [f'test_event_event_{i}' for i in @@ -126,7 +129,8 @@ class HookTest(_common.TestCase, TestHelper): self.assertTrue(os.path.isfile(path)) os.remove(path) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_hook_argument_substitution(self): temporary_paths = [ get_temporary_path() for i in range(self.TEST_HOOK_COUNT) @@ -145,7 +149,8 @@ class HookTest(_common.TestCase, TestHelper): self.assertTrue(os.path.isfile(path)) os.remove(path) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_hook_bytes_interpolation(self): temporary_paths = [ get_temporary_path().encode('utf-8') diff --git a/test/test_play.py b/test/test_play.py index 2007686c7..8577aee70 100644 --- a/test/test_play.py +++ b/test/test_play.py @@ -72,7 +72,8 @@ class PlayPluginTest(unittest.TestCase, TestHelper): self.run_and_assert( open_mock, ['title:aNiceTitle'], 'echo other') - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_relative_to(self, open_mock): self.config['play']['command'] = 'echo' self.config['play']['relative_to'] = '/something' diff --git a/test/test_query.py b/test/test_query.py index 709f42bd5..14f3f082a 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -425,7 +425,8 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, []) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_parent_directory_no_slash(self): q = 'path:/a' results = self.lib.items(q) @@ -434,7 +435,8 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, ['path album']) - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_parent_directory_with_slash(self): q = 'path:/a/' results = self.lib.items(q) diff --git a/test/test_ui.py b/test/test_ui.py index 9804b0a12..dd24fce1a 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -918,7 +918,8 @@ class ConfigTest(unittest.TestCase, TestHelper, _common.Assertions): # '--config', cli_overwrite_config_path, 'test') # self.assertEqual(config['anoption'].get(), 'cli overwrite') - @unittest.skipIf(sys.platform, 'win32') # FIXME: fails on windows + # FIXME: fails on windows + @unittest.skipIf(sys.platform == 'win32', 'win32') def test_cli_config_paths_resolve_relative_to_user_dir(self): cli_config_path = os.path.join(self.temp_dir, b'config.yaml') with open(cli_config_path, 'w') as file: From 438262844a1697a38a33db060489e272a639090a Mon Sep 17 00:00:00 2001 From: mousecloak Date: Fri, 7 Jan 2022 21:39:19 -0800 Subject: [PATCH 091/357] Fixed style violation --- beetsplug/convert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 16b6e7075..82e62af62 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -528,7 +528,8 @@ class ConvertPlugin(BeetsPlugin): if self.config['delete_originals']: if not quiet: - self._log.info('Removing original file {0}', source_path) + self._log.info('Removing original file {0}', + source_path) util.remove(source_path, False) def _cleanup(self, task, session): From e35c767e2c935b392e7ca70419fe2c1861a7fa76 Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Sun, 9 Jan 2022 14:29:47 +0100 Subject: [PATCH 092/357] Skip Discogs query on insufficiently tagged files - When files are missing both, album and artist tags, the Discogs metadata plugin sends empty information to the Discogs API which returns arbitrary query results. - This patch catches this case and states it in beets import verbose output. --- beetsplug/discogs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index d015e4201..8c950c521 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -157,6 +157,11 @@ class DiscogsPlugin(BeetsPlugin): if not self.discogs_client: return + if not album and not artist: + self._log.debug('Skipping Discogs query. Files missing album and ' + 'artist tags.') + return [] + if va_likely: query = album else: From 4401de94f7b70d55867b1cfed24ebbfa330e6fa9 Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Mon, 10 Jan 2022 08:20:46 +0100 Subject: [PATCH 093/357] Add changelog entry for PR #4227 (discogs: Skip Discogs query on insufficiently tagged files). --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d13dcdd4a..88d465d7d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,7 +22,7 @@ Bug fixes: * :doc:`/plugins/unimported`: The new ``ignore_subdirectories`` configuration option added in 1.6.0 now has a default value if it hasn't been set. * :doc:`/plugins/deezer`: Tolerate missing fields when searching for singleton - tracks + tracks. :bug:`4116` * :doc:`/plugins/replaygain`: The type of the internal ``r128_track_gain`` and ``r128_album_gain`` fields was changed from integer to float to fix loss of @@ -35,6 +35,9 @@ Bug fixes: * :doc:`plugins/web`: Fix handling of "query" requests. Previously queries consisting of more than one token (separated by a slash) always returned an empty result. +* :doc:`/plugins/discogs`: Skip Discogs query on insufficiently tagged files + (artist and album tags missing) to prevent arbitrary candidate results. + :bug:`4227` For packagers: From 2a53b890be70b6cf537e045c3a6ccbbb96b67f31 Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Mon, 10 Jan 2022 09:10:19 +0100 Subject: [PATCH 094/357] Add to discogs plugin docs regarding PR #4227 - Clarify basic search behaviour in intro chapter of discogs plugin, - and state change introduced in PR#4227 (discogs: Discogs query on insufficiently tagged files) --- docs/plugins/discogs.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 40875b022..5aea1ae6b 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -19,7 +19,8 @@ authentication credentials via a personal access token or an OAuth2 authorization. Matches from Discogs will now show up during import alongside matches from -MusicBrainz. +MusicBrainz. The search terms sent to the Discogs API are based on the artist +and album tags of your tracks. If those are empty no query will be issued. If you have a Discogs ID for an album you want to tag, you can also enter it at the "enter Id" prompt in the importer. From 3f896ab28117cd9032a0d9fcc8fc5c871f954324 Mon Sep 17 00:00:00 2001 From: ybnd Date: Mon, 10 Jan 2022 19:03:36 +0100 Subject: [PATCH 095/357] Make Tekstowo scraper more specific --- beetsplug/lyrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 7d026def1..0856ebb34 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -488,11 +488,11 @@ class Tekstowo(Backend): if not soup: return None - lyrics_div = soup.find("div", class_="song-text") + lyrics_div = soup.select("div.song-text > div.inner-text") if not lyrics_div: return None - return lyrics_div.get_text() + return lyrics_div[0].get_text() def remove_credits(text): From 3a8520e30ab9ec6c435cacfa1d1508421ca2ecbe Mon Sep 17 00:00:00 2001 From: ybnd Date: Mon, 10 Jan 2022 19:07:59 +0100 Subject: [PATCH 096/357] Add changelog entry --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d13dcdd4a..e4a10dc9c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,8 @@ Bug fixes: * :doc:`plugins/web`: Fix handling of "query" requests. Previously queries consisting of more than one token (separated by a slash) always returned an empty result. +* :doc:`plugins/lyrics`: Fixed an issue with the Tekstowo.pl scraper where some + non-lyrics content got included in the lyrics For packagers: From 414760282b9ec374a26488a89da230f545f35b52 Mon Sep 17 00:00:00 2001 From: ybnd Date: Mon, 10 Jan 2022 22:07:58 +0100 Subject: [PATCH 097/357] Remove footer text from Genius lyrics --- beetsplug/lyrics.py | 6 ++++++ docs/changelog.rst | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0856ebb34..1f215df45 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -419,11 +419,17 @@ class Genius(Backend): lyrics_div = verse_div.parent for br in lyrics_div.find_all("br"): br.replace_with("\n") + ads = lyrics_div.find_all("div", class_=re.compile("InreadAd__Container")) for ad in ads: ad.replace_with("\n") + footers = lyrics_div.find_all("div", + class_=re.compile("Lyrics__Footer")) + for footer in footers: + footer.replace_with("") + return lyrics_div.get_text() diff --git a/docs/changelog.rst b/docs/changelog.rst index e4a10dc9c..715853b66 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,8 +35,8 @@ Bug fixes: * :doc:`plugins/web`: Fix handling of "query" requests. Previously queries consisting of more than one token (separated by a slash) always returned an empty result. -* :doc:`plugins/lyrics`: Fixed an issue with the Tekstowo.pl scraper where some - non-lyrics content got included in the lyrics +* :doc:`plugins/lyrics`: Fixed issues with the Tekstowo.pl and Genius + backends where some non-lyrics content got included in the lyrics For packagers: From 919e8be4e351a2e53065e4b96e5d1747a6cb8aae Mon Sep 17 00:00:00 2001 From: Alex <37914724+jmizv@users.noreply.github.com> Date: Fri, 14 Jan 2022 23:13:29 +0100 Subject: [PATCH 098/357] use correct headings --- docs/plugins/limit.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/limit.rst b/docs/plugins/limit.rst index ac8cc72c0..8c4330aa8 100644 --- a/docs/plugins/limit.rst +++ b/docs/plugins/limit.rst @@ -32,13 +32,13 @@ you can use the query prefix in ``smartplaylist`` playlist for applications like most played and recently added. Configuration -============= +------------- Enable the ``limit`` plugin in your configuration (see :ref:`using-plugins`). Examples -======== +-------- First 10 tracks From a09829f47f61112090d5073ce168b53b807ac252 Mon Sep 17 00:00:00 2001 From: Alex <37914724+jmizv@users.noreply.github.com> Date: Fri, 14 Jan 2022 23:22:27 +0100 Subject: [PATCH 099/357] Update changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 71ef9973b..f4df82e5c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -43,6 +43,7 @@ Bug fixes: :bug:`4227` * :doc:`plugins/lyrics`: Fixed issues with the Tekstowo.pl and Genius backends where some non-lyrics content got included in the lyrics +* :doc:`plugins/limit`: Better header formatting to improve index For packagers: From c1df4fc8c253099a33360ce6001a0b4bc12e66a0 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Wed, 19 Jan 2022 22:53:08 +0100 Subject: [PATCH 100/357] CONTRIBUTING.rst: minor Python 3 adaptation --- CONTRIBUTING.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 32a9d2552..18ca9b9e4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -203,11 +203,10 @@ There are a few coding conventions we use in beets: instead. In particular, we have our own logging shim, so you’ll see ``from beets import logging`` in most files. - - Always log Unicode strings (e.g., ``log.debug(u"hello world")``). - The loggers use `str.format `__-style logging instead of ``%``-style, so you can type - ``log.debug(u"{0}", obj)`` to do your formatting. + ``log.debug("{0}", obj)`` to do your formatting. - Exception handlers must use ``except A as B:`` instead of ``except A, B:``. From 807f124ef81a5f42cde75e7ff68cb1ad205eb5ee Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 20 Jan 2022 00:23:39 +0100 Subject: [PATCH 101/357] really remove all six imports apparently, pyupgrade didn't know how to handle these... --- beetsplug/replaygain.py | 2 +- test/helper.py | 2 +- test/test_importer.py | 2 +- test/test_logging.py | 2 +- test/test_spotify.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index b6297d937..383bb3da3 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -22,7 +22,7 @@ import subprocess import sys import warnings from multiprocessing.pool import ThreadPool, RUN -from six.moves import queue +import queue from threading import Thread, Event from beets import ui diff --git a/test/helper.py b/test/helper.py index ba71ddc24..988995f48 100644 --- a/test/helper.py +++ b/test/helper.py @@ -37,7 +37,7 @@ import shutil import subprocess from tempfile import mkdtemp, mkstemp from contextlib import contextmanager -from six import StringIO +from io import StringIO from enum import Enum import beets diff --git a/test/test_importer.py b/test/test_importer.py index 306491af2..60c0b793f 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -21,7 +21,7 @@ import shutil import unicodedata import sys import stat -from six import StringIO +from io import StringIO from tempfile import mkstemp from zipfile import ZipFile from tarfile import TarFile diff --git a/test/test_logging.py b/test/test_logging.py index 76a73e931..8a9fd8742 100644 --- a/test/test_logging.py +++ b/test/test_logging.py @@ -3,7 +3,7 @@ import sys import threading import logging as log -from six import StringIO +from io import StringIO import unittest import beets.logging as blog diff --git a/test/test_spotify.py b/test/test_spotify.py index f90ecd907..76148862d 100644 --- a/test/test_spotify.py +++ b/test/test_spotify.py @@ -10,7 +10,7 @@ from beets import config from beets.library import Item from beetsplug import spotify from test.helper import TestHelper -from six.moves.urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, urlparse class ArgumentsMock: From cf69cad56f6066418a1fcf6b10a769168a98e280 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 20 Jan 2022 14:24:08 -0500 Subject: [PATCH 102/357] Pin Sphinx <4.4.0 in CI A recent change in Sphinx introduced a new warning about missed extlink opportunities: https://github.com/sphinx-doc/sphinx/issues/10112 These warnings are causing spurious CI failures. Some of these suggestions are good but many of them are not, and there is not currently a way to disable the warning (globally or locally). So the only workable solution currently seems to be to pin an old version of Sphinx in CI for now. Hopefully there will be an option to disable this in 4.4.1, at which point we can unpin. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 137f74b72..d4ec20061 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,7 +86,7 @@ jobs: - name: Install base dependencies run: | python -m pip install --upgrade pip - python -m pip install tox sphinx + python -m pip install tox 'sphinx<4.4.0' - name: Add problem matcher run: echo "::add-matcher::.github/sphinx-problem-matcher.json" From 1bcec4e5f0cd18c82733ceec2c16b6458308d47c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 20 Jan 2022 14:39:04 -0500 Subject: [PATCH 103/357] Move Sphinx pin to tox.ini Turns out the Sphinx installed "outside" of the tox virtualenv is not the one that actually builds the docs... --- .github/workflows/ci.yaml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d4ec20061..137f74b72 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,7 +86,7 @@ jobs: - name: Install base dependencies run: | python -m pip install --upgrade pip - python -m pip install tox 'sphinx<4.4.0' + python -m pip install tox sphinx - name: Add problem matcher run: echo "::add-matcher::.github/sphinx-problem-matcher.json" diff --git a/tox.ini b/tox.ini index 5a5b78b31..5f9de07f6 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ commands = [testenv:docs] basepython = python3.9 -deps = sphinx +deps = sphinx<4.4.0 commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} # checks all links in the docs From d26b0bc19bf2666810c4ad0b14c03edf60c6ee3b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 20 Jan 2022 21:12:59 +0100 Subject: [PATCH 104/357] test_replaygain: fix complicated and incorrect exception handling This is an incorrect translation of a python 2 reraise to python 3. With python 3, however, we can just rely on exception chaining to get the traceback, so get rid of the complicated re-raising entirely, with the additional benefit that the exception from the tear-down is also shown. --- test/test_replaygain.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index b39a4e990..58b487fad 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -55,18 +55,8 @@ class ReplayGainCliTestBase(TestHelper): try: self.load_plugins('replaygain') except Exception: - import sys - # store exception info so an error in teardown does not swallow it - exc_info = sys.exc_info() - try: - self.teardown_beets() - self.unload_plugins() - except Exception: - # if load_plugins() failed then setup is incomplete and - # teardown operations may fail. In particular # {Item,Album} - # may not have the _original_types attribute in unload_plugins - pass - raise None.with_traceback(exc_info[2]) + self.teardown_beets() + self.unload_plugins() album = self.add_album_fixture(2) for item in album.items(): From c272696d9f6bc8426ebb961906d0deea5c841d9e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 20 Jan 2022 21:16:17 +0100 Subject: [PATCH 105/357] test_logging: fix incorrect exception handling Another incorrect py2 -> py3 translation. Since python 3 attached the traceback to the exception, this should preserve the traceback without needing to resort to sys.exc_info --- test/test_logging.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/test/test_logging.py b/test/test_logging.py index 8a9fd8742..79ff5cae2 100644 --- a/test/test_logging.py +++ b/test/test_logging.py @@ -175,7 +175,7 @@ class ConcurrentEventsTest(TestCase, helper.TestHelper): self.lock1 = threading.Lock() self.lock2 = threading.Lock() self.test_case = test_case - self.exc_info = None + self.exc = None self.t1_step = self.t2_step = 0 def log_all(self, name): @@ -190,9 +190,8 @@ class ConcurrentEventsTest(TestCase, helper.TestHelper): self.lock1.acquire() self.test_case.assertEqual(self._log.level, log.INFO) self.t1_step = 2 - except Exception: - import sys - self.exc_info = sys.exc_info() + except Exception as e: + self.exc = e def listener2(self): try: @@ -201,9 +200,8 @@ class ConcurrentEventsTest(TestCase, helper.TestHelper): self.lock2.acquire() self.test_case.assertEqual(self._log.level, log.DEBUG) self.t2_step = 2 - except Exception: - import sys - self.exc_info = sys.exc_info() + except Exception as e: + self.exc = e def setUp(self): self.setup_beets(disk=True) @@ -215,8 +213,8 @@ class ConcurrentEventsTest(TestCase, helper.TestHelper): dp = self.DummyPlugin(self) def check_dp_exc(): - if dp.exc_info: - raise None.with_traceback(dp.exc_info[2]) + if dp.exc: + raise dp.exc try: dp.lock1.acquire() From 356a775c5e0f8fad2dbb926fe4d7255f5528ac04 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 14:58:14 +0100 Subject: [PATCH 106/357] replaygain: rewrite long conditionals in a more imperative manner (1/2) This is significantly easier to parse (for me, at least). Also, void building some lists inside of any(...) in the process. --- beetsplug/replaygain.py | 53 ++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 383bb3da3..ca5f08ad8 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1037,24 +1037,55 @@ class ReplayGainPlugin(BeetsPlugin): """ return item.format in self.r128_whitelist + @staticmethod + def has_r128_track_data(item): + return item.r128_track_gain is not None + + @staticmethod + def has_rg_track_data(item): + return (item.rg_track_gain is not None + and item.rg_track_peak is not None) + def track_requires_gain(self, item): - return self.overwrite or \ - (self.should_use_r128(item) and not item.r128_track_gain) or \ - (not self.should_use_r128(item) and - (not item.rg_track_gain or not item.rg_track_peak)) + if self.overwrite: + return True + + if self.should_use_r128(item): + if not self.has_r128_track_data(item): + return True + else: + if not self.has_rg_track_data(item): + return True + + return False + + @staticmethod + def has_r128_album_data(item): + return (item.r128_track_gain is not None + and item.r128_album_gain is not None) + + @staticmethod + def has_rg_album_data(item): + return (item.rg_album_gain is not None + and item.rg_album_peak is not None) def album_requires_gain(self, album): # Skip calculating gain only when *all* files don't need # recalculation. This way, if any file among an album's tracks # needs recalculation, we still get an accurate album gain # value. - return self.overwrite or \ - any([self.should_use_r128(item) and - (not item.r128_track_gain or not item.r128_album_gain) - for item in album.items()]) or \ - any([not self.should_use_r128(item) and - (not item.rg_album_gain or not item.rg_album_peak) - for item in album.items()]) + if self.overwrite: + return True + + for item in album.items(): + if self.should_use_r128(item): + if not self.has_r128_album_data(item): + return True + else: + if not self.has_rg_album_data(item): + return True + + return False def store_track_gain(self, item, track_gain): item.rg_track_gain = track_gain.gain From 6689502854056bb80125e629a0ef67d91a775c9b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 14:58:40 +0100 Subject: [PATCH 107/357] replaygain: rewrite long conditionals in a more imperative manner (2/2) This is significantly easier to parse (for me, at least). Also, void building some lists inside of any(...) in the process. --- beetsplug/replaygain.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index ca5f08ad8..c1266f46f 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1145,8 +1145,9 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info('Skipping album {0}', album) return - if (any([self.should_use_r128(item) for item in album.items()]) and not - all([self.should_use_r128(item) for item in album.items()])): + items_iter = iter(album.items()) + use_r128 = self.should_use_r128(next(items_iter)) + if any(use_r128 != self.should_use_r128(i) for i in items_iter): self._log.error( "Cannot calculate gain for album {0} (incompatible formats)", album) From 3eb49fca2e5d4934881ea3cea80473b362c02ad9 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 15:09:05 +0100 Subject: [PATCH 108/357] replaygain: clarify docs for overwrite/force and actually respect it correctly The code used to always check the 'overwrite' config value, while that should only apply during imports. The manual 'replaygain' command has it's own '-f' flag. The logic for this flag was changed quite a few times recently, see https://github.com/beetbox/beets/pull/3816 https://github.com/beetbox/beets/issues/3872 https://github.com/beetbox/beets/pull/3890 but apparently we (me and @ybnd) never really got it right... If it is some comfort, the logic was never correct in the first place. --- beetsplug/replaygain.py | 14 +++++--------- docs/plugins/replaygain.rst | 5 ++++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index c1266f46f..eb322dadf 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -989,7 +989,9 @@ class ReplayGainPlugin(BeetsPlugin): 'r128_targetlevel': lufs_to_db(-23), }) - self.overwrite = self.config['overwrite'].get(bool) + # FIXME: Consider renaming the configuration option and deprecating the + # old name 'overwrite'. + self.force_on_import = self.config['overwrite'].get(bool) self.per_disc = self.config['per_disc'].get(bool) # Remember which backend is used for CLI feedback @@ -1047,9 +1049,6 @@ class ReplayGainPlugin(BeetsPlugin): and item.rg_track_peak is not None) def track_requires_gain(self, item): - if self.overwrite: - return True - if self.should_use_r128(item): if not self.has_r128_track_data(item): return True @@ -1074,9 +1073,6 @@ class ReplayGainPlugin(BeetsPlugin): # recalculation. This way, if any file among an album's tracks # needs recalculation, we still get an accurate album gain # value. - if self.overwrite: - return True - for item in album.items(): if self.should_use_r128(item): if not self.has_r128_album_data(item): @@ -1340,9 +1336,9 @@ class ReplayGainPlugin(BeetsPlugin): """ if self.config['auto']: if task.is_album: - self.handle_album(task.album, False) + self.handle_album(task.album, False, self.force_on_import) else: - self.handle_track(task.item, False) + self.handle_track(task.item, False, self.force_on_import) def command_func(self, lib, opts, args): try: diff --git a/docs/plugins/replaygain.rst b/docs/plugins/replaygain.rst index fa0e10b75..4ba882686 100644 --- a/docs/plugins/replaygain.rst +++ b/docs/plugins/replaygain.rst @@ -112,7 +112,10 @@ configuration file. The available options are: - **backend**: The analysis backend; either ``gstreamer``, ``command``, ``audiotools`` or ``ffmpeg``. Default: ``command``. -- **overwrite**: Re-analyze files that already have ReplayGain tags. +- **overwrite**: On import, re-analyze files that already have ReplayGain tags. + Note that, for historical reasons, the name of this option is somewhat + unfortunate: It does not decide whether tags are written to the files (which + is controlled by the :ref:`import.write ` option). Default: ``no``. - **targetlevel**: A number of decibels for the target loudness level for files using ``REPLAYGAIN_`` tags. From 96025c6de683505eec21998e624326365c23fe66 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 15:13:04 +0100 Subject: [PATCH 109/357] replaygain: avoid determining the method again and again --- beetsplug/replaygain.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index eb322dadf..b5faf44d6 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1110,13 +1110,13 @@ class ReplayGainPlugin(BeetsPlugin): self._log.debug('applied r128 album gain {0} LU', item.r128_album_gain) - def tag_specific_values(self, items): + def tag_specific_values(self, use_r128): """Return some tag specific values. Returns a tuple (store_track_gain, store_album_gain, target_level, peak_method). """ - if any([self.should_use_r128(item) for item in items]): + if use_r128: store_track_gain = self.store_track_r128_gain store_album_gain = self.store_album_r128_gain target_level = self.config['r128_targetlevel'].as_number() @@ -1151,7 +1151,7 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info('analyzing {0}', album) - tag_vals = self.tag_specific_values(album.items()) + tag_vals = self.tag_specific_values(use_r128) store_track_gain, store_album_gain, target_level, peak = tag_vals discs = {} @@ -1210,7 +1210,8 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info('Skipping track {0}', item) return - tag_vals = self.tag_specific_values([item]) + use_r128 = self.should_use_r128(item) + tag_vals = self.tag_specific_values(use_r128) store_track_gain, store_album_gain, target_level, peak = tag_vals def _store_track(track_gains): From edf2bda1ceeb333d4b4824ca60acdd15cde84f8e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 16:59:42 +0100 Subject: [PATCH 110/357] replaygain: store backend name as class attribute Doesn't change any functionality, but appears a little cleaner to me. --- beetsplug/replaygain.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index b5faf44d6..0ac02039e 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -109,6 +109,7 @@ class Backend: """An abstract class representing engine for calculating RG values. """ + NAME = "" do_parallel = False def __init__(self, config, log): @@ -135,6 +136,7 @@ class FfmpegBackend(Backend): """A replaygain backend using ffmpeg's ebur128 filter. """ + NAME = "ffmpeg" do_parallel = True def __init__(self, config, log): @@ -379,6 +381,7 @@ class FfmpegBackend(Backend): # mpgain/aacgain CLI tool backend. class CommandBackend(Backend): + NAME = "command" do_parallel = True def __init__(self, config, log): @@ -508,6 +511,8 @@ class CommandBackend(Backend): # GStreamer-based backend. class GStreamerBackend(Backend): + NAME = "gstreamer" + def __init__(self, config, log): super().__init__(config, log) self._import_gst() @@ -779,6 +784,7 @@ class AudioToolsBackend(Backend): `_ and its capabilities to read more file formats and compute ReplayGain values using it replaygain module. """ + NAME = "audiotools" def __init__(self, config, log): super().__init__(config, log) @@ -956,17 +962,19 @@ class ExceptionWatcher(Thread): # Main plugin logic. +BACKEND_CLASSES = [ + CommandBackend, + GStreamerBackend, + AudioToolsBackend, + FfmpegBackend, +] +BACKENDS = {b.NAME: b for b in BACKEND_CLASSES} + + class ReplayGainPlugin(BeetsPlugin): """Provides ReplayGain analysis. """ - backends = { - "command": CommandBackend, - "gstreamer": GStreamerBackend, - "audiotools": AudioToolsBackend, - "ffmpeg": FfmpegBackend, - } - peak_methods = { "true": Peak.true, "sample": Peak.sample, @@ -997,12 +1005,12 @@ class ReplayGainPlugin(BeetsPlugin): # Remember which backend is used for CLI feedback self.backend_name = self.config['backend'].as_str() - if self.backend_name not in self.backends: + if self.backend_name not in BACKENDS: raise ui.UserError( "Selected ReplayGain backend {} is not supported. " "Please select one of: {}".format( self.backend_name, - ', '.join(self.backends.keys()) + ', '.join(BACKENDS.keys()) ) ) peak_method = self.config["peak"].as_str() @@ -1026,7 +1034,7 @@ class ReplayGainPlugin(BeetsPlugin): self.r128_whitelist = self.config['r128'].as_str_seq() try: - self.backend_instance = self.backends[self.backend_name]( + self.backend_instance = BACKENDS[self.backend_name]( self.config, self._log ) except (ReplayGainError, FatalReplayGainError) as e: From ae3e95f9d3a3095e0ec2eea1d8ca35db14af2a3c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 15:40:39 +0100 Subject: [PATCH 111/357] replaygain: Convert the ad-hoc tag_specific_values to classes The plugin has loads of indirection and nested functions which make it really hard to reason about. The larger picture here is that I'd like to make the code more manageable before reworking the parallelism issues. In particular, instead of manually implementing an interface using a function that returns a tuple of function pointers, this commit creates proper classes. Again, no functionality is changed, this only moves code around. --- beetsplug/replaygain.py | 149 +++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 69 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 0ac02039e..0f9713ac3 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -99,10 +99,61 @@ Gain = collections.namedtuple("Gain", "gain peak") AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") -class Peak(enum.Enum): - none = 0 - true = 1 - sample = 2 +ALL_PEAK_METHODS = ["true", "sample"] +Peak = enum.Enum("Peak", ["none"] + ALL_PEAK_METHODS) + + +class GainHandler(): + def __init__(self, config, peak, log): + self.config = config + self.peak = peak + self._log = log + + @property + def target_level(self): + """This currently needs to reloaded from the config since the tests + modify its value on-the-fly. + """ + return self.config['targetlevel'].as_number() + + def store_track_gain(self, item, track_gain): + item.rg_track_gain = track_gain.gain + item.rg_track_peak = track_gain.peak + item.store() + self._log.debug('applied track gain {0} LU, peak {1} of FS', + item.rg_track_gain, item.rg_track_peak) + + def store_album_gain(self, item, album_gain): + item.rg_album_gain = album_gain.gain + item.rg_album_peak = album_gain.peak + item.store() + self._log.debug('applied album gain {0} LU, peak {1} of FS', + item.rg_album_gain, item.rg_album_peak) + + +class R128GainHandler(GainHandler): + def __init__(self, config, log): + # R128_* tags do not store the track/album peak + super().__init__(config, Peak.none, log) + + @property + def target_level(self): + """This currently needs to reloaded from the config since the tests + modify its value on-the-fly. + """ + return self.config['r128_targetlevel'].as_number() + + def store_track_gain(self, item, track_gain): + item.r128_track_gain = track_gain.gain + item.store() + self._log.debug('applied r128 track gain {0} LU', + item.r128_track_gain) + + def store_album_gain(self, item, album_gain): + item.r128_album_gain = album_gain.gain + item.store() + self._log.debug('applied r128 album gain {0} LU', + item.r128_album_gain) class Backend: @@ -975,11 +1026,6 @@ class ReplayGainPlugin(BeetsPlugin): """Provides ReplayGain analysis. """ - peak_methods = { - "true": Peak.true, - "sample": Peak.sample, - } - def __init__(self): super().__init__() @@ -1013,16 +1059,29 @@ class ReplayGainPlugin(BeetsPlugin): ', '.join(BACKENDS.keys()) ) ) + peak_method = self.config["peak"].as_str() - if peak_method not in self.peak_methods: + if peak_method not in ALL_PEAK_METHODS: raise ui.UserError( "Selected ReplayGain peak method {} is not supported. " "Please select one of: {}".format( peak_method, - ', '.join(self.peak_methods.keys()) + ', '.join(ALL_PEAK_METHODS) ) ) - self._peak_method = self.peak_methods[peak_method] + + # The key in this dict is the `use_r128` flag. + self.gain_handlers = { + True: R128GainHandler( + self.config, + self._log, + ), + False: GainHandler( + self.config, + Peak[peak_method], + self._log, + ), + } # On-import analysis. if self.config['auto']: @@ -1091,52 +1150,6 @@ class ReplayGainPlugin(BeetsPlugin): return False - def store_track_gain(self, item, track_gain): - item.rg_track_gain = track_gain.gain - item.rg_track_peak = track_gain.peak - item.store() - self._log.debug('applied track gain {0} LU, peak {1} of FS', - item.rg_track_gain, item.rg_track_peak) - - def store_album_gain(self, item, album_gain): - item.rg_album_gain = album_gain.gain - item.rg_album_peak = album_gain.peak - item.store() - self._log.debug('applied album gain {0} LU, peak {1} of FS', - item.rg_album_gain, item.rg_album_peak) - - def store_track_r128_gain(self, item, track_gain): - item.r128_track_gain = track_gain.gain - item.store() - - self._log.debug('applied r128 track gain {0} LU', - item.r128_track_gain) - - def store_album_r128_gain(self, item, album_gain): - item.r128_album_gain = album_gain.gain - item.store() - self._log.debug('applied r128 album gain {0} LU', - item.r128_album_gain) - - def tag_specific_values(self, use_r128): - """Return some tag specific values. - - Returns a tuple (store_track_gain, store_album_gain, target_level, - peak_method). - """ - if use_r128: - store_track_gain = self.store_track_r128_gain - store_album_gain = self.store_album_r128_gain - target_level = self.config['r128_targetlevel'].as_number() - peak = Peak.none # R128_* tags do not store the track/album peak - else: - store_track_gain = self.store_track_gain - store_album_gain = self.store_album_gain - target_level = self.config['targetlevel'].as_number() - peak = self._peak_method - - return store_track_gain, store_album_gain, target_level, peak - def handle_album(self, album, write, force=False): """Compute album and track replay gain store it in all of the album's items. @@ -1159,8 +1172,7 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info('analyzing {0}', album) - tag_vals = self.tag_specific_values(use_r128) - store_track_gain, store_album_gain, target_level, peak = tag_vals + handler = self.gain_handlers[use_r128] discs = {} if self.per_disc: @@ -1185,8 +1197,8 @@ class ReplayGainPlugin(BeetsPlugin): ) for item, track_gain in zip(items, album_gain.track_gains): - store_track_gain(item, track_gain) - store_album_gain(item, album_gain.album_gain) + handler.store_track_gain(item, track_gain) + handler.store_album_gain(item, album_gain.album_gain) if write: item.try_write() self._log.debug('done analyzing {0}', item) @@ -1196,8 +1208,8 @@ class ReplayGainPlugin(BeetsPlugin): self.backend_instance.compute_album_gain, args=(), kwds={ "items": list(items), - "target_level": target_level, - "peak": peak + "target_level": handler.target_level, + "peak": handler.peak, }, callback=_store_album ) @@ -1219,8 +1231,7 @@ class ReplayGainPlugin(BeetsPlugin): return use_r128 = self.should_use_r128(item) - tag_vals = self.tag_specific_values(use_r128) - store_track_gain, store_album_gain, target_level, peak = tag_vals + handler = self.gain_handlers[use_r128] def _store_track(track_gains): if not track_gains or len(track_gains) != 1: @@ -1232,7 +1243,7 @@ class ReplayGainPlugin(BeetsPlugin): .format(self.backend_name, item) ) - store_track_gain(item, track_gains[0]) + handler.store_track_gain(item, track_gains[0]) if write: item.try_write() self._log.debug('done analyzing {0}', item) @@ -1242,8 +1253,8 @@ class ReplayGainPlugin(BeetsPlugin): self.backend_instance.compute_track_gain, args=(), kwds={ "items": [item], - "target_level": target_level, - "peak": peak, + "target_level": handler.target_level, + "peak": handler.peak, }, callback=_store_track ) From 67d85d18ad5e5e4c219024b4e5038a7fd0687d2f Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Fri, 19 Mar 2021 11:18:01 +0100 Subject: [PATCH 112/357] replaygain: introduce Task objects to bundle the state related to computations Renames *GainHandler -> *Task and instead of having a singleton instance, creates a *Task object for each album/item to process. The advantage is that now, related data can be bundled in the instance, instead of passing multiple arguments around. --- beetsplug/replaygain.py | 235 ++++++++++++++++++++-------------------- 1 file changed, 118 insertions(+), 117 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 0f9713ac3..3551ddeb2 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -103,18 +103,15 @@ ALL_PEAK_METHODS = ["true", "sample"] Peak = enum.Enum("Peak", ["none"] + ALL_PEAK_METHODS) -class GainHandler(): - def __init__(self, config, peak, log): - self.config = config +class RgTask(): + def __init__(self, items, album, target_level, peak, log): + self.items = items + self.album = album + self.target_level = target_level self.peak = peak self._log = log - - @property - def target_level(self): - """This currently needs to reloaded from the config since the tests - modify its value on-the-fly. - """ - return self.config['targetlevel'].as_number() + self.album_gain = None + self.track_gains = None def store_track_gain(self, item, track_gain): item.rg_track_gain = track_gain.gain @@ -131,18 +128,7 @@ class GainHandler(): item.rg_album_gain, item.rg_album_peak) -class R128GainHandler(GainHandler): - def __init__(self, config, log): - # R128_* tags do not store the track/album peak - super().__init__(config, Peak.none, log) - - @property - def target_level(self): - """This currently needs to reloaded from the config since the tests - modify its value on-the-fly. - """ - return self.config['r128_targetlevel'].as_number() - +class R128Task(RgTask): def store_track_gain(self, item, track_gain): item.r128_track_gain = track_gain.gain item.store() @@ -169,15 +155,15 @@ class Backend: """ self._log = log - def compute_track_gain(self, items, target_level, peak): - """Computes the track gain of the given tracks, returns a list - of Gain objects. + def compute_track_gain(self, task): + """Computes the track gain for the tracks belonging to `task`, and sets + the `track_gains` attribute on the task. Returns `task`. """ raise NotImplementedError() - def compute_album_gain(self, items, target_level, peak): - """Computes the album gain of the given album, returns an - AlbumGain object. + def compute_album_gain(self, task): + """Computes the album gain for the album belonging to `task`, and sets + the `album_gain` attribute on the task. Returns `task`. """ raise NotImplementedError() @@ -218,27 +204,28 @@ class FfmpegBackend(Backend): "the --enable-libebur128 configuration option is required." ) - def compute_track_gain(self, items, target_level, peak): - """Computes the track gain of the given tracks, returns a list - of Gain objects (the track gains). + def compute_track_gain(self, task): + """Computes the track gain for the tracks belonging to `task`, and sets + the `track_gains` attribute on the task. Returns `task`. """ gains = [] - for item in items: + for item in task.items: gains.append( self._analyse_item( item, - target_level, - peak, + task.target_level, + task.peak, count_blocks=False, )[0] # take only the gain, discarding number of gating blocks ) - return gains + task.track_gains = gains + return task - def compute_album_gain(self, items, target_level, peak): - """Computes the album gain of the given album, returns an - AlbumGain object. + def compute_album_gain(self, task): + """Computes the album gain for the album belonging to `task`, and sets + the `album_gain` attribute on the task. Returns `task`. """ - target_level_lufs = db_to_lufs(target_level) + target_level_lufs = db_to_lufs(task.target_level) # analyse tracks # list of track Gain objects @@ -250,9 +237,9 @@ class FfmpegBackend(Backend): # total number of BS.1770 gating blocks n_blocks = 0 - for item in items: + for item in task.items: track_gain, track_n_blocks = self._analyse_item( - item, target_level, peak + item, task.target_level, task.peak ) track_gains.append(track_gain) @@ -287,10 +274,11 @@ class FfmpegBackend(Backend): self._log.debug( "{}: gain {} LU, peak {}" - .format(items, album_gain, album_peak) + .format(task.items, album_gain, album_peak) ) - return AlbumGain(Gain(album_gain, album_peak), track_gains) + task.album_gain = AlbumGain(Gain(album_gain, album_peak), track_gains) + return task def _construct_cmd(self, item, peak_method): """Construct the shell command to analyse items.""" @@ -466,28 +454,30 @@ class CommandBackend(Backend): self.noclip = config['noclip'].get(bool) - def compute_track_gain(self, items, target_level, peak): - """Computes the track gain of the given tracks, returns a list - of TrackGain objects. + def compute_track_gain(self, task): + """Computes the track gain for the tracks belonging to `task`, and sets + the `track_gains` attribute on the task. Returns `task`. """ - supported_items = list(filter(self.format_supported, items)) - output = self.compute_gain(supported_items, target_level, False) - return output + supported_items = list(filter(self.format_supported, task.items)) + output = self.compute_gain(supported_items, task.target_level, False) + task.track_gains = output + return task - def compute_album_gain(self, items, target_level, peak): - """Computes the album gain of the given album, returns an - AlbumGain object. + def compute_album_gain(self, task): + """Computes the album gain for the album belonging to `task`, and sets + the `album_gain` attribute on the task. Returns `task`. """ # TODO: What should be done when not all tracks in the album are # supported? - supported_items = list(filter(self.format_supported, items)) - if len(supported_items) != len(items): + supported_items = list(filter(self.format_supported, task.items)) + if len(supported_items) != len(task.items): self._log.debug('tracks are of unsupported format') return AlbumGain(None, []) - output = self.compute_gain(supported_items, target_level, True) - return AlbumGain(output[-1], output[:-1]) + output = self.compute_gain(supported_items, task.target_level, True) + task.album_gain = AlbumGain(output[-1], output[:-1]) + return task def format_supported(self, item): """Checks whether the given item is supported by the selected tool. @@ -668,21 +658,28 @@ class GStreamerBackend(Backend): if self._error is not None: raise self._error - def compute_track_gain(self, items, target_level, peak): - self.compute(items, target_level, False) - if len(self._file_tags) != len(items): + def compute_track_gain(self, task): + """Computes the track gain for the tracks belonging to `task`, and sets + the `track_gains` attribute on the task. Returns `task`. + """ + self.compute(task.items, task.target_level, False) + if len(self._file_tags) != len(task.items): raise ReplayGainError("Some tracks did not receive tags") ret = [] - for item in items: + for item in task.items: ret.append(Gain(self._file_tags[item]["TRACK_GAIN"], self._file_tags[item]["TRACK_PEAK"])) - return ret + task.track_gains = ret + return task - def compute_album_gain(self, items, target_level, peak): - items = list(items) - self.compute(items, target_level, True) + def compute_album_gain(self, task): + """Computes the album gain for the album belonging to `task`, and sets + the `album_gain` attribute on the task. Returns `task`. + """ + items = list(task.items) + self.compute(items, task.target_level, True) if len(self._file_tags) != len(items): raise ReplayGainError("Some items in album did not receive tags") @@ -704,7 +701,8 @@ class GStreamerBackend(Backend): except KeyError: raise ReplayGainError("results missing for album") - return AlbumGain(Gain(gain, peak), track_gains) + task.album_gain = AlbumGain(Gain(gain, peak), track_gains) + return task def close(self): self._bus.remove_signal_watch() @@ -897,12 +895,14 @@ class AudioToolsBackend(Backend): return return rg - def compute_track_gain(self, items, target_level, peak): - """Compute ReplayGain values for the requested items. - - :return list: list of :class:`Gain` objects + def compute_track_gain(self, task): + """Computes the track gain for the tracks belonging to `task`, and sets + the `track_gains` attribute on the task. Returns `task`. """ - return [self._compute_track_gain(item, target_level) for item in items] + gains = [self._compute_track_gain(i, task.target_level) + for i in task.items] + task.track_gains = gains + return task def _with_target_level(self, gain, target_level): """Return `gain` relative to `target_level`. @@ -947,23 +947,22 @@ class AudioToolsBackend(Backend): item.artist, item.title, rg_track_gain, rg_track_peak) return Gain(gain=rg_track_gain, peak=rg_track_peak) - def compute_album_gain(self, items, target_level, peak): - """Compute ReplayGain values for the requested album and its items. - - :rtype: :class:`AlbumGain` + def compute_album_gain(self, task): + """Computes the album gain for the album belonging to `task`, and sets + the `album_gain` attribute on the task. Returns `task`. """ # The first item is taken and opened to get the sample rate to # initialize the replaygain object. The object is used for all the # tracks in the album to get the album values. - item = list(items)[0] + item = list(task.items)[0] audiofile = self.open_audio_file(item) rg = self.init_replaygain(audiofile, item) track_gains = [] - for item in items: + for item in task.items: audiofile = self.open_audio_file(item) rg_track_gain, rg_track_peak = self._title_gain( - rg, audiofile, target_level + rg, audiofile, task.target_level ) track_gains.append( Gain(gain=rg_track_gain, peak=rg_track_peak) @@ -974,14 +973,16 @@ class AudioToolsBackend(Backend): # After getting the values for all tracks, it's possible to get the # album values. rg_album_gain, rg_album_peak = rg.album_gain() - rg_album_gain = self._with_target_level(rg_album_gain, target_level) + rg_album_gain = self._with_target_level( + rg_album_gain, task.target_level) self._log.debug('ReplayGain for album {0}: {1:.2f}, {2:.2f}', - items[0].album, rg_album_gain, rg_album_peak) + task.items[0].album, rg_album_gain, rg_album_peak) - return AlbumGain( + task.album_gain = AlbumGain( Gain(gain=rg_album_gain, peak=rg_album_peak), track_gains=track_gains ) + return task class ExceptionWatcher(Thread): @@ -1070,17 +1071,10 @@ class ReplayGainPlugin(BeetsPlugin): ) ) - # The key in this dict is the `use_r128` flag. - self.gain_handlers = { - True: R128GainHandler( - self.config, - self._log, - ), - False: GainHandler( - self.config, - Peak[peak_method], - self._log, - ), + # The key in these dicts is the `use_r128` flag. + self.peak_methods = { + True: Peak.none, + False: Peak[peak_method] } # On-import analysis. @@ -1150,6 +1144,22 @@ class ReplayGainPlugin(BeetsPlugin): return False + def create_task(self, items, use_r128, album=None): + if use_r128: + return R128Task( + items, album, + self.config["r128_targetlevel"].as_number(), + Peak.none, # R128_* tags do not store the track/album peak + self._log, + ) + else: + return RgTask( + items, album, + self.config["targetlevel"].as_number(), + self.peak_methods[use_r128], + self._log, + ) + def handle_album(self, album, write, force=False): """Compute album and track replay gain store it in all of the album's items. @@ -1172,8 +1182,6 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info('analyzing {0}', album) - handler = self.gain_handlers[use_r128] - discs = {} if self.per_disc: for item in album.items(): @@ -1184,33 +1192,30 @@ class ReplayGainPlugin(BeetsPlugin): discs[1] = album.items() for discnumber, items in discs.items(): - def _store_album(album_gain): - if not album_gain or not album_gain.album_gain \ - or len(album_gain.track_gains) != len(items): + def _store_album(task): + if task.album_gain is None or not task.album_gain.album_gain \ + or len(task.album_gain.track_gains) != len(task.items): # In some cases, backends fail to produce a valid # `album_gain` without throwing FatalReplayGainError # => raise non-fatal exception & continue raise ReplayGainError( "ReplayGain backend `{}` failed " "for some tracks in album {}" - .format(self.backend_name, album) + .format(self.backend_name, task.album) ) - for item, track_gain in zip(items, - album_gain.track_gains): - handler.store_track_gain(item, track_gain) - handler.store_album_gain(item, album_gain.album_gain) + for item, track_gain in zip(task.items, + task.album_gain.track_gains): + task.store_track_gain(item, track_gain) + task.store_album_gain(item, task.album_gain.album_gain) if write: item.try_write() self._log.debug('done analyzing {0}', item) + task = self.create_task(items, use_r128, album=album) try: self._apply( - self.backend_instance.compute_album_gain, args=(), - kwds={ - "items": list(items), - "target_level": handler.target_level, - "peak": handler.peak, - }, + self.backend_instance.compute_album_gain, + args=[task], kwds={}, callback=_store_album ) except ReplayGainError as e: @@ -1231,10 +1236,9 @@ class ReplayGainPlugin(BeetsPlugin): return use_r128 = self.should_use_r128(item) - handler = self.gain_handlers[use_r128] - def _store_track(track_gains): - if not track_gains or len(track_gains) != 1: + def _store_track(task): + if task.track_gains is None or len(task.track_gains) != 1: # In some cases, backends fail to produce a valid # `track_gains` without throwing FatalReplayGainError # => raise non-fatal exception & continue @@ -1243,19 +1247,16 @@ class ReplayGainPlugin(BeetsPlugin): .format(self.backend_name, item) ) - handler.store_track_gain(item, track_gains[0]) + task.store_track_gain(item, task.track_gains[0]) if write: item.try_write() self._log.debug('done analyzing {0}', item) + task = self.create_task([item], use_r128) try: self._apply( - self.backend_instance.compute_track_gain, args=(), - kwds={ - "items": [item], - "target_level": handler.target_level, - "peak": handler.peak, - }, + self.backend_instance.compute_track_gain, + args=[task], kwds={}, callback=_store_track ) except ReplayGainError as e: From ad75cbf4a5299f54ab6413391cd8d29b69584405 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Fri, 19 Mar 2021 11:42:23 +0100 Subject: [PATCH 113/357] replaygain: Flatten data structures Now that we have the *Task objects, the AlbumGain tuple can be removed. --- beetsplug/replaygain.py | 49 +++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3551ddeb2..f95d42df3 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -94,9 +94,6 @@ def lufs_to_db(db): # gain: in LU to reference level # peak: part of full scale (FS is 1.0) Gain = collections.namedtuple("Gain", "gain peak") -# album_gain: Gain object -# track_gains: list of Gain objects -AlbumGain = collections.namedtuple("AlbumGain", "album_gain track_gains") ALL_PEAK_METHODS = ["true", "sample"] @@ -120,9 +117,13 @@ class RgTask(): self._log.debug('applied track gain {0} LU, peak {1} of FS', item.rg_track_gain, item.rg_track_peak) - def store_album_gain(self, item, album_gain): - item.rg_album_gain = album_gain.gain - item.rg_album_peak = album_gain.peak + def store_album_gain(self, item): + """ + + The caller needs to ensure that `self.album_gain is not None`. + """ + item.rg_album_gain = self.album_gain.gain + item.rg_album_peak = self.album_gain.peak item.store() self._log.debug('applied album gain {0} LU, peak {1} of FS', item.rg_album_gain, item.rg_album_peak) @@ -135,8 +136,12 @@ class R128Task(RgTask): self._log.debug('applied r128 track gain {0} LU', item.r128_track_gain) - def store_album_gain(self, item, album_gain): - item.r128_album_gain = album_gain.gain + def store_album_gain(self, item): + """ + + The caller needs to ensure that `self.album_gain is not None`. + """ + item.r128_album_gain = self.album_gain.gain item.store() self._log.debug('applied r128 album gain {0} LU', item.r128_album_gain) @@ -277,7 +282,8 @@ class FfmpegBackend(Backend): .format(task.items, album_gain, album_peak) ) - task.album_gain = AlbumGain(Gain(album_gain, album_peak), track_gains) + task.album_gain = Gain(album_gain, album_peak) + task.track_gains = track_gains return task def _construct_cmd(self, item, peak_method): @@ -473,10 +479,13 @@ class CommandBackend(Backend): supported_items = list(filter(self.format_supported, task.items)) if len(supported_items) != len(task.items): self._log.debug('tracks are of unsupported format') - return AlbumGain(None, []) + task.album_gain = None + task.track_gains = None + return task output = self.compute_gain(supported_items, task.target_level, True) - task.album_gain = AlbumGain(output[-1], output[:-1]) + task.album_gain = output[-1] + task.track_gains = output[:-1] return task def format_supported(self, item): @@ -701,7 +710,8 @@ class GStreamerBackend(Backend): except KeyError: raise ReplayGainError("results missing for album") - task.album_gain = AlbumGain(Gain(gain, peak), track_gains) + task.album_gain = Gain(gain, peak) + task.track_gains = track_gains return task def close(self): @@ -978,10 +988,8 @@ class AudioToolsBackend(Backend): self._log.debug('ReplayGain for album {0}: {1:.2f}, {2:.2f}', task.items[0].album, rg_album_gain, rg_album_peak) - task.album_gain = AlbumGain( - Gain(gain=rg_album_gain, peak=rg_album_peak), - track_gains=track_gains - ) + task.album_gain = Gain(gain=rg_album_gain, peak=rg_album_peak) + task.track_gains = track_gains return task @@ -1193,8 +1201,8 @@ class ReplayGainPlugin(BeetsPlugin): for discnumber, items in discs.items(): def _store_album(task): - if task.album_gain is None or not task.album_gain.album_gain \ - or len(task.album_gain.track_gains) != len(task.items): + if (task.album_gain is None or task.track_gains is None + or len(task.track_gains) != len(task.items)): # In some cases, backends fail to produce a valid # `album_gain` without throwing FatalReplayGainError # => raise non-fatal exception & continue @@ -1203,10 +1211,9 @@ class ReplayGainPlugin(BeetsPlugin): "for some tracks in album {}" .format(self.backend_name, task.album) ) - for item, track_gain in zip(task.items, - task.album_gain.track_gains): + for item, track_gain in zip(task.items, task.track_gains): task.store_track_gain(item, track_gain) - task.store_album_gain(item, task.album_gain.album_gain) + task.store_album_gain(item) if write: item.try_write() self._log.debug('done analyzing {0}', item) From 9ead9cdbb8c4ca9c21cb65bdbbffd09c13d7fa2c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Fri, 19 Mar 2021 11:46:01 +0100 Subject: [PATCH 114/357] replaygain: pass the `write` flag explicitly The variable was previously captured by the closure. This is in preparation for moving these nested functions elsewhere. --- beetsplug/replaygain.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index f95d42df3..9c3582c5d 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1200,7 +1200,7 @@ class ReplayGainPlugin(BeetsPlugin): discs[1] = album.items() for discnumber, items in discs.items(): - def _store_album(task): + def _store_album(task, write): if (task.album_gain is None or task.track_gains is None or len(task.track_gains) != len(task.items)): # In some cases, backends fail to produce a valid @@ -1223,7 +1223,7 @@ class ReplayGainPlugin(BeetsPlugin): self._apply( self.backend_instance.compute_album_gain, args=[task], kwds={}, - callback=_store_album + callback=lambda task: _store_album(task, write) ) except ReplayGainError as e: self._log.info("ReplayGain error: {0}", e) @@ -1244,7 +1244,7 @@ class ReplayGainPlugin(BeetsPlugin): use_r128 = self.should_use_r128(item) - def _store_track(task): + def _store_track(task, write): if task.track_gains is None or len(task.track_gains) != 1: # In some cases, backends fail to produce a valid # `track_gains` without throwing FatalReplayGainError @@ -1264,7 +1264,7 @@ class ReplayGainPlugin(BeetsPlugin): self._apply( self.backend_instance.compute_track_gain, args=[task], kwds={}, - callback=_store_track + callback=lambda task: _store_track(task, write) ) except ReplayGainError as e: self._log.info("ReplayGain error: {0}", e) From a2df6df9da26fb7b4a22cafdf5b71a3acc7f7a34 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Fri, 19 Mar 2021 11:53:13 +0100 Subject: [PATCH 115/357] replaygain: store_track, store_album are methods on *Task Also, add a convenience function `store()` that dispatches two the either of the two methods. This will be useful later, when rewriting the parallel code (but doesn't simplify the code now). --- beetsplug/replaygain.py | 90 +++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 9c3582c5d..3f597dcc1 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -101,23 +101,24 @@ Peak = enum.Enum("Peak", ["none"] + ALL_PEAK_METHODS) class RgTask(): - def __init__(self, items, album, target_level, peak, log): + def __init__(self, items, album, target_level, peak, backend_name, log): self.items = items self.album = album self.target_level = target_level self.peak = peak + self.backend_name = backend_name self._log = log self.album_gain = None self.track_gains = None - def store_track_gain(self, item, track_gain): + def _store_track_gain(self, item, track_gain): item.rg_track_gain = track_gain.gain item.rg_track_peak = track_gain.peak item.store() self._log.debug('applied track gain {0} LU, peak {1} of FS', item.rg_track_gain, item.rg_track_peak) - def store_album_gain(self, item): + def _store_album_gain(self, item): """ The caller needs to ensure that `self.album_gain is not None`. @@ -128,15 +129,55 @@ class RgTask(): self._log.debug('applied album gain {0} LU, peak {1} of FS', item.rg_album_gain, item.rg_album_peak) + def _store_track(self, write): + item = self.items[0] + if self.track_gains is None or len(self.track_gains) != 1: + # In some cases, backends fail to produce a valid + # `track_gains` without throwing FatalReplayGainError + # => raise non-fatal exception & continue + raise ReplayGainError( + "ReplayGain backend `{}` failed for track {}" + .format(self.backend_name, item) + ) + + self._store_track_gain(item, self.track_gains[0]) + if write: + item.try_write() + self._log.debug('done analyzing {0}', item) + + def _store_album(self, write): + if (self.album_gain is None or self.track_gains is None + or len(self.track_gains) != len(self.items)): + # In some cases, backends fail to produce a valid + # `album_gain` without throwing FatalReplayGainError + # => raise non-fatal exception & continue + raise ReplayGainError( + "ReplayGain backend `{}` failed " + "for some tracks in album {}" + .format(self.backend_name, self.album) + ) + for item, track_gain in zip(self.items, self.track_gains): + self._store_track_gain(item, track_gain) + self._store_album_gain(item) + if write: + item.try_write() + self._log.debug('done analyzing {0}', item) + + def store(self, write): + if self.album is not None: + self._store_album(write) + else: + self._store_track(write) + class R128Task(RgTask): - def store_track_gain(self, item, track_gain): + def _store_track_gain(self, item, track_gain): item.r128_track_gain = track_gain.gain item.store() self._log.debug('applied r128 track gain {0} LU', item.r128_track_gain) - def store_album_gain(self, item): + def _store_album_gain(self, item): """ The caller needs to ensure that `self.album_gain is not None`. @@ -1158,6 +1199,7 @@ class ReplayGainPlugin(BeetsPlugin): items, album, self.config["r128_targetlevel"].as_number(), Peak.none, # R128_* tags do not store the track/album peak + self.backend_instance.NAME, self._log, ) else: @@ -1165,6 +1207,7 @@ class ReplayGainPlugin(BeetsPlugin): items, album, self.config["targetlevel"].as_number(), self.peak_methods[use_r128], + self.backend_instance.NAME, self._log, ) @@ -1200,30 +1243,12 @@ class ReplayGainPlugin(BeetsPlugin): discs[1] = album.items() for discnumber, items in discs.items(): - def _store_album(task, write): - if (task.album_gain is None or task.track_gains is None - or len(task.track_gains) != len(task.items)): - # In some cases, backends fail to produce a valid - # `album_gain` without throwing FatalReplayGainError - # => raise non-fatal exception & continue - raise ReplayGainError( - "ReplayGain backend `{}` failed " - "for some tracks in album {}" - .format(self.backend_name, task.album) - ) - for item, track_gain in zip(task.items, task.track_gains): - task.store_track_gain(item, track_gain) - task.store_album_gain(item) - if write: - item.try_write() - self._log.debug('done analyzing {0}', item) - task = self.create_task(items, use_r128, album=album) try: self._apply( self.backend_instance.compute_album_gain, args=[task], kwds={}, - callback=lambda task: _store_album(task, write) + callback=lambda task: task.store(write) ) except ReplayGainError as e: self._log.info("ReplayGain error: {0}", e) @@ -1244,27 +1269,12 @@ class ReplayGainPlugin(BeetsPlugin): use_r128 = self.should_use_r128(item) - def _store_track(task, write): - if task.track_gains is None or len(task.track_gains) != 1: - # In some cases, backends fail to produce a valid - # `track_gains` without throwing FatalReplayGainError - # => raise non-fatal exception & continue - raise ReplayGainError( - "ReplayGain backend `{}` failed for track {}" - .format(self.backend_name, item) - ) - - task.store_track_gain(item, task.track_gains[0]) - if write: - item.try_write() - self._log.debug('done analyzing {0}', item) - task = self.create_task([item], use_r128) try: self._apply( self.backend_instance.compute_track_gain, args=[task], kwds={}, - callback=lambda task: _store_track(task, write) + callback=lambda task: task.store(write) ) except ReplayGainError as e: self._log.info("ReplayGain error: {0}", e) From ea0d905b22e4aa736c2206daab0390aa1b25354e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 11:17:33 +0100 Subject: [PATCH 116/357] replaygain tests: remmove duplicate store() probably leftover from debugging database issue --- test/test_replaygain.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 58b487fad..194e83688 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -43,8 +43,6 @@ def reset_replaygain(item): item['rg_album_gain'] = None item.write() item.store() - item.store() - item.store() class ReplayGainCliTestBase(TestHelper): From c0af86c04ac48bf8bc6ed0fa110084e158fe1d86 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 11:18:54 +0100 Subject: [PATCH 117/357] replaygain tests: remove duplicated function Not sure how this got here, the git blame doesn't make a lot of sense.. maybe an incorrect merge commit? --- test/test_replaygain.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 194e83688..548220c0f 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -41,6 +41,8 @@ def reset_replaygain(item): item['rg_track_gain'] = None item['rg_album_gain'] = None item['rg_album_gain'] = None + item['r128_track_gain'] = None + item['r128_album_gain'] = None item.write() item.store() @@ -64,16 +66,6 @@ class ReplayGainCliTestBase(TestHelper): self.teardown_beets() self.unload_plugins() - def _reset_replaygain(self, item): - item['rg_track_peak'] = None - item['rg_track_gain'] = None - item['rg_album_peak'] = None - item['rg_album_gain'] = None - item['r128_track_gain'] = None - item['r128_album_gain'] = None - item.write() - item.store() - def test_cli_saves_track_gain(self): for item in self.lib.items(): self.assertIsNone(item.rg_track_peak) @@ -137,7 +129,7 @@ class ReplayGainCliTestBase(TestHelper): album = self.add_album_fixture(2, ext="opus") for item in album.items(): - self._reset_replaygain(item) + reset_replaygain(item) self.run_command('replaygain', '-a') @@ -155,7 +147,7 @@ class ReplayGainCliTestBase(TestHelper): def analyse(target_level): self.config['replaygain']['targetlevel'] = target_level - self._reset_replaygain(item) + reset_replaygain(item) self.run_command('replaygain', '-f') mediafile = MediaFile(item.path) return mediafile.rg_track_gain From ad4c68112f6ac25e582babb196d2dd59723c84d6 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 11:21:59 +0100 Subject: [PATCH 118/357] test helper: add support for multi-disc album fixtures --- test/helper.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/test/helper.py b/test/helper.py index 988995f48..f7d37b654 100644 --- a/test/helper.py +++ b/test/helper.py @@ -373,21 +373,23 @@ class TestHelper: items.append(item) return items - def add_album_fixture(self, track_count=1, ext='mp3'): + def add_album_fixture(self, track_count=1, ext='mp3', disc_count=1): """Add an album with files to the database. """ items = [] path = os.path.join(_common.RSRC, util.bytestring_path('full.' + ext)) - for i in range(track_count): - item = Item.from_path(path) - item.album = '\u00e4lbum' # Check unicode paths - item.title = f't\u00eftle {i}' - # mtime needs to be set last since other assignments reset it. - item.mtime = 12345 - item.add(self.lib) - item.move(operation=MoveOperation.COPY) - item.store() - items.append(item) + for discnumber in range(1, disc_count + 1): + for i in range(track_count): + item = Item.from_path(path) + item.album = '\u00e4lbum' # Check unicode paths + item.title = f't\u00eftle {i}' + item.disc = discnumber + # mtime needs to be set last since other assignments reset it. + item.mtime = 12345 + item.add(self.lib) + item.move(operation=MoveOperation.COPY) + item.store() + items.append(item) return self.lib.add_album(items) def create_mediafile_fixture(self, ext='mp3', images=[]): From 81396412ef2c7d92df6787c4a2a08a75757a441c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 11:26:48 +0100 Subject: [PATCH 119/357] replaygain test: add album fixtures in individual tests instead of setUp preparation for adding more tests that want different fixtures --- test/test_replaygain.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 548220c0f..73345261d 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -58,15 +58,20 @@ class ReplayGainCliTestBase(TestHelper): self.teardown_beets() self.unload_plugins() - album = self.add_album_fixture(2) + def _add_album(self, *args, **kwargs): + album = self.add_album_fixture(*args, **kwargs) for item in album.items(): reset_replaygain(item) + return album + def tearDown(self): self.teardown_beets() self.unload_plugins() def test_cli_saves_track_gain(self): + self._add_album(2) + for item in self.lib.items(): self.assertIsNone(item.rg_track_peak) self.assertIsNone(item.rg_track_gain) @@ -92,6 +97,8 @@ class ReplayGainCliTestBase(TestHelper): mediafile.rg_track_gain, item.rg_track_gain, places=2) def test_cli_skips_calculated_tracks(self): + self._add_album(2) + self.run_command('replaygain') item = self.lib.items()[0] peak = item.rg_track_peak @@ -101,6 +108,8 @@ class ReplayGainCliTestBase(TestHelper): self.assertEqual(item.rg_track_peak, peak) def test_cli_saves_album_gain_to_file(self): + self._add_album(2) + for item in self.lib.items(): mediafile = MediaFile(item.path) self.assertIsNone(mediafile.rg_album_peak) @@ -127,9 +136,7 @@ class ReplayGainCliTestBase(TestHelper): # opus not supported by command backend return - album = self.add_album_fixture(2, ext="opus") - for item in album.items(): - reset_replaygain(item) + album = self._add_album(2, ext="opus") self.run_command('replaygain', '-a') @@ -143,7 +150,8 @@ class ReplayGainCliTestBase(TestHelper): self.assertIsNotNone(mediafile.r128_album_gain) def test_target_level_has_effect(self): - item = self.lib.items()[0] + album = self._add_album(1) + item = album.items()[0] def analyse(target_level): self.config['replaygain']['targetlevel'] = target_level From 80f7c58e5adbe58304b0ea9cceadd34851f55339 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 11:46:10 +0100 Subject: [PATCH 120/357] replaygain test: add basic per_disc testcase The per_disc codepath is currently not exercised at all by the tests. This test is not very meaningful, but better than nothing. --- beetsplug/replaygain.py | 3 +-- test/test_replaygain.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 3f597dcc1..2c1ccd52f 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1096,7 +1096,6 @@ class ReplayGainPlugin(BeetsPlugin): # FIXME: Consider renaming the configuration option and deprecating the # old name 'overwrite'. self.force_on_import = self.config['overwrite'].get(bool) - self.per_disc = self.config['per_disc'].get(bool) # Remember which backend is used for CLI feedback self.backend_name = self.config['backend'].as_str() @@ -1234,7 +1233,7 @@ class ReplayGainPlugin(BeetsPlugin): self._log.info('analyzing {0}', album) discs = {} - if self.per_disc: + if self.config['per_disc'].get(bool): for item in album.items(): if discs.get(item.disc) is None: discs[item.disc] = [] diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 73345261d..e63034a26 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -169,6 +169,18 @@ class ReplayGainCliTestBase(TestHelper): self.assertNotEqual(gain_relative_to_84, gain_relative_to_89) + def test_per_disc(self): + # Use the per_disc option and add a little more concurrency. + album = self._add_album(track_count=4, disc_count=3) + self.config['replaygain']['per_disc'] = True + self.run_command('replaygain', '-a') + + # FIXME: Add fixtures with known track/album gain (within a suitable + # tolerance) so that we can actually check per-disc operation here. + for item in album.items(): + self.assertIsNotNone(item.rg_track_gain) + self.assertIsNotNone(item.rg_album_gain) + @unittest.skipIf(not GST_AVAILABLE, 'gstreamer cannot be found') class ReplayGainGstCliTest(ReplayGainCliTestBase, unittest.TestCase): From 1442594e5ba8a64e2eb5acb2f47095a8d5df9bc2 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 16:28:14 +0100 Subject: [PATCH 121/357] replaygain test: reorganize backend handling in an attempt to make it simpler to add new test classes --- test/test_replaygain.py | 73 +++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index e63034a26..4d22f3208 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -47,8 +47,48 @@ def reset_replaygain(item): item.store() +class GstBackendMixin(): + backend = 'gstreamer' + has_r128_support = True + + def test_backend(self): + """Check whether the backend actually has all required functionality. + """ + try: + # Check if required plugins can be loaded by instantiating a + # GStreamerBackend (via its .__init__). + config['replaygain']['targetlevel'] = 89 + GStreamerBackend(config['replaygain'], None) + except FatalGstreamerPluginReplayGainError as e: + # Skip the test if plugins could not be loaded. + self.skipTest(str(e)) + + +class CmdBackendMixin(): + backend = 'command' + has_r128_support = False + + def test_backend(self): + """Check whether the backend actually has all required functionality. + """ + pass + + +class FfmpegBackendMixin(): + backend = 'ffmpeg' + has_r128_support = True + + def test_backend(self): + """Check whether the backend actually has all required functionality. + """ + pass + + class ReplayGainCliTestBase(TestHelper): def setUp(self): + # Implemented by Mixins, see above. This may decide to skip the test. + self.test_backend() + self.setup_beets(disk=True) self.config['replaygain']['backend'] = self.backend @@ -132,9 +172,9 @@ class ReplayGainCliTestBase(TestHelper): self.assertNotEqual(max(peaks), 0.0) def test_cli_writes_only_r128_tags(self): - if self.backend == "command": - # opus not supported by command backend - return + if not self.has_r128_support: + self.skipTest("r128 tags for opus not supported on backend {}" + .format(self.backend)) album = self._add_album(2, ext="opus") @@ -183,30 +223,21 @@ class ReplayGainCliTestBase(TestHelper): @unittest.skipIf(not GST_AVAILABLE, 'gstreamer cannot be found') -class ReplayGainGstCliTest(ReplayGainCliTestBase, unittest.TestCase): - backend = 'gstreamer' - - def setUp(self): - try: - # Check if required plugins can be loaded by instantiating a - # GStreamerBackend (via its .__init__). - config['replaygain']['targetlevel'] = 89 - GStreamerBackend(config['replaygain'], None) - except FatalGstreamerPluginReplayGainError as e: - # Skip the test if plugins could not be loaded. - self.skipTest(str(e)) - - super().setUp() +class ReplayGainGstCliTest(ReplayGainCliTestBase, unittest.TestCase, + GstBackendMixin): + pass @unittest.skipIf(not GAIN_PROG_AVAILABLE, 'no *gain command found') -class ReplayGainCmdCliTest(ReplayGainCliTestBase, unittest.TestCase): - backend = 'command' +class ReplayGainCmdCliTest(ReplayGainCliTestBase, unittest.TestCase, + CmdBackendMixin): + pass @unittest.skipIf(not FFMPEG_AVAILABLE, 'ffmpeg cannot be found') -class ReplayGainFfmpegTest(ReplayGainCliTestBase, unittest.TestCase): - backend = 'ffmpeg' +class ReplayGainFfmpegCliTest(ReplayGainCliTestBase, unittest.TestCase, + FfmpegBackendMixin): + pass def suite(): From e8960501f2890e6d6f410e557b16df63c9fdb4f2 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 16:28:34 +0100 Subject: [PATCH 122/357] replaygain test: extend tests for skipping items Test more combinations of tags that might initially be present and expected tags. The R128 codepath and the case of having the wrong type of tags wasn't really tested before. --- test/test_replaygain.py | 78 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 4d22f3208..3d9023e0a 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -137,15 +137,81 @@ class ReplayGainCliTestBase(TestHelper): mediafile.rg_track_gain, item.rg_track_gain, places=2) def test_cli_skips_calculated_tracks(self): - self._add_album(2) + album_rg = self._add_album(1) + item_rg = album_rg.items()[0] + + if self.has_r128_support: + album_r128 = self._add_album(1, ext="opus") + item_r128 = album_r128.items()[0] self.run_command('replaygain') - item = self.lib.items()[0] - peak = item.rg_track_peak - item.rg_track_gain = 0.0 + + item_rg.load() + self.assertIsNotNone(item_rg.rg_track_gain) + self.assertIsNotNone(item_rg.rg_track_peak) + self.assertIsNone(item_rg.r128_track_gain) + + item_rg.rg_track_gain += 1.0 + item_rg.rg_track_peak += 1.0 + item_rg.store() + rg_track_gain = item_rg.rg_track_gain + rg_track_peak = item_rg.rg_track_peak + + if self.has_r128_support: + item_r128.load() + self.assertIsNotNone(item_r128.r128_track_gain) + self.assertIsNone(item_r128.rg_track_gain) + self.assertIsNone(item_r128.rg_track_peak) + + item_r128.r128_track_gain += 1.0 + item_r128.store() + r128_track_gain = item_r128.r128_track_gain + self.run_command('replaygain') - self.assertEqual(item.rg_track_gain, 0.0) - self.assertEqual(item.rg_track_peak, peak) + + item_rg.load() + self.assertEqual(item_rg.rg_track_gain, rg_track_gain) + self.assertEqual(item_rg.rg_track_peak, rg_track_peak) + + if self.has_r128_support: + item_r128.load() + self.assertEqual(item_r128.r128_track_gain, r128_track_gain) + + def test_cli_does_not_skip_wrong_tag_type(self): + """Check that items that have tags of the wrong type won't be skipped. + """ + if not self.has_r128_support: + # This test is a lot less interesting if the backend cannot write + # both tag types. + self.skipTest("r128 tags for opus not supported on backend {}" + .format(self.backend)) + + album_rg = self._add_album(1) + item_rg = album_rg.items()[0] + + album_r128 = self._add_album(1, ext="opus") + item_r128 = album_r128.items()[0] + + item_rg.r128_track_gain = 0.0 + item_rg.store() + + item_r128.rg_track_gain = 0.0 + item_r128.rg_track_peak = 42.0 + item_r128.store() + + self.run_command('replaygain') + item_rg.load() + item_r128.load() + + self.assertIsNotNone(item_rg.rg_track_gain) + self.assertIsNotNone(item_rg.rg_track_peak) + # FIXME: Should the plugin null this field? + # self.assertIsNone(item_rg.r128_track_gain) + + self.assertIsNotNone(item_r128.r128_track_gain) + # FIXME: Should the plugin null these fields? + # self.assertIsNone(item_r128.rg_track_gain) + # self.assertIsNone(item_r128.rg_track_peak) def test_cli_saves_album_gain_to_file(self): self._add_album(2) From 676c8fe45ebabc2c0720074008c86b7389ecc12c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 16:30:32 +0100 Subject: [PATCH 123/357] replaygain test: add r128 targetlevel test --- test/test_replaygain.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 3d9023e0a..db63d5838 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -255,23 +255,37 @@ class ReplayGainCliTestBase(TestHelper): self.assertIsNotNone(mediafile.r128_track_gain) self.assertIsNotNone(mediafile.r128_album_gain) - def test_target_level_has_effect(self): + def test_targetlevel_has_effect(self): album = self._add_album(1) item = album.items()[0] def analyse(target_level): self.config['replaygain']['targetlevel'] = target_level - reset_replaygain(item) self.run_command('replaygain', '-f') - mediafile = MediaFile(item.path) - return mediafile.rg_track_gain + item.load() + return item.rg_track_gain gain_relative_to_84 = analyse(84) gain_relative_to_89 = analyse(89) - # check that second calculation did work - if gain_relative_to_84 is not None: - self.assertIsNotNone(gain_relative_to_89) + self.assertNotEqual(gain_relative_to_84, gain_relative_to_89) + + def test_r128_targetlevel_has_effect(self): + if not self.has_r128_support: + self.skipTest("r128 tags for opus not supported on backend {}" + .format(self.backend)) + + album = self._add_album(1, ext="opus") + item = album.items()[0] + + def analyse(target_level): + self.config['replaygain']['r128_targetlevel'] = target_level + self.run_command('replaygain', '-f') + item.load() + return item.r128_track_gain + + gain_relative_to_84 = analyse(84) + gain_relative_to_89 = analyse(89) self.assertNotEqual(gain_relative_to_84, gain_relative_to_89) From fd7919e6275dd63fdb9f1c28a7bd831472974138 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 20 Mar 2021 16:30:55 +0100 Subject: [PATCH 124/357] replaygain test: add basic importer test --- test/test_replaygain.py | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index db63d5838..499befee2 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -320,6 +320,75 @@ class ReplayGainFfmpegCliTest(ReplayGainCliTestBase, unittest.TestCase, pass +class ImportTest(TestHelper): + threaded = False + + def setUp(self): + # Implemented by Mixins, see above. This may decide to skip the test. + self.test_backend() + + self.setup_beets(disk=True) + self.config['threaded'] = self.threaded + self.config['replaygain'] = { + 'backend': self.backend, + } + + try: + self.load_plugins('replaygain') + except Exception: + import sys + # store exception info so an error in teardown does not swallow it + exc_info = sys.exc_info() + try: + self.teardown_beets() + self.unload_plugins() + except Exception: + # if load_plugins() failed then setup is incomplete and + # teardown operations may fail. In particular # {Item,Album} + # may not have the _original_types attribute in unload_plugins + pass + raise None.with_traceback(exc_info[2]) + + self.importer = self.create_importer() + + def tearDown(self): + self.unload_plugins() + self.teardown_beets() + + def test_import_converted(self): + self.importer.run() + for item in self.lib.items(): + # FIXME: Add fixtures with known track/album gain (within a + # suitable tolerance) so that we can actually check correct + # operation here. + self.assertIsNotNone(item.rg_track_gain) + self.assertIsNotNone(item.rg_album_gain) + + +@unittest.skipIf(not GST_AVAILABLE, 'gstreamer cannot be found') +class ReplayGainGstImportTest(ImportTest, unittest.TestCase, + GstBackendMixin): + pass + + +@unittest.skipIf(not GAIN_PROG_AVAILABLE, 'no *gain command found') +class ReplayGainCmdImportTest(ImportTest, unittest.TestCase, + CmdBackendMixin): + pass + + +@unittest.skipIf(not FFMPEG_AVAILABLE, 'ffmpeg cannot be found') +class ReplayGainFfmpegImportTest(ImportTest, unittest.TestCase, + FfmpegBackendMixin): + pass + + +@unittest.skipIf(not FFMPEG_AVAILABLE, 'ffmpeg cannot be found') +class ReplayGainFfmpegThreadedImportTest(ImportTest, unittest.TestCase, + FfmpegBackendMixin): + threaded = True + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From d19cc593e33ceb75080fcf7b57b73ed9355f3d27 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 12 Sep 2021 16:07:52 +0200 Subject: [PATCH 125/357] replaygain: better types to represent peak methods thanks @ybnd in review https://github.com/beetbox/beets/pull/3996 for this suggestion to properly represent the sum type --- beetsplug/replaygain.py | 49 +++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 2c1ccd52f..bbfcd1f2a 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -96,16 +96,18 @@ def lufs_to_db(db): Gain = collections.namedtuple("Gain", "gain peak") -ALL_PEAK_METHODS = ["true", "sample"] -Peak = enum.Enum("Peak", ["none"] + ALL_PEAK_METHODS) +class PeakMethod(enum.Enum): + true = 1 + sample = 2 class RgTask(): - def __init__(self, items, album, target_level, peak, backend_name, log): + def __init__(self, items, album, target_level, peak_method, backend_name, + log): self.items = items self.album = album self.target_level = target_level - self.peak = peak + self.peak_method = peak_method self.backend_name = backend_name self._log = log self.album_gain = None @@ -171,6 +173,11 @@ class RgTask(): class R128Task(RgTask): + def __init__(self, items, album, target_level, backend_name, log): + # R128_* tags do not store the track/album peak + super().__init__(items, album, target_level, None, backend_name, + log) + def _store_track_gain(self, item, track_gain): item.r128_track_gain = track_gain.gain item.store() @@ -260,7 +267,7 @@ class FfmpegBackend(Backend): self._analyse_item( item, task.target_level, - task.peak, + task.peak_method, count_blocks=False, )[0] # take only the gain, discarding number of gating blocks ) @@ -285,7 +292,7 @@ class FfmpegBackend(Backend): for item in task.items: track_gain, track_n_blocks = self._analyse_item( - item, task.target_level, task.peak + item, task.target_level, task.peak_method ) track_gains.append(track_gain) @@ -338,13 +345,15 @@ class FfmpegBackend(Backend): "-map", "a:0", "-filter", - f"ebur128=peak={peak_method}", + "ebur128=peak={}".format( + "none" if peak_method is None else peak_method.name), "-f", "null", "-", ] - def _analyse_item(self, item, target_level, peak, count_blocks=True): + def _analyse_item(self, item, target_level, peak_method, + count_blocks=True): """Analyse item. Return a pair of a Gain object and the number of gating blocks above the threshold. @@ -352,7 +361,6 @@ class FfmpegBackend(Backend): will be 0. """ target_level_lufs = db_to_lufs(target_level) - peak_method = peak.name # call ffmpeg self._log.debug(f"analyzing {item}") @@ -364,12 +372,13 @@ class FfmpegBackend(Backend): # parse output - if peak == Peak.none: + if peak_method is None: peak = 0 else: line_peak = self._find_line( output, - f" {peak_method.capitalize()} peak:".encode(), + # `peak_method` is non-`None` in this arm of the conditional + f" {peak_method.name.capitalize()} peak:".encode(), start_line=len(output) - 1, step_size=-1, ) peak = self._parse_float( @@ -1109,21 +1118,20 @@ class ReplayGainPlugin(BeetsPlugin): ) ) + # FIXME: Consider renaming the configuration option to 'peak_method' + # and deprecating the old name 'peak'. peak_method = self.config["peak"].as_str() - if peak_method not in ALL_PEAK_METHODS: + if peak_method not in PeakMethod.__members__: raise ui.UserError( "Selected ReplayGain peak method {} is not supported. " "Please select one of: {}".format( peak_method, - ', '.join(ALL_PEAK_METHODS) + ', '.join(PeakMethod.__members__) ) ) - - # The key in these dicts is the `use_r128` flag. - self.peak_methods = { - True: Peak.none, - False: Peak[peak_method] - } + # This only applies to plain old rg tags, r128 doesn't store peak + # values. + self.peak_method = PeakMethod[peak_method] # On-import analysis. if self.config['auto']: @@ -1197,7 +1205,6 @@ class ReplayGainPlugin(BeetsPlugin): return R128Task( items, album, self.config["r128_targetlevel"].as_number(), - Peak.none, # R128_* tags do not store the track/album peak self.backend_instance.NAME, self._log, ) @@ -1205,7 +1212,7 @@ class ReplayGainPlugin(BeetsPlugin): return RgTask( items, album, self.config["targetlevel"].as_number(), - self.peak_methods[use_r128], + self.peak_method, self.backend_instance.NAME, self._log, ) From 8572e9bcecc5681a9ff955cfae5c8d1ed552fc2a Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Wed, 15 Sep 2021 08:09:17 +0200 Subject: [PATCH 126/357] replaygain: add docstrings --- beetsplug/replaygain.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index bbfcd1f2a..78f146a82 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -102,6 +102,15 @@ class PeakMethod(enum.Enum): class RgTask(): + """State and methods for a single replaygain calculation (rg version). + + Bundles the state (parameters and results) of a single replaygain + calculation (either for one item, one disk, or one full album). + + This class provides methods to store the resulting gains and peaks as plain + old rg tags. + """ + def __init__(self, items, album, target_level, peak_method, backend_name, log): self.items = items @@ -114,6 +123,8 @@ class RgTask(): self.track_gains = None def _store_track_gain(self, item, track_gain): + """Store track gain for a single item in the database. + """ item.rg_track_gain = track_gain.gain item.rg_track_peak = track_gain.peak item.store() @@ -121,7 +132,7 @@ class RgTask(): item.rg_track_gain, item.rg_track_peak) def _store_album_gain(self, item): - """ + """Store album gain for a single item in the database. The caller needs to ensure that `self.album_gain is not None`. """ @@ -132,6 +143,8 @@ class RgTask(): item.rg_album_gain, item.rg_album_peak) def _store_track(self, write): + """Store track gain for the first track of the task in the database. + """ item = self.items[0] if self.track_gains is None or len(self.track_gains) != 1: # In some cases, backends fail to produce a valid @@ -148,6 +161,8 @@ class RgTask(): self._log.debug('done analyzing {0}', item) def _store_album(self, write): + """Store track/album gains for all tracks of the task in the database. + """ if (self.album_gain is None or self.track_gains is None or len(self.track_gains) != len(self.items)): # In some cases, backends fail to produce a valid @@ -166,6 +181,8 @@ class RgTask(): self._log.debug('done analyzing {0}', item) def store(self, write): + """Store computed gains for the items of this task in the database. + """ if self.album is not None: self._store_album(write) else: @@ -173,6 +190,15 @@ class RgTask(): class R128Task(RgTask): + """State and methods for a single replaygain calculation (r128 version). + + Bundles the state (parameters and results) of a single replaygain + calculation (either for one item, one disk, or one full album). + + This class provides methods to store the resulting gains and peaks as R128 + tags. + """ + def __init__(self, items, album, target_level, backend_name, log): # R128_* tags do not store the track/album peak super().__init__(items, album, target_level, None, backend_name, From 6ddf2fa006267848304a59fc77096394ad69ed17 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 22 Jan 2022 13:58:58 +0100 Subject: [PATCH 127/357] replaygain: update changelog for 3eb49fca --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4df82e5c..e803b5bfa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,9 @@ Bug fixes: * :doc:`plugins/lyrics`: Fixed issues with the Tekstowo.pl and Genius backends where some non-lyrics content got included in the lyrics * :doc:`plugins/limit`: Better header formatting to improve index +* :doc:`plugins/replaygain`: Correctly handle the ``overwrite`` config option, + which forces recomputing ReplayGain values on import even for tracks + that already have the tags. For packagers: From 6457532274aeb51024cd9964bbfd2ca2a31dafc8 Mon Sep 17 00:00:00 2001 From: Rob Crowell Date: Wed, 19 Jan 2022 18:32:57 -0800 Subject: [PATCH 128/357] Add query prefixes :~ and := --- beets/dbcore/query.py | 17 +++++++++++++ beets/library.py | 6 ++++- docs/changelog.rst | 1 + docs/reference/query.rst | 37 +++++++++++++++++++++++++--- test/test_query.py | 52 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 96476a5b1..c020deacb 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -177,6 +177,23 @@ class StringFieldQuery(FieldQuery): raise NotImplementedError() +class StringQuery(StringFieldQuery): + """A query that matches a whole string in a specific item field.""" + + def col_clause(self): + search = (self.pattern + .replace('\\', '\\\\') + .replace('%', '\\%') + .replace('_', '\\_')) + clause = self.field + " like ? escape '\\'" + subvals = [search] + return clause, subvals + + @classmethod + def string_match(cls, pattern, value): + return pattern == value + + class SubstringQuery(StringFieldQuery): """A query that matches a substring in a specific item field.""" diff --git a/beets/library.py b/beets/library.py index c8993f85b..69fcd34cf 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1385,7 +1385,11 @@ def parse_query_parts(parts, model_cls): special path query detection. """ # Get query types and their prefix characters. - prefixes = {':': dbcore.query.RegexpQuery} + prefixes = { + ':': dbcore.query.RegexpQuery, + '~': dbcore.query.StringQuery, + '=': dbcore.query.MatchQuery, + } prefixes.update(plugins.queries()) # Special-case path-like queries, which are non-field queries diff --git a/docs/changelog.rst b/docs/changelog.rst index f4df82e5c..3b68e4eb6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ New features: * :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances :bug:`4101` * Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. +* Add query prefixes ``=`` and ``~``. Bug fixes: diff --git a/docs/reference/query.rst b/docs/reference/query.rst index 5c16f610b..75fac3015 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -93,14 +93,45 @@ backslashes are not part of beets' syntax; I'm just using the escaping functionality of my shell (bash or zsh, for instance) to pass ``the rebel`` as a single argument instead of two. +Exact Matches +------------- + +While ordinary queries perform *substring* matches, beets can also match whole +strings by adding either ``=`` (case-sensitive) or ``~`` (ignore case) after the +field name's colon and before the expression:: + + $ beet list artist:air + $ beet list artist:~air + $ beet list artist:=AIR + +The first query is a simple substring one that returns tracks by Air, AIR, and +Air Supply. The second query returns tracks by Air and AIR, since both are a +case-insensitive match for the entire expression, but does not return anything +by Air Supply. The third query, which requires a case-sensitive exact match, +returns tracks by AIR only. + +Exact matches may be performed on phrases as well:: + + $ beet list artist:~"dave matthews" + $ beet list artist:="Dave Matthews" + +Both of these queries return tracks by Dave Matthews, but not by Dave Matthews +Band. + +To search for exact matches across *all* fields, just prefix the expression with +a single ``=`` or ``~``:: + + $ beet list ~crash + $ beet list ="American Football" + .. _regex: Regular Expressions ------------------- -While ordinary keywords perform simple substring matches, beets also supports -regular expression matching for more advanced queries. To run a regex query, use -an additional ``:`` between the field name and the expression:: +In addition to simple substring and exact matches, beets also supports regular +expression matching for more advanced queries. To run a regex query, use an +additional ``:`` between the field name and the expression:: $ beet list "artist::Ann(a|ie)" diff --git a/test/test_query.py b/test/test_query.py index 14f3f082a..0b857ef7c 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -94,16 +94,19 @@ class DummyDataTestCase(_common.TestCase, AssertsMixin): items[0].album = 'baz' items[0].year = 2001 items[0].comp = True + items[0].genre = 'rock' items[1].title = 'baz qux' items[1].artist = 'two' items[1].album = 'baz' items[1].year = 2002 items[1].comp = True + items[1].genre = 'Rock' items[2].title = 'beets 4 eva' items[2].artist = 'three' items[2].album = 'foo' items[2].year = 2003 items[2].comp = False + items[2].genre = 'Hard Rock' for item in items: self.lib.add(item) self.album = self.lib.add_album(items[:2]) @@ -132,6 +135,22 @@ class GetTest(DummyDataTestCase): results = self.lib.items(q) self.assert_items_matched(results, ['baz qux']) + def test_get_one_keyed_exact(self): + q = 'genre:=rock' + results = self.lib.items(q) + self.assert_items_matched(results, ['foo bar']) + q = 'genre:=Rock' + results = self.lib.items(q) + self.assert_items_matched(results, ['baz qux']) + q = 'genre:="Hard Rock"' + results = self.lib.items(q) + self.assert_items_matched(results, ['beets 4 eva']) + + def test_get_one_keyed_exact_nocase(self): + q = 'genre:~"hard rock"' + results = self.lib.items(q) + self.assert_items_matched(results, ['beets 4 eva']) + def test_get_one_keyed_regexp(self): q = 'artist::t.+r' results = self.lib.items(q) @@ -142,6 +161,16 @@ class GetTest(DummyDataTestCase): results = self.lib.items(q) self.assert_items_matched(results, ['beets 4 eva']) + def test_get_one_unkeyed_exact(self): + q = '=rock' + results = self.lib.items(q) + self.assert_items_matched(results, ['foo bar']) + + def test_get_one_unkeyed_exact_nocase(self): + q = '~"hard rock"' + results = self.lib.items(q) + self.assert_items_matched(results, ['beets 4 eva']) + def test_get_one_unkeyed_regexp(self): q = ':x$' results = self.lib.items(q) @@ -159,6 +188,11 @@ class GetTest(DummyDataTestCase): # objects. self.assert_items_matched(results, []) + def test_get_no_matches_exact(self): + q = 'genre:="hard rock"' + results = self.lib.items(q) + self.assert_items_matched(results, []) + def test_term_case_insensitive(self): q = 'oNE' results = self.lib.items(q) @@ -182,6 +216,14 @@ class GetTest(DummyDataTestCase): results = self.lib.items(q) self.assert_items_matched(results, ['beets 4 eva']) + def test_keyed_matches_exact_nocase(self): + q = 'genre:~rock' + results = self.lib.items(q) + self.assert_items_matched(results, [ + 'foo bar', + 'baz qux', + ]) + def test_unkeyed_term_matches_multiple_columns(self): q = 'baz' results = self.lib.items(q) @@ -350,6 +392,16 @@ class MatchTest(_common.TestCase): q = dbcore.query.SubstringQuery('disc', '6') self.assertTrue(q.match(self.item)) + def test_exact_match_nocase_positive(self): + q = dbcore.query.StringQuery('genre', 'the genre') + self.assertTrue(q.match(self.item)) + + def test_exact_match_nocase_negative(self): + q = dbcore.query.StringQuery('genre', 'genre') + self.assertFalse(q.match(self.item)) + q = dbcore.query.StringQuery('genre', 'THE GENRE') + self.assertFalse(q.match(self.item)) + def test_year_match_positive(self): q = dbcore.query.NumericQuery('year', '1') self.assertTrue(q.match(self.item)) From 2cab2d670aa011006f4322a59176ba3dbb6bb22b Mon Sep 17 00:00:00 2001 From: Rob Crowell Date: Tue, 25 Jan 2022 16:13:05 -0800 Subject: [PATCH 129/357] Fix bug in StringQuery.string_match --- beets/dbcore/query.py | 2 +- test/test_query.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index c020deacb..b0c769790 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -191,7 +191,7 @@ class StringQuery(StringFieldQuery): @classmethod def string_match(cls, pattern, value): - return pattern == value + return pattern.lower() == value.lower() class SubstringQuery(StringFieldQuery): diff --git a/test/test_query.py b/test/test_query.py index 0b857ef7c..0be4b7d7f 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -395,12 +395,12 @@ class MatchTest(_common.TestCase): def test_exact_match_nocase_positive(self): q = dbcore.query.StringQuery('genre', 'the genre') self.assertTrue(q.match(self.item)) + q = dbcore.query.StringQuery('genre', 'THE GENRE') + self.assertTrue(q.match(self.item)) def test_exact_match_nocase_negative(self): q = dbcore.query.StringQuery('genre', 'genre') self.assertFalse(q.match(self.item)) - q = dbcore.query.StringQuery('genre', 'THE GENRE') - self.assertFalse(q.match(self.item)) def test_year_match_positive(self): q = dbcore.query.NumericQuery('year', '1') From 088cdfe995c14927bfa277edd9663c0cb79bad78 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 29 Jan 2022 18:31:48 -0500 Subject: [PATCH 130/357] Revert some of #4226 Rectify a couple of things in that PR, pointed out here: https://github.com/beetbox/beets/pull/4226#issuecomment-1011499620 - Undo the `pretend` sensitivity in the import path, because it's not clear how this setting could ever be true. - Preserve the log message in debug mode, even when quiet. --- beetsplug/convert.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 82e62af62..63bfea9b3 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -22,6 +22,7 @@ import subprocess import tempfile import shlex from string import Template +import logging from beets import ui, util, plugins, config from beets.plugins import BeetsPlugin @@ -514,23 +515,21 @@ class ConvertPlugin(BeetsPlugin): except subprocess.CalledProcessError: return - pretend = self.config['pretend'].get(bool) - quiet = self.config['quiet'].get(bool) + # Change the newly-imported database entry to point to the + # converted file. + source_path = item.path + item.path = dest + item.write() + item.read() # Load new audio information data. + item.store() - if not pretend: - # Change the newly-imported database entry to point to the - # converted file. - source_path = item.path - item.path = dest - item.write() - item.read() # Load new audio information data. - item.store() - - if self.config['delete_originals']: - if not quiet: - self._log.info('Removing original file {0}', - source_path) - util.remove(source_path, False) + if self.config['delete_originals']: + self._log.log( + logging.DEBUG if self.config['quiet'] else logging.INFO, + 'Removing original file {0}', + source_path, + ) + util.remove(source_path, False) def _cleanup(self, task, session): for path in task.old_paths: From 0788197c7665387774d62570ec7367215054cc43 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 29 Jan 2022 18:33:10 -0500 Subject: [PATCH 131/357] Remove a relevant changelog entry --- docs/changelog.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6d631ebbd..a28a992b9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,8 +32,6 @@ Bug fixes: * Fix a regression in the previous release that caused a `TypeError` when moving files across filesystems. :bug:`4168` -* :doc:`/plugins/convert`: Files are no longer converted when running import in - ``--pretend`` mode. * :doc:`/plugins/convert`: Deleting the original files during conversion no longer logs output when the ``quiet`` flag is enabled. * :doc:`plugins/web`: Fix handling of "query" requests. Previously queries From 2b51b2443b68feafb12511bca2d7611db555922b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 29 Jan 2022 18:35:36 -0500 Subject: [PATCH 132/357] Remove a relevant test --- test/test_convert.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/test_convert.py b/test/test_convert.py index 493d4ecca..cd32e34b1 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -127,16 +127,6 @@ class ImportConvertTest(unittest.TestCase, TestHelper): 'Non-empty import directory {}' .format(util.displayable_path(path))) - def test_delete_originals_keeps_originals_when_pretend_enabled(self): - import_file_count = self.get_count_of_import_files() - - self.config['convert']['delete_originals'] = True - self.config['convert']['pretend'] = True - self.importer.run() - - self.assertEqual(self.get_count_of_import_files(), import_file_count, - 'Count of files differs after running import') - def get_count_of_import_files(self): import_file_count = 0 From a903e6649a6c0cdc514ab27040656a1a3b996e75 Mon Sep 17 00:00:00 2001 From: Michael Klein Date: Sun, 30 Jan 2022 18:10:58 +0100 Subject: [PATCH 133/357] zsh completion: handle destination for "convert" Use directory name completion for -d/--dest option. --- extra/_beet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/_beet b/extra/_beet index 129c0485e..ca5389632 100644 --- a/extra/_beet +++ b/extra/_beet @@ -151,7 +151,7 @@ _beet_subcmd_options() { libfile=("$matchany" ':file:database file:{_files -g *.db}') regex_words+=("$opt:$optdesc:\$libfile") ;; - (DIR|DIRECTORY) + (DIR|DIRECTORY|DEST) local -a dirs dirs=("$matchany" ':dir:directory:_dirs') regex_words+=("$opt:$optdesc:\$dirs") From c86d5d0264165a2329506c2a670cde8ec16e5658 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 31 Jan 2022 20:29:50 +0100 Subject: [PATCH 134/357] lyrics: Rework lyrics assertions for more useful output on failure Uses a custom assertion to have more detailed output (which should help with getting an idea what the difference between expected and actual lyrics is), and use `subTest` to clearly associate test failure to a backend. In addition, this strip parenthesis from the lyrics' words. That helps with the currently failing Tekstowo test, but doesn't entirely fix it. Requires Python 3.4 for subTest. --- test/test_lyrics.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 019cf3d88..3adf6e359 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -231,13 +231,24 @@ class MockFetchUrl: return content -def is_lyrics_content_ok(title, text): - """Compare lyrics text to expected lyrics for given title.""" - if not text: - return - keywords = set(LYRICS_TEXTS[google.slugify(title)].split()) - words = {x.strip(".?, ") for x in text.lower().split()} - return keywords <= words +class LyricsAssertions: + """A mixin with lyrics-specific assertions.""" + + def assertLyricsContentOk(self, title, text, msg=""): # noqa: N802 + """Compare lyrics text to expected lyrics for given title.""" + if not text: + return + + keywords = set(LYRICS_TEXTS[google.slugify(title)].split()) + words = {x.strip(".?, ()") for x in text.lower().split()} + + if not keywords <= words: + details = ( + f"{keywords!r} is not a subset of {words!r}." + f" Words only in first {keywords - words!r}," + f" Words only in second {words - keywords!r}." + ) + self.fail(f"{details} : {msg}") LYRICS_ROOT_DIR = os.path.join(_common.RSRC, b'lyrics') @@ -255,7 +266,7 @@ class LyricsGoogleBaseTest(unittest.TestCase): self.skipTest('Beautiful Soup 4 not available') -class LyricsPluginSourcesTest(LyricsGoogleBaseTest): +class LyricsPluginSourcesTest(LyricsGoogleBaseTest, LyricsAssertions): """Check that beets google custom search engine sources are correctly scraped. """ @@ -327,15 +338,13 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): def test_backend_sources_ok(self): """Test default backends with songs known to exist in respective databases. """ - errors = [] # Don't test any sources marked as skipped. sources = [s for s in self.DEFAULT_SOURCES if not s.get("skip", False)] for s in sources: - res = s['backend'](self.plugin.config, self.plugin._log).fetch( - s['artist'], s['title']) - if not is_lyrics_content_ok(s['title'], res): - errors.append(s['backend'].__name__) - self.assertFalse(errors) + with self.subTest(s['backend'].__name__): + backend = s['backend'](self.plugin.config, self.plugin._log) + res = backend.fetch(s['artist'], s['title']) + self.assertLyricsContentOk(s['title'], res) @unittest.skipUnless( os.environ.get('INTEGRATION_TEST', '0') == '1', @@ -351,10 +360,10 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): res = lyrics.scrape_lyrics_from_html( raw_backend.fetch_url(url)) self.assertTrue(google.is_lyrics(res), url) - self.assertTrue(is_lyrics_content_ok(s['title'], res), url) + self.assertLyricsContentOk(s['title'], res, url) -class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): +class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest, LyricsAssertions): """Test scraping heuristics on a fake html page. """ @@ -372,8 +381,7 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): url = self.source['url'] + self.source['path'] res = lyrics.scrape_lyrics_from_html(raw_backend.fetch_url(url)) self.assertTrue(google.is_lyrics(res), url) - self.assertTrue(is_lyrics_content_ok(self.source['title'], res), - url) + self.assertLyricsContentOk(self.source['title'], res, url) @patch.object(lyrics.Backend, 'fetch_url', MockFetchUrl()) def test_is_page_candidate_exact_match(self): From 0cc452137205a1e500e81c575595b7beda82d634 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 31 Jan 2022 20:31:57 +0100 Subject: [PATCH 135/357] tests: allow passing the INTEGRATION_TEST variable from the commandline this allows to run INTEGRATION_TEST=1 tox -e test/test_.py which is very useful when testing locally. Otherwise, tox will clean the environment, such that INTEGRATION_TEST will not be passed to the test runner. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 5f9de07f6..95f250f96 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ files = beets beetsplug beet test setup.py docs deps = {test,cov}: {[_test]deps} lint: {[_lint]deps} +passenv = INTEGRATION_TEST commands = test: python -bb -m pytest -rs {posargs} cov: coverage run -m pytest -rs {posargs} From cc8c3529fbf528da8eadadf2ff61389db9adf767 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 31 Jan 2022 21:26:32 +0100 Subject: [PATCH 136/357] confit: Improve deprecation warning Show the actual origin of the import statement, cf. #4024 --- beets/util/confit.py | 8 +++++++- test/test_util.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/beets/util/confit.py b/beets/util/confit.py index dd912c444..927a9f087 100644 --- a/beets/util/confit.py +++ b/beets/util/confit.py @@ -16,7 +16,13 @@ import confuse import warnings -warnings.warn("beets.util.confit is deprecated; use confuse instead") +warnings.warn( + "beets.util.confit is deprecated; use confuse instead", + # Show the location of the `import confit` statement as the warning's + # source, rather than this file, such that the offending module can be + # identified easily. + stacklevel=2, +) # Import everything from the confuse module into this module. for key, value in confuse.__dict__.items(): diff --git a/test/test_util.py b/test/test_util.py index 32614ab72..fcaf9f5ce 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -182,6 +182,21 @@ class PathTruncationTest(_common.TestCase): self.assertEqual(p, 'abcde/f.ext') +class ConfitDeprecationTest(_common.TestCase): + def test_confit_deprecattion_warning_origin(self): + """Test that importing `confit` raises a warning. + + In addition, ensure that the warning originates from the actual + import statement, not the `confit` module. + """ + # See https://github.com/beetbox/beets/discussions/4024 + with self.assertWarns(UserWarning) as w: + import beets.util.confit # noqa: F401 + + self.assertIn(__file__, w.filename) + self.assertNotIn("confit.py", w.filename) + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From ef59465869b350b3a11c7b0e40d550d65fffb8a5 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 1 Feb 2022 21:39:42 +0100 Subject: [PATCH 137/357] test_replaygain: fix exception handling again This is the same as d26b0bc1, which I reintroduced again in a new test due to a botched rebase... --- test/test_replaygain.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 499befee2..47e27b844 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -329,25 +329,13 @@ class ImportTest(TestHelper): self.setup_beets(disk=True) self.config['threaded'] = self.threaded - self.config['replaygain'] = { - 'backend': self.backend, - } + self.config['replaygain']['backend'] = self.backend try: self.load_plugins('replaygain') except Exception: - import sys - # store exception info so an error in teardown does not swallow it - exc_info = sys.exc_info() - try: - self.teardown_beets() - self.unload_plugins() - except Exception: - # if load_plugins() failed then setup is incomplete and - # teardown operations may fail. In particular # {Item,Album} - # may not have the _original_types attribute in unload_plugins - pass - raise None.with_traceback(exc_info[2]) + self.teardown_beets() + self.unload_plugins() self.importer = self.create_importer() From af5be3dbedf4fd66db3751648dd3f5c4a6dd443d Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sat, 5 Feb 2022 21:55:31 +1000 Subject: [PATCH 138/357] Update CI for latest nightly --- .github/workflows/ci.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb1d42ceb..4ae995b89 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11-a05'] env: PY_COLORS: 1 @@ -45,17 +45,17 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.9' && matrix.python-version != '3.10' + if: matrix.python-version != '3.10' && matrix.python-version != '3.11-a05' run: | tox -e py-test - name: Test latest Python version with tox and get coverage - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.10' run: | tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.11-a05' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 @@ -64,7 +64,7 @@ jobs: tox -e py-test - name: Upload code coverage - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.10' run: | pip install codecov || true codecov || true @@ -78,10 +78,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.10 - name: Install base dependencies run: | @@ -100,10 +100,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.10 - name: Install base dependencies run: | From ec010d49509a34e9d594c4f3062776360af17ec6 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sat, 5 Feb 2022 22:01:25 +1000 Subject: [PATCH 139/357] Naming and quoting --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4ae995b89..62cb36914 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11-a05'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11.0-alpha.5'] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.10' && matrix.python-version != '3.11-a05' + if: matrix.python-version != '3.10' && matrix.python-version != '3.11.0-alpha.5' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.11-a05' + if: matrix.python-version == '3.11.0-alpha.5' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 @@ -81,7 +81,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - name: Install base dependencies run: | @@ -103,7 +103,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: '3.10' - name: Install base dependencies run: | From b24ed6e78203d911900d41911a48dbb3328bde3d Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sun, 6 Feb 2022 08:44:55 +1000 Subject: [PATCH 140/357] Update tox as well --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 95f250f96..1c0a984ed 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ commands = lint: python -m flake8 {posargs} {[_lint]files} [testenv:docs] -basepython = python3.9 +basepython = python3.10 deps = sphinx<4.4.0 commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} From 21dbb0436004438c3970576906a4c40b992e81d6 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sun, 6 Feb 2022 08:49:56 +1000 Subject: [PATCH 141/357] Try just py3.11 --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 62cb36914..dc0454d89 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11.0-alpha.5'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.10' && matrix.python-version != '3.11.0-alpha.5' + if: matrix.python-version != '3.10' && matrix.python-version != '3.11' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.11.0-alpha.5' + if: matrix.python-version == '3.11' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 From 4a9771430e9aea2659b9249387a08b7252d2b0cb Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sun, 6 Feb 2022 09:41:23 +1000 Subject: [PATCH 142/357] Try a subversion of 3.11 --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc0454d89..07ac5f51e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11.0'] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.10' && matrix.python-version != '3.11' + if: matrix.python-version != '3.10' && matrix.python-version != '3.11.0' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.11' + if: matrix.python-version == '3.11.0' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 From 07eb26f276d5f55b6a9cef05f8130f1e6cb5458d Mon Sep 17 00:00:00 2001 From: Adam Fontenot Date: Tue, 8 Feb 2022 16:39:46 -0800 Subject: [PATCH 143/357] Resize album art when embedding (convert plugin) Fixes #2116 --- beetsplug/convert.py | 35 ++++++++++++++++++++++------------- docs/changelog.rst | 2 ++ docs/plugins/convert.rst | 3 ++- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 63bfea9b3..f384656f1 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -343,9 +343,10 @@ class ConvertPlugin(BeetsPlugin): if self.config['embed'] and not linked: album = item._cached_album if album and album.artpath: + maxwidth = self._get_art_resize(album.artpath) self._log.debug('embedding album art from {}', util.displayable_path(album.artpath)) - art.embed_item(self._log, item, album.artpath, + art.embed_item(self._log, item, album.artpath, maxwidth, itempath=converted, id3v23=id3v23) if keep_new: @@ -389,20 +390,10 @@ class ConvertPlugin(BeetsPlugin): return # Decide whether we need to resize the cover-art image. - resize = False - maxwidth = None - if self.config['album_art_maxwidth']: - maxwidth = self.config['album_art_maxwidth'].get(int) - size = ArtResizer.shared.get_size(album.artpath) - self._log.debug('image size: {}', size) - if size: - resize = size[0] > maxwidth - else: - self._log.warning('Could not get size of image (please see ' - 'documentation for dependencies).') + maxwidth = self._get_art_resize(album.artpath) # Either copy or resize (while copying) the image. - if resize: + if maxwidth is not None: self._log.info('Resizing cover art from {0} to {1}', util.displayable_path(album.artpath), util.displayable_path(dest)) @@ -531,6 +522,24 @@ class ConvertPlugin(BeetsPlugin): ) util.remove(source_path, False) + def _get_art_resize(self, artpath): + """For a given piece of album art, determine whether or not it needs + to be resized according to the user's settings. If so, returns the + new size. If not, returns None. + """ + newwidth = None + if self.config['album_art_maxwidth']: + maxwidth = self.config['album_art_maxwidth'].get(int) + size = ArtResizer.shared.get_size(artpath) + self._log.debug('image size: {}', size) + if size: + if size[0] > maxwidth: + newwidth = maxwidth + else: + self._log.warning('Could not get size of image (please see ' + 'documentation for dependencies).') + return newwidth + def _cleanup(self, task, session): for path in task.old_paths: if path in _temp_files: diff --git a/docs/changelog.rst b/docs/changelog.rst index a28a992b9..2878c7953 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,8 @@ New features: Bug fixes: +* :doc:`/plugins/convert`: Resize album art when embedding + :bug:`2116` * :doc:`/plugins/deezer`: Fix auto tagger pagination issues (fetch beyond the first 25 tracks of a release). * :doc:`/plugins/spotify`: Fix auto tagger pagination issues (fetch beyond the diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index d53b8dc6d..799a20dbb 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -72,7 +72,8 @@ file. The available options are: using the ``-a`` option. Default: ``no``. - **album_art_maxwidth**: Downscale album art if it's too big. The resize operation reduces image width to at most ``maxwidth`` pixels while - preserving the aspect ratio. + preserving the aspect ratio. The specified image size will apply to both + embedded album art and external image files. - **dest**: The directory where the files will be converted (or copied) to. Default: none. - **embed**: Embed album art in converted items. Default: ``yes``. From 497b91606241c70e2e7b9f462f7ab07ee8aeb67c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 17:35:18 +0100 Subject: [PATCH 144/357] artresizer: import *path helpers directly shortens lines a bit, and should pose no problem for understanding the code given the prevalence of these functions in our code --- beets/util/artresizer.py | 51 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 8683e2287..a5ccaa4e9 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -24,6 +24,7 @@ from tempfile import NamedTemporaryFile from urllib.parse import urlencode from beets import logging from beets import util +from beets.util import bytestring_path, displayable_path, py3_path, syspath # Resizing methods PIL = 1 @@ -55,8 +56,8 @@ def temp_file_for(path): specified path. """ ext = os.path.splitext(path)[1] - with NamedTemporaryFile(suffix=util.py3_path(ext), delete=False) as f: - return util.bytestring_path(f.name) + with NamedTemporaryFile(suffix=py3_path(ext), delete=False) as f: + return bytestring_path(f.name) def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): @@ -67,10 +68,10 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): from PIL import Image log.debug('artresizer: PIL resizing {0} to {1}', - util.displayable_path(path_in), util.displayable_path(path_out)) + displayable_path(path_in), displayable_path(path_out)) try: - im = Image.open(util.syspath(path_in)) + im = Image.open(syspath(path_in)) size = maxwidth, maxwidth im.thumbnail(size, Image.ANTIALIAS) @@ -80,7 +81,7 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # progressive=False only affects JPEGs and is the default, # but we include it here for explicitness. - im.save(util.py3_path(path_out), quality=quality, progressive=False) + im.save(py3_path(path_out), quality=quality, progressive=False) if max_filesize > 0: # If maximum filesize is set, we attempt to lower the quality of @@ -92,7 +93,7 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): lower_qual = 95 for i in range(5): # 5 attempts is an abitrary choice - filesize = os.stat(util.syspath(path_out)).st_size + filesize = os.stat(syspath(path_out)).st_size log.debug("PIL Pass {0} : Output size: {1}B", i, filesize) if filesize <= max_filesize: return path_out @@ -103,7 +104,7 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): if lower_qual < 10: lower_qual = 10 # Use optimize flag to improve filesize decrease - im.save(util.py3_path(path_out), quality=lower_qual, + im.save(py3_path(path_out), quality=lower_qual, optimize=True, progressive=False) log.warning("PIL Failed to resize file to below {0}B", max_filesize) @@ -113,7 +114,7 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): return path_out except OSError: log.error("PIL cannot create thumbnail for '{0}'", - util.displayable_path(path_in)) + displayable_path(path_in)) return path_in @@ -125,7 +126,7 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """ path_out = path_out or temp_file_for(path_in) log.debug('artresizer: ImageMagick resizing {0} to {1}', - util.displayable_path(path_in), util.displayable_path(path_out)) + displayable_path(path_in), displayable_path(path_out)) # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio @@ -133,7 +134,7 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # ImageMagick already seems to default to no interlace, but we include it # here for the sake of explicitness. cmd = ArtResizer.shared.im_convert_cmd + [ - util.syspath(path_in, prefix=False), + syspath(path_in, prefix=False), '-resize', f'{maxwidth}x>', '-interlace', 'none', ] @@ -146,13 +147,13 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): if max_filesize > 0: cmd += ['-define', f'jpeg:extent={max_filesize}b'] - cmd.append(util.syspath(path_out, prefix=False)) + cmd.append(syspath(path_out, prefix=False)) try: util.command_output(cmd) except subprocess.CalledProcessError: log.warning('artresizer: IM convert failed for {0}', - util.displayable_path(path_in)) + displayable_path(path_in)) return path_in return path_out @@ -168,16 +169,16 @@ def pil_getsize(path_in): from PIL import Image try: - im = Image.open(util.syspath(path_in)) + im = Image.open(syspath(path_in)) return im.size except OSError as exc: log.error("PIL could not read file {}: {}", - util.displayable_path(path_in), exc) + displayable_path(path_in), exc) def im_getsize(path_in): cmd = ArtResizer.shared.im_identify_cmd + \ - ['-format', '%w %h', util.syspath(path_in, prefix=False)] + ['-format', '%w %h', syspath(path_in, prefix=False)] try: out = util.command_output(cmd).stdout @@ -206,8 +207,8 @@ def pil_deinterlace(path_in, path_out=None): from PIL import Image try: - im = Image.open(util.syspath(path_in)) - im.save(util.py3_path(path_out), progressive=False) + im = Image.open(syspath(path_in)) + im.save(py3_path(path_out), progressive=False) return path_out except IOError: return path_in @@ -217,9 +218,9 @@ def im_deinterlace(path_in, path_out=None): path_out = path_out or temp_file_for(path_in) cmd = ArtResizer.shared.im_convert_cmd + [ - util.syspath(path_in, prefix=False), + syspath(path_in, prefix=False), '-interlace', 'none', - util.syspath(path_out, prefix=False), + syspath(path_out, prefix=False), ] try: @@ -238,7 +239,7 @@ DEINTERLACE_FUNCS = { def im_get_format(filepath): cmd = ArtResizer.shared.im_identify_cmd + [ '-format', '%[magick]', - util.syspath(filepath) + syspath(filepath) ] try: @@ -251,7 +252,7 @@ def pil_get_format(filepath): from PIL import Image, UnidentifiedImageError try: - with Image.open(util.syspath(filepath)) as im: + with Image.open(syspath(filepath)) as im: return im.format except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError): log.exception("failed to detect image format for {}", filepath) @@ -266,9 +267,9 @@ BACKEND_GET_FORMAT = { def im_convert_format(source, target, deinterlaced): cmd = ArtResizer.shared.im_convert_cmd + [ - util.syspath(source), + syspath(source), *(["-interlace", "none"] if deinterlaced else []), - util.syspath(target), + syspath(target), ] try: @@ -286,8 +287,8 @@ def pil_convert_format(source, target, deinterlaced): from PIL import Image, UnidentifiedImageError try: - with Image.open(util.syspath(source)) as im: - im.save(util.py3_path(target), progressive=not deinterlaced) + with Image.open(syspath(source)) as im: + im.save(py3_path(target), progressive=not deinterlaced) return target except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError, OSError): From 890662f93d80bfbb9a226f7c1a967dee4f4b2a5b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 17:41:47 +0100 Subject: [PATCH 145/357] artresizer: don't manually cache can_compare it's only computed once on startup anyway (see the embedart plugin, which is the only user) --- beets/util/artresizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index a5ccaa4e9..b366df330 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -329,7 +329,6 @@ class ArtResizer(metaclass=Shareable): """ self.method = self._check_method() log.debug("artresizer: method is {0}", self.method) - self.can_compare = self._can_compare() # Use ImageMagick's magick binary when it's available. If it's # not, fall back to the older, separate convert and identify @@ -430,7 +429,8 @@ class ArtResizer(metaclass=Shareable): os.unlink(path_in) return result_path - def _can_compare(self): + @property + def can_compare(self): """A boolean indicating whether image comparison is available""" return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7) From a17a5f2fa6e7fddd9df51219176207f539435163 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 17:54:38 +0100 Subject: [PATCH 146/357] art: move the backend-specific code to util.artresizer all the other backend-specific code is already there. --- beets/art.py | 68 ++------------------------------ beets/util/artresizer.py | 83 ++++++++++++++++++++++++++++++++++++++++ test/test_embedart.py | 2 +- 3 files changed, 87 insertions(+), 66 deletions(-) diff --git a/beets/art.py b/beets/art.py index 13d5dfbd4..1dff8b39a 100644 --- a/beets/art.py +++ b/beets/art.py @@ -17,8 +17,6 @@ music and items' embedded album art. """ -import subprocess -import platform from tempfile import NamedTemporaryFile import os @@ -121,70 +119,10 @@ def check_art_similarity(log, item, imagepath, compare_threshold): with NamedTemporaryFile(delete=True) as f: art = extract(log, f.name, item) - if art: - is_windows = platform.system() == "Windows" + if not art: + return True - # Converting images to grayscale tends to minimize the weight - # of colors in the diff score. So we first convert both images - # to grayscale and then pipe them into the `compare` command. - # On Windows, ImageMagick doesn't support the magic \\?\ prefix - # on paths, so we pass `prefix=False` to `syspath`. - convert_cmd = ['convert', syspath(imagepath, prefix=False), - syspath(art, prefix=False), - '-colorspace', 'gray', 'MIFF:-'] - compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:'] - log.debug('comparing images with pipeline {} | {}', - convert_cmd, compare_cmd) - convert_proc = subprocess.Popen( - convert_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=not is_windows, - ) - compare_proc = subprocess.Popen( - compare_cmd, - stdin=convert_proc.stdout, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=not is_windows, - ) - - # Check the convert output. We're not interested in the - # standard output; that gets piped to the next stage. - convert_proc.stdout.close() - convert_stderr = convert_proc.stderr.read() - convert_proc.stderr.close() - convert_proc.wait() - if convert_proc.returncode: - log.debug( - 'ImageMagick convert failed with status {}: {!r}', - convert_proc.returncode, - convert_stderr, - ) - return - - # Check the compare output. - stdout, stderr = compare_proc.communicate() - if compare_proc.returncode: - if compare_proc.returncode != 1: - log.debug('ImageMagick compare failed: {0}, {1}', - displayable_path(imagepath), - displayable_path(art)) - return - out_str = stderr - else: - out_str = stdout - - try: - phash_diff = float(out_str) - except ValueError: - log.debug('IM output is not a number: {0!r}', out_str) - return - - log.debug('ImageMagick compare score: {0}', phash_diff) - return phash_diff <= compare_threshold - - return True + return ArtResizer.shared.compare(art, imagepath, compare_threshold) def extract(log, outpath, item): diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index b366df330..552f21140 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -19,6 +19,7 @@ public resizing proxy if neither is available. import subprocess import os import os.path +import platform import re from tempfile import NamedTemporaryFile from urllib.parse import urlencode @@ -301,6 +302,79 @@ BACKEND_CONVERT_IMAGE_FORMAT = { IMAGEMAGICK: im_convert_format, } +def im_compare(im1, im2, compare_threshold): + is_windows = platform.system() == "Windows" + + # Converting images to grayscale tends to minimize the weight + # of colors in the diff score. So we first convert both images + # to grayscale and then pipe them into the `compare` command. + # On Windows, ImageMagick doesn't support the magic \\?\ prefix + # on paths, so we pass `prefix=False` to `syspath`. + convert_cmd = ['convert', syspath(im2, prefix=False), + syspath(im1, prefix=False), + '-colorspace', 'gray', 'MIFF:-'] + compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:'] + log.debug('comparing images with pipeline {} | {}', + convert_cmd, compare_cmd) + convert_proc = subprocess.Popen( + convert_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=not is_windows, + ) + compare_proc = subprocess.Popen( + compare_cmd, + stdin=convert_proc.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=not is_windows, + ) + + # Check the convert output. We're not interested in the + # standard output; that gets piped to the next stage. + convert_proc.stdout.close() + convert_stderr = convert_proc.stderr.read() + convert_proc.stderr.close() + convert_proc.wait() + if convert_proc.returncode: + log.debug( + 'ImageMagick convert failed with status {}: {!r}', + convert_proc.returncode, + convert_stderr, + ) + return + + # Check the compare output. + stdout, stderr = compare_proc.communicate() + if compare_proc.returncode: + if compare_proc.returncode != 1: + log.debug('ImageMagick compare failed: {0}, {1}', + displayable_path(im2), displayable_path(im1)) + return + out_str = stderr + else: + out_str = stdout + + try: + phash_diff = float(out_str) + except ValueError: + log.debug('IM output is not a number: {0!r}', out_str) + return + + log.debug('ImageMagick compare score: {0}', phash_diff) + return phash_diff <= compare_threshold + + +def pil_compare(im1, im2, compare_threshold): + # It is an error to call this when ArtResizer.can_compare is not True. + raise NotImplementedError() + + +BACKEND_COMPARE = { + PIL: pil_compare, + IMAGEMAGICK: im_compare, +} + class Shareable(type): """A pseudo-singleton metaclass that allows both shared and @@ -435,6 +509,15 @@ class ArtResizer(metaclass=Shareable): return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7) + def compare(self, im1, im2, compare_threshold): + """Return a boolean indicating whether two images are similar. + + Only available locally. + """ + if self.local: + func = BACKEND_COMPARE[self.method[0]] + return func(im1, im2, compare_threshold) + @staticmethod def _check_method(): """Return a tuple indicating an available method and its version. diff --git a/test/test_embedart.py b/test/test_embedart.py index 6b6d61614..b02989175 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -216,7 +216,7 @@ class EmbedartCliTest(_common.TestCase, TestHelper): self.assertEqual(mediafile.images[0].data, self.image_data) -@patch('beets.art.subprocess') +@patch('beets.util.artresizer.subprocess') @patch('beets.art.extract') class ArtSimilarityTest(unittest.TestCase): def setUp(self): From 8ca19de6bccd4567ec5e27755bf57a3cd2c3e425 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 17:57:51 +0100 Subject: [PATCH 147/357] artresizer: switch ImageMagick compare commmand depending on IM version similar to other commands Fixes #4272 --- beets/util/artresizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 552f21140..f076b3ab5 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -310,10 +310,13 @@ def im_compare(im1, im2, compare_threshold): # to grayscale and then pipe them into the `compare` command. # On Windows, ImageMagick doesn't support the magic \\?\ prefix # on paths, so we pass `prefix=False` to `syspath`. - convert_cmd = ['convert', syspath(im2, prefix=False), - syspath(im1, prefix=False), - '-colorspace', 'gray', 'MIFF:-'] - compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:'] + convert_cmd = ArtResizer.shared.im_convert_cmd + [ + syspath(im2, prefix=False), syspath(im1, prefix=False), + '-colorspace', 'gray', 'MIFF:-' + ] + compare_cmd = ArtResizer.shared.im_compare_cmd + [ + '-metric', 'PHASH', '-', 'null:', + ] log.debug('comparing images with pipeline {} | {}', convert_cmd, compare_cmd) convert_proc = subprocess.Popen( @@ -412,9 +415,11 @@ class ArtResizer(metaclass=Shareable): if self.im_legacy: self.im_convert_cmd = ['convert'] self.im_identify_cmd = ['identify'] + self.im_compare_cmd = ['compare'] else: self.im_convert_cmd = ['magick'] self.im_identify_cmd = ['magick', 'identify'] + self.im_compare_cmd = ['magick', 'compare'] def resize( self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 From 0125b1cd428f79552ee8c9ab358793bb5440f8ca Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 18:05:24 +0100 Subject: [PATCH 148/357] artresizer: in backends, always use the appropriate ArtResizer instance This didn't cause any issues since we only use the shared instance anyway, but logically it doesn't make a lot of sense for the backends always using ArtResizer.shared (which they should be oblivious of). --- beets/util/artresizer.py | 52 +++++++++++++++++++++------------------- test/test_art_resize.py | 8 +++++-- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index f076b3ab5..0260c6f82 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -61,7 +61,8 @@ def temp_file_for(path): return bytestring_path(f.name) -def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): +def pil_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, + max_filesize=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -119,7 +120,8 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): return path_in -def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): +def im_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, + max_filesize=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -134,7 +136,7 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # with regards to the height. # ImageMagick already seems to default to no interlace, but we include it # here for the sake of explicitness. - cmd = ArtResizer.shared.im_convert_cmd + [ + cmd = artresizer.im_convert_cmd + [ syspath(path_in, prefix=False), '-resize', f'{maxwidth}x>', '-interlace', 'none', @@ -166,7 +168,7 @@ BACKEND_FUNCS = { } -def pil_getsize(path_in): +def pil_getsize(artresizer, path_in): from PIL import Image try: @@ -177,8 +179,8 @@ def pil_getsize(path_in): displayable_path(path_in), exc) -def im_getsize(path_in): - cmd = ArtResizer.shared.im_identify_cmd + \ +def im_getsize(artresizer, path_in): + cmd = artresizer.im_identify_cmd + \ ['-format', '%w %h', syspath(path_in, prefix=False)] try: @@ -203,7 +205,7 @@ BACKEND_GET_SIZE = { } -def pil_deinterlace(path_in, path_out=None): +def pil_deinterlace(artresizer, path_in, path_out=None): path_out = path_out or temp_file_for(path_in) from PIL import Image @@ -215,10 +217,10 @@ def pil_deinterlace(path_in, path_out=None): return path_in -def im_deinterlace(path_in, path_out=None): +def im_deinterlace(artresizer, path_in, path_out=None): path_out = path_out or temp_file_for(path_in) - cmd = ArtResizer.shared.im_convert_cmd + [ + cmd = artresizer.im_convert_cmd + [ syspath(path_in, prefix=False), '-interlace', 'none', syspath(path_out, prefix=False), @@ -237,8 +239,8 @@ DEINTERLACE_FUNCS = { } -def im_get_format(filepath): - cmd = ArtResizer.shared.im_identify_cmd + [ +def im_get_format(artresizer, filepath): + cmd = artresizer.im_identify_cmd + [ '-format', '%[magick]', syspath(filepath) ] @@ -249,7 +251,7 @@ def im_get_format(filepath): return None -def pil_get_format(filepath): +def pil_get_format(artresizer, filepath): from PIL import Image, UnidentifiedImageError try: @@ -266,8 +268,8 @@ BACKEND_GET_FORMAT = { } -def im_convert_format(source, target, deinterlaced): - cmd = ArtResizer.shared.im_convert_cmd + [ +def im_convert_format(artresizer, source, target, deinterlaced): + cmd = artresizer.im_convert_cmd + [ syspath(source), *(["-interlace", "none"] if deinterlaced else []), syspath(target), @@ -284,7 +286,7 @@ def im_convert_format(source, target, deinterlaced): return source -def pil_convert_format(source, target, deinterlaced): +def pil_convert_format(artresizer, source, target, deinterlaced): from PIL import Image, UnidentifiedImageError try: @@ -302,7 +304,7 @@ BACKEND_CONVERT_IMAGE_FORMAT = { IMAGEMAGICK: im_convert_format, } -def im_compare(im1, im2, compare_threshold): +def im_compare(artresizer, im1, im2, compare_threshold): is_windows = platform.system() == "Windows" # Converting images to grayscale tends to minimize the weight @@ -310,11 +312,11 @@ def im_compare(im1, im2, compare_threshold): # to grayscale and then pipe them into the `compare` command. # On Windows, ImageMagick doesn't support the magic \\?\ prefix # on paths, so we pass `prefix=False` to `syspath`. - convert_cmd = ArtResizer.shared.im_convert_cmd + [ + convert_cmd = artresizer.im_convert_cmd + [ syspath(im2, prefix=False), syspath(im1, prefix=False), '-colorspace', 'gray', 'MIFF:-' ] - compare_cmd = ArtResizer.shared.im_compare_cmd + [ + compare_cmd = artresizer.im_compare_cmd + [ '-metric', 'PHASH', '-', 'null:', ] log.debug('comparing images with pipeline {} | {}', @@ -368,7 +370,7 @@ def im_compare(im1, im2, compare_threshold): return phash_diff <= compare_threshold -def pil_compare(im1, im2, compare_threshold): +def pil_compare(artresizer, im1, im2, compare_threshold): # It is an error to call this when ArtResizer.can_compare is not True. raise NotImplementedError() @@ -431,7 +433,7 @@ class ArtResizer(metaclass=Shareable): """ if self.local: func = BACKEND_FUNCS[self.method[0]] - return func(maxwidth, path_in, path_out, + return func(self, maxwidth, path_in, path_out, quality=quality, max_filesize=max_filesize) else: return path_in @@ -439,7 +441,7 @@ class ArtResizer(metaclass=Shareable): def deinterlace(self, path_in, path_out=None): if self.local: func = DEINTERLACE_FUNCS[self.method[0]] - return func(path_in, path_out) + return func(self, path_in, path_out) else: return path_in @@ -468,7 +470,7 @@ class ArtResizer(metaclass=Shareable): """ if self.local: func = BACKEND_GET_SIZE[self.method[0]] - return func(path_in) + return func(self, path_in) def get_format(self, path_in): """Returns the format of the image as a string. @@ -477,7 +479,7 @@ class ArtResizer(metaclass=Shareable): """ if self.local: func = BACKEND_GET_FORMAT[self.method[0]] - return func(path_in) + return func(self, path_in) def reformat(self, path_in, new_format, deinterlaced=True): """Converts image to desired format, updating its extension, but @@ -502,7 +504,7 @@ class ArtResizer(metaclass=Shareable): # file path was removed result_path = path_in try: - result_path = func(path_in, path_new, deinterlaced) + result_path = func(self, path_in, path_new, deinterlaced) finally: if result_path != path_in: os.unlink(path_in) @@ -521,7 +523,7 @@ class ArtResizer(metaclass=Shareable): """ if self.local: func = BACKEND_COMPARE[self.method[0]] - return func(im1, im2, compare_threshold) + return func(self, im1, im2, compare_threshold) @staticmethod def _check_method(): diff --git a/test/test_art_resize.py b/test/test_art_resize.py index 73847e0a6..4600bab77 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -50,6 +50,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): """Test resizing based on file size, given a resize_func.""" # Check quality setting unaffected by new parameter im_95_qual = resize_func( + ArtResizer.shared, 225, self.IMG_225x225, quality=95, @@ -60,6 +61,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): # Attempt a lower filesize with same quality im_a = resize_func( + ArtResizer.shared, 225, self.IMG_225x225, quality=95, @@ -72,6 +74,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): # Attempt with lower initial quality im_75_qual = resize_func( + ArtResizer.shared, 225, self.IMG_225x225, quality=75, @@ -80,6 +83,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): self.assertExists(im_75_qual) im_b = resize_func( + ArtResizer.shared, 225, self.IMG_225x225, quality=95, @@ -107,7 +111,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): Check if pil_deinterlace function returns images that are non-progressive """ - path = pil_deinterlace(self.IMG_225x225) + path = pil_deinterlace(ArtResizer.shared, self.IMG_225x225) from PIL import Image with Image.open(path) as img: self.assertFalse('progression' in img.info) @@ -119,7 +123,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): Check if im_deinterlace function returns images that are non-progressive. """ - path = im_deinterlace(self.IMG_225x225) + path = im_deinterlace(ArtResizer.shared, self.IMG_225x225) cmd = ArtResizer.shared.im_identify_cmd + [ '-format', '%[interlace]', syspath(path, prefix=False), ] From fa967f3efc7bccd9d6d6b6991d8aab5e35e45730 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 22:59:25 +0100 Subject: [PATCH 149/357] artresizer: add a comment --- beets/art.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/art.py b/beets/art.py index 1dff8b39a..eb283d600 100644 --- a/beets/art.py +++ b/beets/art.py @@ -115,6 +115,10 @@ def resize_image(log, imagepath, maxwidth, quality): def check_art_similarity(log, item, imagepath, compare_threshold): """A boolean indicating if an image is similar to embedded item art. + + If no embedded art exists, always return `True`. + + This must only be called if `ArtResizer.shared.can_compare` is `True`. """ with NamedTemporaryFile(delete=True) as f: art = extract(log, f.name, item) From 2fa37aa22b46946719479fe8f0f7d80a76d4f0d1 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 22:59:52 +0100 Subject: [PATCH 150/357] artresizer: return None explicitly `None` is used both as a marker when errors occured, and when a function is not implemented by the backend. That is already confusing enough without there being implicit `None` return values when falling of the tail of a method --- beets/util/artresizer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 0260c6f82..d1a544bd8 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -177,6 +177,7 @@ def pil_getsize(artresizer, path_in): except OSError as exc: log.error("PIL could not read file {}: {}", displayable_path(path_in), exc) + return None def im_getsize(artresizer, path_in): @@ -192,11 +193,12 @@ def im_getsize(artresizer, path_in): 'getting size with command {}:\n{}', exc.returncode, cmd, exc.output.strip() ) - return + return None try: return tuple(map(int, out.split(b' '))) except IndexError: log.warning('Could not understand IM output: {0!r}', out) + return None BACKEND_GET_SIZE = { @@ -347,7 +349,7 @@ def im_compare(artresizer, im1, im2, compare_threshold): convert_proc.returncode, convert_stderr, ) - return + return None # Check the compare output. stdout, stderr = compare_proc.communicate() @@ -355,7 +357,7 @@ def im_compare(artresizer, im1, im2, compare_threshold): if compare_proc.returncode != 1: log.debug('ImageMagick compare failed: {0}, {1}', displayable_path(im2), displayable_path(im1)) - return + return None out_str = stderr else: out_str = stdout @@ -364,7 +366,7 @@ def im_compare(artresizer, im1, im2, compare_threshold): phash_diff = float(out_str) except ValueError: log.debug('IM output is not a number: {0!r}', out_str) - return + return None log.debug('ImageMagick compare score: {0}', phash_diff) return phash_diff <= compare_threshold @@ -471,6 +473,7 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_GET_SIZE[self.method[0]] return func(self, path_in) + return None def get_format(self, path_in): """Returns the format of the image as a string. @@ -480,6 +483,7 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_GET_FORMAT[self.method[0]] return func(self, path_in) + return None def reformat(self, path_in, new_format, deinterlaced=True): """Converts image to desired format, updating its extension, but @@ -524,6 +528,7 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_COMPARE[self.method[0]] return func(self, im1, im2, compare_threshold) + return None @staticmethod def _check_method(): From 03ce35f4d3e01862fddb6f527c9d3a58ac2f528b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 23:10:22 +0100 Subject: [PATCH 151/357] art: whitespace/comment improvements --- beets/art.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beets/art.py b/beets/art.py index eb283d600..619c472a5 100644 --- a/beets/art.py +++ b/beets/art.py @@ -51,14 +51,17 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, quality=0): """Embed an image into the item's media file. """ - # Conditions and filters. + # Conditions. if compare_threshold: if not check_art_similarity(log, item, imagepath, compare_threshold): log.info('Image not similar; skipping.') return + if ifempty and get_art(log, item): log.info('media file already contained art') return + + # Filters. if maxwidth and not as_album: imagepath = resize_image(log, imagepath, maxwidth, quality) From 8235af9770d876bf2152493806f1ae744af72f39 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 23:16:05 +0100 Subject: [PATCH 152/357] art: log errors more explicitly, add some comments error handling was previously implicit in the `if not check_art_similarity(...)` check, which is pretty unintuitive --- beets/art.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/beets/art.py b/beets/art.py index 619c472a5..e7f18ee1d 100644 --- a/beets/art.py +++ b/beets/art.py @@ -53,7 +53,12 @@ def embed_item(log, item, imagepath, maxwidth=None, itempath=None, """ # Conditions. if compare_threshold: - if not check_art_similarity(log, item, imagepath, compare_threshold): + is_similar = check_art_similarity( + log, item, imagepath, compare_threshold) + if is_similar is None: + log.warning('Error while checking art similarity; skipping.') + return + elif not is_similar: log.info('Image not similar; skipping.') return @@ -119,7 +124,8 @@ def resize_image(log, imagepath, maxwidth, quality): def check_art_similarity(log, item, imagepath, compare_threshold): """A boolean indicating if an image is similar to embedded item art. - If no embedded art exists, always return `True`. + If no embedded art exists, always return `True`. If the comparison fails + for some reason, the return value is `None`. This must only be called if `ArtResizer.shared.can_compare` is `True`. """ From f558c091b47c5eca256704c0e07c48d974d338e3 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 23:26:25 +0100 Subject: [PATCH 153/357] update changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a28a992b9..bbe617638 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,6 +46,9 @@ Bug fixes: * :doc:`plugins/replaygain`: Correctly handle the ``overwrite`` config option, which forces recomputing ReplayGain values on import even for tracks that already have the tags. +* :doc:`plugins/embedart`: Fix a crash when using recent versions of + ImageMagick and the ``compare_threshold`` option. + :bug:`4272` For packagers: From 959e24e463c764da1cd8d05535d50222209573b5 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Feb 2022 23:34:35 +0100 Subject: [PATCH 154/357] artresizer: whitespace fixes --- beets/util/artresizer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index d1a544bd8..0041db230 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -62,7 +62,7 @@ def temp_file_for(path): def pil_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, - max_filesize=0): + max_filesize=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -121,7 +121,7 @@ def pil_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, def im_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, - max_filesize=0): + max_filesize=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -306,6 +306,7 @@ BACKEND_CONVERT_IMAGE_FORMAT = { IMAGEMAGICK: im_convert_format, } + def im_compare(artresizer, im1, im2, compare_threshold): is_windows = platform.system() == "Windows" From 1815d083925846ff1e7b5223d2b00f1bd9f7efbb Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 14 Feb 2022 22:52:00 +0100 Subject: [PATCH 155/357] artresizer: fix image comparison test Since the ImageMagick based comparison is now abstracted via ArtResizer, it becomes a little more involved to mock it for testing. --- beets/art.py | 13 +++++++++++-- test/test_embedart.py | 21 ++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/beets/art.py b/beets/art.py index e7f18ee1d..6e0a5f82b 100644 --- a/beets/art.py +++ b/beets/art.py @@ -121,7 +121,13 @@ def resize_image(log, imagepath, maxwidth, quality): return imagepath -def check_art_similarity(log, item, imagepath, compare_threshold): +def check_art_similarity( + log, + item, + imagepath, + compare_threshold, + artresizer=None, +): """A boolean indicating if an image is similar to embedded item art. If no embedded art exists, always return `True`. If the comparison fails @@ -135,7 +141,10 @@ def check_art_similarity(log, item, imagepath, compare_threshold): if not art: return True - return ArtResizer.shared.compare(art, imagepath, compare_threshold) + if artresizer is None: + artresizer = ArtResizer.shared + + return artresizer.compare(art, imagepath, compare_threshold) def extract(log, outpath, item): diff --git a/test/test_embedart.py b/test/test_embedart.py index b02989175..0fed08f98 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -24,7 +24,7 @@ from test.helper import TestHelper from mediafile import MediaFile from beets import config, logging, ui -from beets.util import syspath, displayable_path +from beets.util import artresizer, syspath, displayable_path from beets.util.artresizer import ArtResizer from beets import art @@ -216,16 +216,31 @@ class EmbedartCliTest(_common.TestCase, TestHelper): self.assertEqual(mediafile.images[0].data, self.image_data) +class DummyArtResizer(ArtResizer): + """An `ArtResizer` which pretends that ImageMagick is available, and has + a sufficiently recent version to support image comparison. + """ + @staticmethod + def _check_method(): + return artresizer.IMAGEMAGICK, (7, 0, 0), True + + @patch('beets.util.artresizer.subprocess') @patch('beets.art.extract') class ArtSimilarityTest(unittest.TestCase): def setUp(self): self.item = _common.item() self.log = logging.getLogger('beets.embedart') + self.artresizer = DummyArtResizer() def _similarity(self, threshold): - return art.check_art_similarity(self.log, self.item, b'path', - threshold) + return art.check_art_similarity( + self.log, + self.item, + b'path', + threshold, + artresizer=self.artresizer, + ) def _popen(self, status=0, stdout="", stderr=""): """Create a mock `Popen` object.""" From 8e5156d01c3c7a940bb19a7497f40bb7a1574b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 15 Feb 2022 23:11:30 +0100 Subject: [PATCH 156/357] fish plugin: Add --output option --- beetsplug/fish.py | 29 +++++++++++++++++++++-------- docs/changelog.rst | 1 + docs/plugins/fish.rst | 3 +++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index 21fd67f60..d0bedbedf 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -81,6 +81,11 @@ class FishPlugin(BeetsPlugin): choices=library.Item.all_keys() + library.Album.all_keys(), help='include specified field *values* in completions') + cmd.parser.add_option( + '-o', + '--output', + default=None, + help='save the script to a specific file, by default it will be saved to ~/.config/fish/completions') return [cmd] def run(self, lib, opts, args): @@ -89,14 +94,22 @@ class FishPlugin(BeetsPlugin): # If specified, also collect the values for these fields. # Make a giant string of all the above, formatted in a way that # allows Fish to do tab completion for the `beet` command. - home_dir = os.path.expanduser("~") - completion_dir = os.path.join(home_dir, '.config/fish/completions') - try: - os.makedirs(completion_dir) - except OSError: - if not os.path.isdir(completion_dir): - raise - completion_file_path = os.path.join(completion_dir, 'beet.fish') + + completion_file_path = opts.output + if completion_file_path is not None: + completion_dir = os.path.dirname(completion_file_path) + else: + home_dir = os.path.expanduser("~") + completion_dir = os.path.join(home_dir, '.config/fish/completions') + completion_file_path = os.path.join(completion_dir, 'beet.fish') + + if completion_dir != '': + try: + os.makedirs(completion_dir) + except OSError: + if not os.path.isdir(completion_dir): + raise + nobasicfields = opts.noFields # Do not complete for album/track fields extravalues = opts.extravalues # e.g., Also complete artists names beetcmds = sorted( diff --git a/docs/changelog.rst b/docs/changelog.rst index 2878c7953..523436bea 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -59,6 +59,7 @@ Other new things: * :doc:`/plugins/limit`: Limit query results to head or tail (``lslimit`` command only) +* :doc:`/plugins/fish`: Add ``--output`` option. 1.6.0 (November 27, 2021) ------------------------- diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst index b2cb096ee..80dd3fb99 100644 --- a/docs/plugins/fish.rst +++ b/docs/plugins/fish.rst @@ -50,3 +50,6 @@ with care when specified fields contain a large number of values. Libraries with for example, very large numbers of genres/artists may result in higher memory utilization, completion latency, et cetera. This option is not meant to replace database queries altogether. + +By default the completion file will be generated at ``~/.config/fish/completions/``, +if you want to save it somewhere else you can use the ``-o`` or ``--output``. From 40fcb25221b719d12295e90014fdbb8e93380981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 16 Feb 2022 08:15:30 +0100 Subject: [PATCH 157/357] fish plugin: Split long line --- beetsplug/fish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index d0bedbedf..a578c7397 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -85,7 +85,8 @@ class FishPlugin(BeetsPlugin): '-o', '--output', default=None, - help='save the script to a specific file, by default it will be saved to ~/.config/fish/completions') + help='save the script to a specific file, by default it will be' + 'saved to ~/.config/fish/completions') return [cmd] def run(self, lib, opts, args): From b46b4d2045aee0242e42e2ecfa9b9c02c02d6f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 16 Feb 2022 21:34:24 +0100 Subject: [PATCH 158/357] fish plugin: Simplify directory creation --- beetsplug/fish.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index a578c7397..5f3ecd78e 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -105,11 +105,7 @@ class FishPlugin(BeetsPlugin): completion_file_path = os.path.join(completion_dir, 'beet.fish') if completion_dir != '': - try: - os.makedirs(completion_dir) - except OSError: - if not os.path.isdir(completion_dir): - raise + os.makedirs(completion_dir, exist_ok=True) nobasicfields = opts.noFields # Do not complete for album/track fields extravalues = opts.extravalues # e.g., Also complete artists names From fedb8b0b8fa77d1e8e2bb6814819881382f13a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 16 Feb 2022 21:42:08 +0100 Subject: [PATCH 159/357] fish plugin: Assign the default output path to the option instead of using None --- beetsplug/fish.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index 5f3ecd78e..4a0312d29 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -84,7 +84,7 @@ class FishPlugin(BeetsPlugin): cmd.parser.add_option( '-o', '--output', - default=None, + default='~/.config/fish/completions/beet.fish', help='save the script to a specific file, by default it will be' 'saved to ~/.config/fish/completions') return [cmd] @@ -96,13 +96,8 @@ class FishPlugin(BeetsPlugin): # Make a giant string of all the above, formatted in a way that # allows Fish to do tab completion for the `beet` command. - completion_file_path = opts.output - if completion_file_path is not None: - completion_dir = os.path.dirname(completion_file_path) - else: - home_dir = os.path.expanduser("~") - completion_dir = os.path.join(home_dir, '.config/fish/completions') - completion_file_path = os.path.join(completion_dir, 'beet.fish') + completion_file_path = os.path.expanduser(opts.output) + completion_dir = os.path.dirname(completion_file_path) if completion_dir != '': os.makedirs(completion_dir, exist_ok=True) From 8e921e4d74870bebff0de67899695952a6dc8ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqu=C3=ADnez=20Ferr=C3=A1ndiz?= Date: Wed, 16 Feb 2022 21:51:19 +0100 Subject: [PATCH 160/357] fish plugin: Style fix Co-authored-by: Adrian Sampson --- docs/plugins/fish.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst index 80dd3fb99..419d0206c 100644 --- a/docs/plugins/fish.rst +++ b/docs/plugins/fish.rst @@ -51,5 +51,5 @@ for example, very large numbers of genres/artists may result in higher memory utilization, completion latency, et cetera. This option is not meant to replace database queries altogether. -By default the completion file will be generated at ``~/.config/fish/completions/``, -if you want to save it somewhere else you can use the ``-o`` or ``--output``. +By default the completion file will be generated at ``~/.config/fish/completions/``. +If you want to save it somewhere else you can use the ``-o`` or ``--output``. From 5f45e9e10894ea94d35a885d25c4c4ffffcfaa91 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 16 Feb 2022 16:34:38 -0500 Subject: [PATCH 161/357] Tiny wording tweaks for #4281 --- beetsplug/fish.py | 4 ++-- docs/plugins/fish.rst | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/beetsplug/fish.py b/beetsplug/fish.py index 4a0312d29..cfb168d9a 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -85,8 +85,8 @@ class FishPlugin(BeetsPlugin): '-o', '--output', default='~/.config/fish/completions/beet.fish', - help='save the script to a specific file, by default it will be' - 'saved to ~/.config/fish/completions') + help='where to save the script. default: ' + '~/.config/fish/completions') return [cmd] def run(self, lib, opts, args): diff --git a/docs/plugins/fish.rst b/docs/plugins/fish.rst index 419d0206c..0c89576c5 100644 --- a/docs/plugins/fish.rst +++ b/docs/plugins/fish.rst @@ -51,5 +51,7 @@ for example, very large numbers of genres/artists may result in higher memory utilization, completion latency, et cetera. This option is not meant to replace database queries altogether. -By default the completion file will be generated at ``~/.config/fish/completions/``. -If you want to save it somewhere else you can use the ``-o`` or ``--output``. +By default, the completion file will be generated at +``~/.config/fish/completions/``. +If you want to save it somewhere else, you can use the ``-o`` or ``--output`` +option. From 8e4047345371322dcaef4ce84ad76a32b4797031 Mon Sep 17 00:00:00 2001 From: Dickson Date: Sat, 26 Feb 2022 21:15:28 +0800 Subject: [PATCH 162/357] fix: dont deduplicate matches if musicbrainz ID is not set for album candidates --- beets/autotag/match.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/match.py b/beets/autotag/match.py index d352a013f..3061aa39b 100644 --- a/beets/autotag/match.py +++ b/beets/autotag/match.py @@ -357,8 +357,8 @@ def _add_candidate(items, results, info): log.debug('No tracks.') return - # Don't duplicate. - if info.album_id in results: + # Prevent duplicates + if info.album_id and info.album_id in results: log.debug('Duplicate.') return From 6d6bb51fde7c75dea996ba2b9ebadf7e7751849b Mon Sep 17 00:00:00 2001 From: Dickson Date: Sun, 27 Feb 2022 13:28:17 +0800 Subject: [PATCH 163/357] Update changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1c7ab25ac..877fb32ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,7 @@ New features: Bug fixes: +* Fix autotagger marking all albums without a musicbrainz id as a duplicate * :doc:`/plugins/convert`: Resize album art when embedding :bug:`2116` * :doc:`/plugins/deezer`: Fix auto tagger pagination issues (fetch beyond the From 5e057078a609ae604a179ae3a73c062cc9c6931c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 27 Feb 2022 14:01:48 -0500 Subject: [PATCH 164/357] Slight rewording --- beets/autotag/match.py | 2 +- docs/changelog.rst | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/autotag/match.py b/beets/autotag/match.py index 3061aa39b..814738cd1 100644 --- a/beets/autotag/match.py +++ b/beets/autotag/match.py @@ -357,7 +357,7 @@ def _add_candidate(items, results, info): log.debug('No tracks.') return - # Prevent duplicates + # Prevent duplicates. if info.album_id and info.album_id in results: log.debug('Duplicate.') return diff --git a/docs/changelog.rst b/docs/changelog.rst index 877fb32ed..7135ecb3e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,7 +15,9 @@ New features: Bug fixes: -* Fix autotagger marking all albums without a musicbrainz id as a duplicate +* The autotagger no longer considers all matches without a MusicBrainz ID as + duplicates of each other. + :bug:`4299` * :doc:`/plugins/convert`: Resize album art when embedding :bug:`2116` * :doc:`/plugins/deezer`: Fix auto tagger pagination issues (fetch beyond the From 55e4917df31f862eac9ae81ec55a2061ca9df48e Mon Sep 17 00:00:00 2001 From: Joseph Heyburn Date: Wed, 2 Mar 2022 22:04:36 +0000 Subject: [PATCH 165/357] discogs: allow style to be appended to genre - Adds a configuration that, when enabled, will append the style to genre - Rationale is to have more verbose genres in genre tag of players that only support genre --- beetsplug/discogs.py | 9 ++++++++- docs/changelog.rst | 1 + docs/plugins/discogs.rst | 6 ++++++ test/test_discogs.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 8c950c521..3c9d16673 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -57,6 +57,7 @@ class DiscogsPlugin(BeetsPlugin): 'user_token': '', 'separator': ', ', 'index_tracks': False, + 'append_style_genre': False, }) self.config['apikey'].redact = True self.config['apisecret'].redact = True @@ -318,7 +319,13 @@ class DiscogsPlugin(BeetsPlugin): country = result.data.get('country') data_url = result.data.get('uri') style = self.format(result.data.get('styles')) - genre = self.format(result.data.get('genres')) + + if self.config['append_style_genre'] and style: + genre = self.config['separator'].as_str() \ + .join([self.format(result.data.get('genres')), style]) + else: + genre = self.format(result.data.get('genres')) + discogs_albumid = self.extract_release_id(result.data.get('uri')) # Extract information for the optional AlbumInfo fields that are diff --git a/docs/changelog.rst b/docs/changelog.rst index 7135ecb3e..633ddda24 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ New features: :bug:`4101` * Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. * Add query prefixes ``=`` and ``~``. +* :doc:`/plugins/discogs`: Permit appending style to genre Bug fixes: diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 5aea1ae6b..b4d0bec1a 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -76,6 +76,12 @@ whereas with ``index_tracks`` disabled you'd get:: This option is useful when importing classical music. +Other configurations available under ``discogs:`` are: + +- **append_style_genre**: Appends the style (if found) to the genre tag, useful if you would like more granular genre styles added to music file tags + Default: ``false`` + + Troubleshooting --------------- diff --git a/test/test_discogs.py b/test/test_discogs.py index 4e62e7124..d6825c978 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -20,6 +20,8 @@ from test import _common from test._common import Bag from test.helper import capture_log +from beets import config + from beetsplug.discogs import DiscogsPlugin @@ -373,6 +375,33 @@ class DGAlbumInfoTest(_common.TestCase): match = '' self.assertEqual(match, expected) + def test_default_genre_style_settings(self): + """Test genre default settings, genres to genre, styles to style""" + release = self._make_release_from_positions(['1', '2']) + + d = DiscogsPlugin().get_album_info(release) + self.assertEqual(d.genre, 'GENRE1, GENRE2') + self.assertEqual(d.style, 'STYLE1, STYLE2') + + def test_append_style_to_genre(self): + """Test appending style to genre if config enabled""" + config['discogs']['append_style_genre'] = True + release = self._make_release_from_positions(['1', '2']) + + d = DiscogsPlugin().get_album_info(release) + self.assertEqual(d.genre, 'GENRE1, GENRE2, STYLE1, STYLE2') + self.assertEqual(d.style, 'STYLE1, STYLE2') + + def test_append_style_to_genre_no_style(self): + """Test nothing appended to genre if style is empty""" + config['discogs']['append_style_genre'] = True + release = self._make_release_from_positions(['1', '2']) + release.data['styles'] = [] + + d = DiscogsPlugin().get_album_info(release) + self.assertEqual(d.genre, 'GENRE1, GENRE2') + self.assertEqual(d.style, None) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 4bde4d082d1a58c6e9a4279d837b06b7a432dbf4 Mon Sep 17 00:00:00 2001 From: Joseph Heyburn Date: Thu, 3 Mar 2022 10:03:31 +0000 Subject: [PATCH 166/357] discogs: allow style to be appended to genre - Added more verbose documentation to `append_style_genre` - Refactor based on code review --- beetsplug/discogs.py | 6 +++--- docs/plugins/discogs.rst | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 3c9d16673..875842f83 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -319,12 +319,12 @@ class DiscogsPlugin(BeetsPlugin): country = result.data.get('country') data_url = result.data.get('uri') style = self.format(result.data.get('styles')) + base_genre = self.format(result.data.get('genres')) if self.config['append_style_genre'] and style: - genre = self.config['separator'].as_str() \ - .join([self.format(result.data.get('genres')), style]) + genre = self.config['separator'].as_str().join([base_genre, style]) else: - genre = self.format(result.data.get('genres')) + genre = base_genre discogs_albumid = self.extract_release_id(result.data.get('uri')) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index b4d0bec1a..d23e4a9c2 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -79,7 +79,10 @@ This option is useful when importing classical music. Other configurations available under ``discogs:`` are: - **append_style_genre**: Appends the style (if found) to the genre tag, useful if you would like more granular genre styles added to music file tags + e.g. A release in Discogs has a genre of "Electronic" and a style of "Techno" - enabling this setting would set the genre to be "Electronic, Techno" (assuming default separator of ``", "``) instead of just "Electronic" Default: ``false`` +- **separator**: How to join genre and style responses from Discogs into a string + Default: ``", "`` Troubleshooting From e1d19c3a3f2c891383230366adfdeba9b14f9253 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 3 Mar 2022 11:17:07 +0100 Subject: [PATCH 167/357] Docs refinement --- docs/plugins/discogs.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index d23e4a9c2..1ed2dfc93 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -78,10 +78,10 @@ This option is useful when importing classical music. Other configurations available under ``discogs:`` are: -- **append_style_genre**: Appends the style (if found) to the genre tag, useful if you would like more granular genre styles added to music file tags - e.g. A release in Discogs has a genre of "Electronic" and a style of "Techno" - enabling this setting would set the genre to be "Electronic, Techno" (assuming default separator of ``", "``) instead of just "Electronic" +- **append_style_genre**: Appends the Discogs style (if found) to the genre tag. This can be useful if you want more granular genres to categorize your music. + For example, a release in Discogs might have a genre of "Electronic" and a style of "Techno": enabling this setting would set the genre to be "Electronic, Techno" (assuming default separator of ``", "``) instead of just "Electronic". Default: ``false`` -- **separator**: How to join genre and style responses from Discogs into a string +- **separator**: How to join multiple genre and style values from Discogs into a string. Default: ``", "`` From c9c1123756ed233b01dd6619e6d548b036c9e1b4 Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Tue, 1 Mar 2022 07:44:39 +0100 Subject: [PATCH 168/357] discogs: Fix discogs_albumid extraction Use extract_release_id_regex instead of extract_release_id to get the release ID out ouf the Discogs release URL. --- beetsplug/discogs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 875842f83..820a0acbd 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -326,7 +326,7 @@ class DiscogsPlugin(BeetsPlugin): else: genre = base_genre - discogs_albumid = self.extract_release_id(result.data.get('uri')) + discogs_albumid = self.extract_release_id_regex(result.data.get('uri')) # Extract information for the optional AlbumInfo fields that are # contained on nested discogs fields. @@ -378,12 +378,6 @@ class DiscogsPlugin(BeetsPlugin): else: return None - def extract_release_id(self, uri): - if uri: - return uri.split("/")[-1] - else: - return None - def get_tracks(self, tracklist): """Returns a list of TrackInfo objects for a discogs tracklist. """ From 76e81199b5c550f1270f3e8dd569e6d7fdc90df8 Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Tue, 1 Mar 2022 08:43:03 +0100 Subject: [PATCH 169/357] discogs: Changelog entry for #4303 Fix discogs_albumid.. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 633ddda24..f1833aad4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,10 @@ New features: Bug fixes: +* The Discogs release ID is now populated correctly to the discogs_albumid + field again (it was no longer working after Discogs changed their release URL + format). + :bug:`4225` * The autotagger no longer considers all matches without a MusicBrainz ID as duplicates of each other. :bug:`4299` From b609047d6463cac69345d50d693be945ba87b27e Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Wed, 2 Mar 2022 16:29:07 +0100 Subject: [PATCH 170/357] discogs: Add extract_release_id_regex sanity check Check whether any input worth pattern checking was passed. --- beetsplug/discogs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 820a0acbd..30aa6d28d 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -194,16 +194,17 @@ class DiscogsPlugin(BeetsPlugin): # patterns. # Regex has been tested here https://regex101.com/r/wyLdB4/2 - for pattern in [ - r'^\[?r?(?P\d+)\]?$', - r'discogs\.com/release/(?P\d+)-', - r'discogs\.com/[^/]+/release/(?P\d+)', - ]: - match = re.search(pattern, album_id) - if match: - return int(match.group('id')) - - return None + if album_id: + for pattern in [ + r'^\[?r?(?P\d+)\]?$', + r'discogs\.com/release/(?P\d+)-', + r'discogs\.com/[^/]+/release/(?P\d+)', + ]: + match = re.search(pattern, album_id) + if match: + return int(match.group('id')) + else: + return None def album_for_id(self, album_id): """Fetches an album by its Discogs ID and returns an AlbumInfo object From c85f903caed5bfd74b45f94d48d6336d63c11f58 Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Fri, 4 Mar 2022 08:02:29 +0100 Subject: [PATCH 171/357] Revert "discogs: Add extract_release_id_regex sanity check" This reverts commit c3cc055fdd3830bbe1c5470fe540684278a6ecc7. We assume the Discogs API never returns a release response without an URI. --- beetsplug/discogs.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 30aa6d28d..820a0acbd 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -194,17 +194,16 @@ class DiscogsPlugin(BeetsPlugin): # patterns. # Regex has been tested here https://regex101.com/r/wyLdB4/2 - if album_id: - for pattern in [ - r'^\[?r?(?P\d+)\]?$', - r'discogs\.com/release/(?P\d+)-', - r'discogs\.com/[^/]+/release/(?P\d+)', - ]: - match = re.search(pattern, album_id) - if match: - return int(match.group('id')) - else: - return None + for pattern in [ + r'^\[?r?(?P\d+)\]?$', + r'discogs\.com/release/(?P\d+)-', + r'discogs\.com/[^/]+/release/(?P\d+)', + ]: + match = re.search(pattern, album_id) + if match: + return int(match.group('id')) + + return None def album_for_id(self, album_id): """Fetches an album by its Discogs ID and returns an AlbumInfo object From b2f4834b76de7aa694319c2f4a10d0fa5062cd2c Mon Sep 17 00:00:00 2001 From: J0J0 T Date: Fri, 4 Mar 2022 07:46:15 +0100 Subject: [PATCH 172/357] discogs: Add URI to test_parse_minimal_release data --- test/test_discogs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_discogs.py b/test/test_discogs.py index d6825c978..c2aa7682c 100644 --- a/test/test_discogs.py +++ b/test/test_discogs.py @@ -337,6 +337,7 @@ class DGAlbumInfoTest(_common.TestCase): def test_parse_minimal_release(self): """Test parsing of a release with the minimal amount of information.""" data = {'id': 123, + 'uri': 'https://www.discogs.com/release/123456-something', 'tracklist': [self._make_track('A', '1', '01:01')], 'artists': [{'name': 'ARTIST NAME', 'id': 321, 'join': ''}], 'title': 'TITLE'} From 18e8b73c33d98b2b66639121aeb7c33bd973e59b Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Fri, 25 Feb 2022 18:26:35 +0100 Subject: [PATCH 173/357] add option to convert items after import convert items after imports but also keep original files intact --- beetsplug/convert.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index f384656f1..44141c691 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -138,6 +138,7 @@ class ConvertPlugin(BeetsPlugin): }, 'max_bitrate': 500, 'auto': False, + 'auto_keep': False, 'tmpdir': None, 'quiet': False, 'embed': True, @@ -148,7 +149,7 @@ class ConvertPlugin(BeetsPlugin): 'album_art_maxwidth': 0, 'delete_originals': False, }) - self.early_import_stages = [self.auto_convert] + self.early_import_stages = [self.auto_convert, self.auto_convert_keep] self.register_listener('import_task_files', self._cleanup) @@ -184,6 +185,34 @@ class ConvertPlugin(BeetsPlugin): par_map(lambda item: self.convert_on_import(config.lib, item), task.imported_items()) + def auto_convert_keep(self, config, task): + if self.config['auto_keep']: + fmt = self.config['format'].as_str().lower() + + dest = self.config['dest'].get() + if not dest: + raise ui.UserError('no convert destination set') + dest = util.bytestring_path(dest) + + path_formats = ui.get_path_formats(self.config['paths'] or None) + + hardlink = self.config['hardlink'].get(bool) + link = self.config['link'].get(bool) + + threads = self.config['threads'].get(int) + + items = task.imported_items() + convert = [self.convert_item(dest, + False, + path_formats, + fmt, + False, + link, + hardlink) + for _ in range(threads)] + pipe = util.pipeline.Pipeline([iter(items), convert]) + pipe.run_parallel() + # Utilities converted from functions to methods on logging overhaul def encode(self, command, source, dest, pretend=False): From af5858d2008c8d92501f6aa8dc329286332034ee Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Mon, 28 Feb 2022 14:17:18 +0100 Subject: [PATCH 174/357] doc + linter + changelog --- beetsplug/convert.py | 14 +++++++------- docs/changelog.rst | 3 ++- docs/plugins/convert.rst | 4 ++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 44141c691..c36d4aacb 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -203,13 +203,13 @@ class ConvertPlugin(BeetsPlugin): items = task.imported_items() convert = [self.convert_item(dest, - False, - path_formats, - fmt, - False, - link, - hardlink) - for _ in range(threads)] + False, + path_formats, + fmt, + False, + link, + hardlink) + for _ in range(threads)] pipe = util.pipeline.Pipeline([iter(items), convert]) pipe.run_parallel() diff --git a/docs/changelog.rst b/docs/changelog.rst index 633ddda24..f0c27d7b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ New features: * Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. * Add query prefixes ``=`` and ``~``. * :doc:`/plugins/discogs`: Permit appending style to genre +* Add auto_keep option to convert. Bug fixes: @@ -64,7 +65,7 @@ For packagers: Other new things: -* :doc:`/plugins/limit`: Limit query results to head or tail (``lslimit`` +* :doc:`/plugins/limit`: Limit query results to head or tail (``lslimit`` command only) * :doc:`/plugins/fish`: Add ``--output`` option. diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 799a20dbb..2cecd5a24 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -66,6 +66,10 @@ file. The available options are: default configuration) non-MP3 files over the maximum bitrate before adding them to your library. Default: ``no``. +- **auto_keep**: As opposite to **auto**, import non transcoded versions of + your files but still convert them to **dest**. It uses the default format + you have defined in your config file. + Default: ``no``. - **tmpdir**: The directory where temporary files will be stored during import. Default: none (system default), - **copy_album_art**: Copy album art when copying or transcoding albums matched From a2325e72adc2597b9f048770df104c3af7c129b2 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Fri, 4 Mar 2022 10:36:45 +0100 Subject: [PATCH 175/357] refactoring --- beetsplug/convert.py | 109 ++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 58 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index c36d4aacb..20d35910b 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -187,31 +187,13 @@ class ConvertPlugin(BeetsPlugin): def auto_convert_keep(self, config, task): if self.config['auto_keep']: - fmt = self.config['format'].as_str().lower() - - dest = self.config['dest'].get() - if not dest: - raise ui.UserError('no convert destination set') - dest = util.bytestring_path(dest) - - path_formats = ui.get_path_formats(self.config['paths'] or None) - - hardlink = self.config['hardlink'].get(bool) - link = self.config['link'].get(bool) - - threads = self.config['threads'].get(int) + empty_opts = self.commands()[0].parser.get_default_values() + (dest, threads, path_formats, fmt, + pretend, hardlink, link) = self._get_opts_and_config(empty_opts) items = task.imported_items() - convert = [self.convert_item(dest, - False, - path_formats, - fmt, - False, - link, - hardlink) - for _ in range(threads)] - pipe = util.pipeline.Pipeline([iter(items), convert]) - pipe.run_parallel() + self._parallel_convert(dest, False, path_formats, fmt, + pretend, link, hardlink, threads, items) # Utilities converted from functions to methods on logging overhaul @@ -452,31 +434,8 @@ class ConvertPlugin(BeetsPlugin): util.copy(album.artpath, dest) def convert_func(self, lib, opts, args): - dest = opts.dest or self.config['dest'].get() - if not dest: - raise ui.UserError('no convert destination set') - dest = util.bytestring_path(dest) - - threads = opts.threads or self.config['threads'].get(int) - - path_formats = ui.get_path_formats(self.config['paths'] or None) - - fmt = opts.format or self.config['format'].as_str().lower() - - if opts.pretend is not None: - pretend = opts.pretend - else: - pretend = self.config['pretend'].get(bool) - - if opts.hardlink is not None: - hardlink = opts.hardlink - link = False - elif opts.link is not None: - hardlink = False - link = opts.link - else: - hardlink = self.config['hardlink'].get(bool) - link = self.config['link'].get(bool) + (dest, threads, path_formats, fmt, + pretend, hardlink, link) = self._get_opts_and_config(opts) if opts.album: albums = lib.albums(ui.decargs(args)) @@ -501,16 +460,8 @@ class ConvertPlugin(BeetsPlugin): self.copy_album_art(album, dest, path_formats, pretend, link, hardlink) - convert = [self.convert_item(dest, - opts.keep_new, - path_formats, - fmt, - pretend, - link, - hardlink) - for _ in range(threads)] - pipe = util.pipeline.Pipeline([iter(items), convert]) - pipe.run_parallel() + self._parallel_convert(dest, opts.keep_new, path_formats, fmt, + pretend, link, hardlink, threads, items) def convert_on_import(self, lib, item): """Transcode a file automatically after it is imported into the @@ -575,3 +526,45 @@ class ConvertPlugin(BeetsPlugin): if os.path.isfile(path): util.remove(path) _temp_files.remove(path) + + def _get_opts_and_config(self, opts): + dest = opts.dest or self.config['dest'].get() + if not dest: + raise ui.UserError('no convert destination set') + dest = util.bytestring_path(dest) + + threads = opts.threads or self.config['threads'].get(int) + + path_formats = ui.get_path_formats(self.config['paths'] or None) + + fmt = opts.format or self.config['format'].as_str().lower() + + if opts.pretend is not None: + pretend = opts.pretend + else: + pretend = self.config['pretend'].get(bool) + + if opts.hardlink is not None: + hardlink = opts.hardlink + link = False + elif opts.link is not None: + hardlink = False + link = opts.link + else: + hardlink = self.config['hardlink'].get(bool) + link = self.config['link'].get(bool) + + return (dest, threads, path_formats, fmt, pretend, hardlink, link) + + def _parallel_convert(self, dest, keep_new, path_formats, fmt, + pretend, link, hardlink, threads, items): + convert = [self.convert_item(dest, + keep_new, + path_formats, + fmt, + pretend, + link, + hardlink) + for _ in range(threads)] + pipe = util.pipeline.Pipeline([iter(items), convert]) + pipe.run_parallel() From c4281ddacc4f00064895cb107644469718589b81 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Wed, 9 Mar 2022 18:34:31 +0100 Subject: [PATCH 176/357] add some documentation --- beetsplug/convert.py | 9 ++++++++- docs/plugins/convert.rst | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 20d35910b..a8674ca79 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -528,6 +528,10 @@ class ConvertPlugin(BeetsPlugin): _temp_files.remove(path) def _get_opts_and_config(self, opts): + """Returns parameters needed for convert function. + Get parameters from command line if available, + default to config if not available. + """ dest = opts.dest or self.config['dest'].get() if not dest: raise ui.UserError('no convert destination set') @@ -554,10 +558,13 @@ class ConvertPlugin(BeetsPlugin): hardlink = self.config['hardlink'].get(bool) link = self.config['link'].get(bool) - return (dest, threads, path_formats, fmt, pretend, hardlink, link) + return dest, threads, path_formats, fmt, pretend, hardlink, link def _parallel_convert(self, dest, keep_new, path_formats, fmt, pretend, link, hardlink, threads, items): + """Run the convert_item function for every items on as many thread as + defined in threads + """ convert = [self.convert_item(dest, keep_new, path_formats, diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 2cecd5a24..f8ee245ce 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -66,9 +66,9 @@ file. The available options are: default configuration) non-MP3 files over the maximum bitrate before adding them to your library. Default: ``no``. -- **auto_keep**: As opposite to **auto**, import non transcoded versions of - your files but still convert them to **dest**. It uses the default format - you have defined in your config file. +- **auto_keep**: Convert your files automatically on import to **dest** but + import the non transcoded version. It uses the default format you have + defined in your config file. Default: ``no``. - **tmpdir**: The directory where temporary files will be stored during import. Default: none (system default), From 03499a3b1eae43f7a69671528df794644415cc8b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 11 Mar 2022 08:08:31 -0500 Subject: [PATCH 177/357] Expand changelog for #4302 --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f0c27d7b2..cc08e6a83 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,9 @@ New features: * Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. * Add query prefixes ``=`` and ``~``. * :doc:`/plugins/discogs`: Permit appending style to genre -* Add auto_keep option to convert. +* :doc:`/plugins/convert`: Add a new `auto_keep` option that automatically + converts files but keeps the *originals* in the library. + :bug:`1840` :bug:`4302` Bug fixes: From bac93e10951242c569af86d8ff8db94a9cc17608 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 11:23:35 +0100 Subject: [PATCH 178/357] artresizer: add a few comments --- beets/util/artresizer.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 0041db230..7cfa6b69e 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -439,13 +439,19 @@ class ArtResizer(metaclass=Shareable): return func(self, maxwidth, path_in, path_out, quality=quality, max_filesize=max_filesize) else: + # Handled by `proxy_url` already. return path_in def deinterlace(self, path_in, path_out=None): + """Deinterlace an image. + + Only available locally. + """ if self.local: func = DEINTERLACE_FUNCS[self.method[0]] return func(self, path_in, path_out) else: + # FIXME: Should probably issue a warning? return path_in def proxy_url(self, maxwidth, url, quality=0): @@ -454,6 +460,7 @@ class ArtResizer(metaclass=Shareable): Otherwise, the URL is returned unmodified. """ if self.local: + # Going to be handled by `resize()`. return url else: return resize_url(url, maxwidth, quality) @@ -474,7 +481,9 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_GET_SIZE[self.method[0]] return func(self, path_in) - return None + else: + # FIXME: Should probably issue a warning? + return None def get_format(self, path_in): """Returns the format of the image as a string. @@ -484,7 +493,9 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_GET_FORMAT[self.method[0]] return func(self, path_in) - return None + else: + # FIXME: Should probably issue a warning? + return None def reformat(self, path_in, new_format, deinterlaced=True): """Converts image to desired format, updating its extension, but @@ -493,6 +504,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if not self.local: + # FIXME: Should probably issue a warning? return path_in new_format = new_format.lower() @@ -529,7 +541,9 @@ class ArtResizer(metaclass=Shareable): if self.local: func = BACKEND_COMPARE[self.method[0]] return func(self, im1, im2, compare_threshold) - return None + else: + # FIXME: Should probably issue a warning? + return None @staticmethod def _check_method(): From 8a969e30410d0b3b47a67de080bf75c4baa21b5e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 11:23:54 +0100 Subject: [PATCH 179/357] artresizer: backend classes part 1: stub classes, version checks --- beets/util/artresizer.py | 244 +++++++++++++++++++++++---------------- beetsplug/thumbnails.py | 26 +++-- test/test_art.py | 4 +- test/test_art_resize.py | 53 ++++++--- test/test_embedart.py | 6 +- test/test_thumbnails.py | 40 ++++--- 6 files changed, 222 insertions(+), 151 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 7cfa6b69e..4369dccd8 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -61,7 +61,98 @@ def temp_file_for(path): return bytestring_path(f.name) -def pil_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, +class LocalBackendNotAvailableError(Exception): + pass + + +_NOT_AVAILABLE = object() + + +class LocalBackend: + @classmethod + def available(cls): + try: + cls.version() + return True + except LocalBackendNotAvailableError: + return False + + +class IMBackend(LocalBackend): + NAME="ImageMagick" + ID=IMAGEMAGICK + _version = None + _legacy = None + + @classmethod + def version(cls): + """Obtain and cache ImageMagick version. + + Raises `LocalBackendNotAvailableError` if not available. + """ + if cls._version is None: + for cmd_name, legacy in (('magick', False), ('convert', True)): + try: + out = util.command_output([cmd_name, "--version"]).stdout + except (subprocess.CalledProcessError, OSError) as exc: + log.debug('ImageMagick version check failed: {}', exc) + cls._version = _NOT_AVAILABLE + else: + if b'imagemagick' in out.lower(): + pattern = br".+ (\d+)\.(\d+)\.(\d+).*" + match = re.search(pattern, out) + if match: + cls._version = (int(match.group(1)), + int(match.group(2)), + int(match.group(3))) + cls._legacy = legacy + + if cls._version is _NOT_AVAILABLE: + raise LocalBackendNotAvailableError() + else: + return cls._version + + def __init__(self): + """Initialize a wrapper around ImageMagick for local image operations. + + Stores the ImageMagick version and legacy flag. If ImageMagick is not + available, raise an Exception. + """ + self.version() + + # Use ImageMagick's magick binary when it's available. + # If it's not, fall back to the older, separate convert + # and identify commands. + if self._legacy: + self.convert_cmd = ['convert'] + self.identify_cmd = ['identify'] + self.compare_cmd = ['compare'] + else: + self.convert_cmd = ['magick'] + self.identify_cmd = ['magick', 'identify'] + self.compare_cmd = ['magick', 'compare'] + + +class PILBackend(LocalBackend): + NAME="PIL" + ID=PIL + + @classmethod + def version(cls): + try: + __import__('PIL', fromlist=['Image']) + except ImportError: + raise LocalBackendNotAvailableError() + + def __init__(self): + """Initialize a wrapper around PIL for local image operations. + + If PIL is not available, raise an Exception. + """ + self.version() + + +def pil_resize(backend, maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. @@ -120,7 +211,7 @@ def pil_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, return path_in -def im_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, +def im_resize(backend, maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """Resize using ImageMagick. @@ -136,7 +227,7 @@ def im_resize(artresizer, maxwidth, path_in, path_out=None, quality=0, # with regards to the height. # ImageMagick already seems to default to no interlace, but we include it # here for the sake of explicitness. - cmd = artresizer.im_convert_cmd + [ + cmd = backend.convert_cmd + [ syspath(path_in, prefix=False), '-resize', f'{maxwidth}x>', '-interlace', 'none', @@ -168,7 +259,7 @@ BACKEND_FUNCS = { } -def pil_getsize(artresizer, path_in): +def pil_getsize(backend, path_in): from PIL import Image try: @@ -180,8 +271,8 @@ def pil_getsize(artresizer, path_in): return None -def im_getsize(artresizer, path_in): - cmd = artresizer.im_identify_cmd + \ +def im_getsize(backend, path_in): + cmd = backend.identify_cmd + \ ['-format', '%w %h', syspath(path_in, prefix=False)] try: @@ -207,7 +298,7 @@ BACKEND_GET_SIZE = { } -def pil_deinterlace(artresizer, path_in, path_out=None): +def pil_deinterlace(backend, path_in, path_out=None): path_out = path_out or temp_file_for(path_in) from PIL import Image @@ -219,10 +310,10 @@ def pil_deinterlace(artresizer, path_in, path_out=None): return path_in -def im_deinterlace(artresizer, path_in, path_out=None): +def im_deinterlace(backend, path_in, path_out=None): path_out = path_out or temp_file_for(path_in) - cmd = artresizer.im_convert_cmd + [ + cmd = backend.convert_cmd + [ syspath(path_in, prefix=False), '-interlace', 'none', syspath(path_out, prefix=False), @@ -241,8 +332,8 @@ DEINTERLACE_FUNCS = { } -def im_get_format(artresizer, filepath): - cmd = artresizer.im_identify_cmd + [ +def im_get_format(backend, filepath): + cmd = backend.identify_cmd + [ '-format', '%[magick]', syspath(filepath) ] @@ -253,7 +344,7 @@ def im_get_format(artresizer, filepath): return None -def pil_get_format(artresizer, filepath): +def pil_get_format(backend, filepath): from PIL import Image, UnidentifiedImageError try: @@ -270,8 +361,8 @@ BACKEND_GET_FORMAT = { } -def im_convert_format(artresizer, source, target, deinterlaced): - cmd = artresizer.im_convert_cmd + [ +def im_convert_format(backend, source, target, deinterlaced): + cmd = backend.convert_cmd + [ syspath(source), *(["-interlace", "none"] if deinterlaced else []), syspath(target), @@ -288,7 +379,7 @@ def im_convert_format(artresizer, source, target, deinterlaced): return source -def pil_convert_format(artresizer, source, target, deinterlaced): +def pil_convert_format(backend, source, target, deinterlaced): from PIL import Image, UnidentifiedImageError try: @@ -307,7 +398,7 @@ BACKEND_CONVERT_IMAGE_FORMAT = { } -def im_compare(artresizer, im1, im2, compare_threshold): +def im_compare(backend, im1, im2, compare_threshold): is_windows = platform.system() == "Windows" # Converting images to grayscale tends to minimize the weight @@ -315,11 +406,11 @@ def im_compare(artresizer, im1, im2, compare_threshold): # to grayscale and then pipe them into the `compare` command. # On Windows, ImageMagick doesn't support the magic \\?\ prefix # on paths, so we pass `prefix=False` to `syspath`. - convert_cmd = artresizer.im_convert_cmd + [ + convert_cmd = backend.convert_cmd + [ syspath(im2, prefix=False), syspath(im1, prefix=False), '-colorspace', 'gray', 'MIFF:-' ] - compare_cmd = artresizer.im_compare_cmd + [ + compare_cmd = backend.compare_cmd + [ '-metric', 'PHASH', '-', 'null:', ] log.debug('comparing images with pipeline {} | {}', @@ -373,7 +464,7 @@ def im_compare(artresizer, im1, im2, compare_threshold): return phash_diff <= compare_threshold -def pil_compare(artresizer, im1, im2, compare_threshold): +def pil_compare(backend, im1, im2, compare_threshold): # It is an error to call this when ArtResizer.can_compare is not True. raise NotImplementedError() @@ -409,22 +500,11 @@ class ArtResizer(metaclass=Shareable): def __init__(self): """Create a resizer object with an inferred method. """ - self.method = self._check_method() - log.debug("artresizer: method is {0}", self.method) - - # Use ImageMagick's magick binary when it's available. If it's - # not, fall back to the older, separate convert and identify - # commands. - if self.method[0] == IMAGEMAGICK: - self.im_legacy = self.method[2] - if self.im_legacy: - self.im_convert_cmd = ['convert'] - self.im_identify_cmd = ['identify'] - self.im_compare_cmd = ['compare'] - else: - self.im_convert_cmd = ['magick'] - self.im_identify_cmd = ['magick', 'identify'] - self.im_compare_cmd = ['magick', 'compare'] + self.local_method = self._check_method() + if self.local_method is None: + log.debug(f"artresizer: method is WEBPROXY") + else: + log.debug(f"artresizer: method is {self.local_method.NAME}") def resize( self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 @@ -435,8 +515,8 @@ class ArtResizer(metaclass=Shareable): For WEBPROXY, returns `path_in` unmodified. """ if self.local: - func = BACKEND_FUNCS[self.method[0]] - return func(self, maxwidth, path_in, path_out, + func = BACKEND_FUNCS[self.local_method] + return func(self.local_method, maxwidth, path_in, path_out, quality=quality, max_filesize=max_filesize) else: # Handled by `proxy_url` already. @@ -448,8 +528,8 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = DEINTERLACE_FUNCS[self.method[0]] - return func(self, path_in, path_out) + func = DEINTERLACE_FUNCS[self.local_method.ID] + return func(self.local_method, path_in, path_out) else: # FIXME: Should probably issue a warning? return path_in @@ -470,7 +550,7 @@ class ArtResizer(metaclass=Shareable): """A boolean indicating whether the resizing method is performed locally (i.e., PIL or ImageMagick). """ - return self.method[0] in BACKEND_FUNCS + return self.local_method is not None def get_size(self, path_in): """Return the size of an image file as an int couple (width, height) @@ -479,11 +559,11 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = BACKEND_GET_SIZE[self.method[0]] - return func(self, path_in) + func = BACKEND_GET_SIZE[self.local_method.ID] + return func(self.local_method, path_in) else: # FIXME: Should probably issue a warning? - return None + return path_in def get_format(self, path_in): """Returns the format of the image as a string. @@ -491,8 +571,8 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = BACKEND_GET_FORMAT[self.method[0]] - return func(self, path_in) + func = BACKEND_GET_FORMAT[self.local_method.ID] + return func(self.local_method, path_in) else: # FIXME: Should probably issue a warning? return None @@ -503,7 +583,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ - if not self.local: + if self.local: # FIXME: Should probably issue a warning? return path_in @@ -515,13 +595,13 @@ class ArtResizer(metaclass=Shareable): fname, ext = os.path.splitext(path_in) path_new = fname + b'.' + new_format.encode('utf8') - func = BACKEND_CONVERT_IMAGE_FORMAT[self.method[0]] + func = BACKEND_CONVERT_IMAGE_FORMAT[self.local_method.ID] # allows the exception to propagate, while still making sure a changed # file path was removed result_path = path_in try: - result_path = func(self, path_in, path_new, deinterlaced) + result_path = func(self.local_method, path_in, path_new, deinterlaced) finally: if result_path != path_in: os.unlink(path_in) @@ -531,7 +611,11 @@ class ArtResizer(metaclass=Shareable): def can_compare(self): """A boolean indicating whether image comparison is available""" - return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7) + return ( + self.local + and self.local_method.ID == IMAGEMAGICK + and self.local_method.version() > (6, 8, 7) + ) def compare(self, im1, im2, compare_threshold): """Return a boolean indicating whether two images are similar. @@ -539,65 +623,23 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = BACKEND_COMPARE[self.method[0]] - return func(self, im1, im2, compare_threshold) + func = BACKEND_COMPARE[self.local_method.ID] + return func(self.local_method, im1, im2, compare_threshold) else: # FIXME: Should probably issue a warning? return None @staticmethod def _check_method(): - """Return a tuple indicating an available method and its version. + """Search availabe methods. - The result has at least two elements: - - The method, eitehr WEBPROXY, PIL, or IMAGEMAGICK. - - The version. - - If the method is IMAGEMAGICK, there is also a third element: a - bool flag indicating whether to use the `magick` binary or - legacy single-purpose executables (`convert`, `identify`, etc.) + If a local backend is availabe, return an instance of the backend + class. Otherwise, when fallback to the web proxy is requird, return + None. """ - version = get_im_version() - if version: - version, legacy = version - return IMAGEMAGICK, version, legacy - - version = get_pil_version() - if version: - return PIL, version - - return WEBPROXY, (0) - - -def get_im_version(): - """Get the ImageMagick version and legacy flag as a pair. Or return - None if ImageMagick is not available. - """ - for cmd_name, legacy in ((['magick'], False), (['convert'], True)): - cmd = cmd_name + ['--version'] - try: - out = util.command_output(cmd).stdout - except (subprocess.CalledProcessError, OSError) as exc: - log.debug('ImageMagick version check failed: {}', exc) - else: - if b'imagemagick' in out.lower(): - pattern = br".+ (\d+)\.(\d+)\.(\d+).*" - match = re.search(pattern, out) - if match: - version = (int(match.group(1)), - int(match.group(2)), - int(match.group(3))) - return version, legacy + return IMBackend() + return PILBackend() + except LocalBackendNotAvailableError: + return None - return None - - -def get_pil_version(): - """Get the PIL/Pillow version, or None if it is unavailable. - """ - try: - __import__('PIL', fromlist=['Image']) - return (0,) - except ImportError: - return None diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index 6bd9cbac6..e3cf6e6a2 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -32,7 +32,7 @@ from xdg import BaseDirectory from beets.plugins import BeetsPlugin from beets.ui import Subcommand, decargs from beets import util -from beets.util.artresizer import ArtResizer, get_im_version, get_pil_version +from beets.util.artresizer import ArtResizer, IMBackend, PILBackend BASE_DIR = os.path.join(BaseDirectory.xdg_cache_home, "thumbnails") @@ -90,14 +90,18 @@ class ThumbnailsPlugin(BeetsPlugin): if not os.path.exists(dir): os.makedirs(dir) - if get_im_version(): + # FIXME: Should we have our own backend instance? + self.backend = ArtResizer.shared.local_method + if isinstance(self.backend, IMBackend): self.write_metadata = write_metadata_im - tool = "IM" - else: - assert get_pil_version() # since we're local + elif isinstance(self.backend, PILBackend): self.write_metadata = write_metadata_pil - tool = "PIL" - self._log.debug("using {0} to write metadata", tool) + else: + # since we're local + raise RuntimeError( + f"Thumbnails: Unexpected ArtResizer backend {self.backend!r}." + ) + self._log.debug(f"using {self.backend.NAME} to write metadata") uri_getter = GioURI() if not uri_getter.available: @@ -171,7 +175,7 @@ class ThumbnailsPlugin(BeetsPlugin): metadata = {"Thumb::URI": self.get_uri(album.artpath), "Thumb::MTime": str(mtime)} try: - self.write_metadata(image_path, metadata) + self.write_metadata(self.backend, image_path, metadata) except Exception: self._log.exception("could not write metadata to {0}", util.displayable_path(image_path)) @@ -188,16 +192,16 @@ class ThumbnailsPlugin(BeetsPlugin): self._log.debug("Wrote file {0}", util.displayable_path(outfilename)) -def write_metadata_im(file, metadata): +def write_metadata_im(im_backend, file, metadata): """Enrich the file metadata with `metadata` dict thanks to IM.""" - command = ['convert', file] + \ + command = im_backend.convert_cmd + [file] + \ list(chain.from_iterable(('-set', k, v) for k, v in metadata.items())) + [file] util.command_output(command) return True -def write_metadata_pil(file, metadata): +def write_metadata_pil(pil_backend, file, metadata): """Enrich the file metadata with `metadata` dict thanks to PIL.""" from PIL import Image, PngImagePlugin im = Image.open(file) diff --git a/test/test_art.py b/test/test_art.py index 498c4cedc..b32285e70 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -31,7 +31,7 @@ from beets import library from beets import importer from beets import logging from beets import util -from beets.util.artresizer import ArtResizer, WEBPROXY +from beets.util.artresizer import ArtResizer import confuse @@ -787,7 +787,7 @@ class ArtForAlbumTest(UseThePlugin): """Skip the test if the art resizer doesn't have ImageMagick or PIL (so comparisons and measurements are unavailable). """ - if ArtResizer.shared.method[0] == WEBPROXY: + if not ArtResizer.shared.local: self.skipTest("ArtResizer has no local imaging backend available") def test_respect_minwidth(self): diff --git a/test/test_art_resize.py b/test/test_art_resize.py index 4600bab77..9d3be19e7 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -22,16 +22,34 @@ from test import _common from test.helper import TestHelper from beets.util import command_output, syspath from beets.util.artresizer import ( + IMBackend, + PILBackend, pil_resize, im_resize, - get_im_version, - get_pil_version, pil_deinterlace, im_deinterlace, - ArtResizer, ) +class DummyIMBackend(IMBackend): + """An `IMBackend` which pretends that ImageMagick is available, and has + a sufficiently recent version to support image comparison. + """ + def __init__(self): + self.version = (7, 0, 0) + self.legacy = False + self.convert_cmd = ['magick'] + self.identify_cmd = ['magick', 'identify'] + self.compare_cmd = ['magick', 'compare'] + + +class DummyPILBackend(PILBackend): + """An `PILBackend` which pretends that PIL is available. + """ + def __init__(self): + pass + + class ArtResizerFileSizeTest(_common.TestCase, TestHelper): """Unittest test case for Art Resizer to a specific filesize.""" @@ -46,11 +64,11 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): """Called after each test, unloading all plugins.""" self.teardown_beets() - def _test_img_resize(self, resize_func): + def _test_img_resize(self, backend, resize_func): """Test resizing based on file size, given a resize_func.""" # Check quality setting unaffected by new parameter im_95_qual = resize_func( - ArtResizer.shared, + backend, 225, self.IMG_225x225, quality=95, @@ -61,7 +79,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): # Attempt a lower filesize with same quality im_a = resize_func( - ArtResizer.shared, + backend, 225, self.IMG_225x225, quality=95, @@ -74,7 +92,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): # Attempt with lower initial quality im_75_qual = resize_func( - ArtResizer.shared, + backend, 225, self.IMG_225x225, quality=75, @@ -83,7 +101,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): self.assertExists(im_75_qual) im_b = resize_func( - ArtResizer.shared, + backend, 225, self.IMG_225x225, quality=95, @@ -94,37 +112,38 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): self.assertLess(os.stat(syspath(im_b)).st_size, os.stat(syspath(im_75_qual)).st_size) - @unittest.skipUnless(get_pil_version(), "PIL not available") + @unittest.skipUnless(PILBackend.available(), "PIL not available") def test_pil_file_resize(self): """Test PIL resize function is lowering file size.""" - self._test_img_resize(pil_resize) + self._test_img_resize(PILBackend(), pil_resize) - @unittest.skipUnless(get_im_version(), "ImageMagick not available") + @unittest.skipUnless(IMBackend.available(), "ImageMagick not available") def test_im_file_resize(self): """Test IM resize function is lowering file size.""" - self._test_img_resize(im_resize) + self._test_img_resize(IMBackend(), im_resize) - @unittest.skipUnless(get_pil_version(), "PIL not available") + @unittest.skipUnless(PILBackend.available(), "PIL not available") def test_pil_file_deinterlace(self): """Test PIL deinterlace function. Check if pil_deinterlace function returns images that are non-progressive """ - path = pil_deinterlace(ArtResizer.shared, self.IMG_225x225) + path = pil_deinterlace(PILBackend(), self.IMG_225x225) from PIL import Image with Image.open(path) as img: self.assertFalse('progression' in img.info) - @unittest.skipUnless(get_im_version(), "ImageMagick not available") + @unittest.skipUnless(IMBackend.available(), "ImageMagick not available") def test_im_file_deinterlace(self): """Test ImageMagick deinterlace function. Check if im_deinterlace function returns images that are non-progressive. """ - path = im_deinterlace(ArtResizer.shared, self.IMG_225x225) - cmd = ArtResizer.shared.im_identify_cmd + [ + im = IMBackend() + path = im_deinterlace(im, self.IMG_225x225) + cmd = im.identify_cmd + [ '-format', '%[interlace]', syspath(path, prefix=False), ] out = command_output(cmd).stdout diff --git a/test/test_embedart.py b/test/test_embedart.py index 0fed08f98..b3f61babe 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -21,6 +21,7 @@ import unittest from test import _common from test.helper import TestHelper +from test.test_art_resize import DummyIMBackend from mediafile import MediaFile from beets import config, logging, ui @@ -220,9 +221,8 @@ class DummyArtResizer(ArtResizer): """An `ArtResizer` which pretends that ImageMagick is available, and has a sufficiently recent version to support image comparison. """ - @staticmethod - def _check_method(): - return artresizer.IMAGEMAGICK, (7, 0, 0), True + def __init__(self): + self.local_method = DummyIMBackend() @patch('beets.util.artresizer.subprocess') diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index e8ab21d72..376969e93 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -20,6 +20,7 @@ from shutil import rmtree import unittest from test.helper import TestHelper +from test.test_art_resize import DummyIMBackend, DummyPILBackend from beets.util import bytestring_path from beetsplug.thumbnails import (ThumbnailsPlugin, NORMAL_DIR, LARGE_DIR, @@ -37,18 +38,21 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): @patch('beetsplug.thumbnails.util') def test_write_metadata_im(self, mock_util): metadata = {"a": "A", "b": "B"} - write_metadata_im("foo", metadata) + im = DummyIMBackend() + write_metadata_im(im, "foo", metadata) try: - command = "convert foo -set a A -set b B foo".split(' ') + command = im.convert_cmd + "foo -set a A -set b B foo".split() mock_util.command_output.assert_called_once_with(command) except AssertionError: - command = "convert foo -set b B -set a A foo".split(' ') + command = im.convert_cmd + "foo -set b B -set a A foo".split() mock_util.command_output.assert_called_once_with(command) @patch('beetsplug.thumbnails.ThumbnailsPlugin._check_local_ok') @patch('beetsplug.thumbnails.os.stat') def test_add_tags(self, mock_stat, _): plugin = ThumbnailsPlugin() + # backend is not set due to _check_local_ok being mocked + plugin.backend = "DummyBackend" plugin.write_metadata = Mock() plugin.get_uri = Mock(side_effect={b"/path/to/cover": "COVER_URI"}.__getitem__) @@ -59,24 +63,26 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): metadata = {"Thumb::URI": "COVER_URI", "Thumb::MTime": "12345"} - plugin.write_metadata.assert_called_once_with(b"/path/to/thumbnail", - metadata) + plugin.write_metadata.assert_called_once_with( + plugin.backend, + b"/path/to/thumbnail", + metadata, + ) mock_stat.assert_called_once_with(album.artpath) @patch('beetsplug.thumbnails.os') @patch('beetsplug.thumbnails.ArtResizer') - @patch('beetsplug.thumbnails.get_im_version') - @patch('beetsplug.thumbnails.get_pil_version') @patch('beetsplug.thumbnails.GioURI') - def test_check_local_ok(self, mock_giouri, mock_pil, mock_im, - mock_artresizer, mock_os): + def test_check_local_ok(self, mock_giouri, mock_artresizer, mock_os): # test local resizing capability mock_artresizer.shared.local = False + mock_artresizer.shared.local_method = None plugin = ThumbnailsPlugin() self.assertFalse(plugin._check_local_ok()) # test dirs creation mock_artresizer.shared.local = True + mock_artresizer.shared.local_method = DummyIMBackend() def exists(path): if path == NORMAL_DIR: @@ -91,18 +97,18 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): # test metadata writer function mock_os.path.exists = lambda _: True - mock_pil.return_value = False - mock_im.return_value = False - with self.assertRaises(AssertionError): + + mock_artresizer.shared.local = True + mock_artresizer.shared.local_method = None + with self.assertRaises(RuntimeError): ThumbnailsPlugin() - mock_pil.return_value = True + mock_artresizer.shared.local = True + mock_artresizer.shared.local_method = DummyPILBackend() self.assertEqual(ThumbnailsPlugin().write_metadata, write_metadata_pil) - mock_im.return_value = True - self.assertEqual(ThumbnailsPlugin().write_metadata, write_metadata_im) - - mock_pil.return_value = False + mock_artresizer.shared.local = True + mock_artresizer.shared.local_method = DummyIMBackend() self.assertEqual(ThumbnailsPlugin().write_metadata, write_metadata_im) self.assertTrue(ThumbnailsPlugin()._check_local_ok()) From c45d2e28a61ccdb5f78c8f0f8c50fa13c5fd8ab3 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:23:37 +0100 Subject: [PATCH 180/357] artresizer: move resize functions to backend classes --- beets/util/artresizer.py | 193 +++++++++++++++++++-------------------- test/test_art_resize.py | 20 ++-- 2 files changed, 99 insertions(+), 114 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 4369dccd8..801f2fdb5 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -132,6 +132,47 @@ class IMBackend(LocalBackend): self.identify_cmd = ['magick', 'identify'] self.compare_cmd = ['magick', 'compare'] + def resize(self, maxwidth, path_in, path_out=None, quality=0, + max_filesize=0): + """Resize using ImageMagick. + + Use the ``magick`` program or ``convert`` on older versions. Return + the output path of resized image. + """ + path_out = path_out or temp_file_for(path_in) + log.debug('artresizer: ImageMagick resizing {0} to {1}', + displayable_path(path_in), displayable_path(path_out)) + + # "-resize WIDTHx>" shrinks images with the width larger + # than the given width while maintaining the aspect ratio + # with regards to the height. + # ImageMagick already seems to default to no interlace, but we include it + # here for the sake of explicitness. + cmd = self.convert_cmd + [ + syspath(path_in, prefix=False), + '-resize', f'{maxwidth}x>', + '-interlace', 'none', + ] + + if quality > 0: + cmd += ['-quality', f'{quality}'] + + # "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to + # SIZE in bytes. + if max_filesize > 0: + cmd += ['-define', f'jpeg:extent={max_filesize}b'] + + cmd.append(syspath(path_out, prefix=False)) + + try: + util.command_output(cmd) + except subprocess.CalledProcessError: + log.warning('artresizer: IM convert failed for {0}', + displayable_path(path_in)) + return path_in + + return path_out + class PILBackend(LocalBackend): NAME="PIL" @@ -151,112 +192,63 @@ class PILBackend(LocalBackend): """ self.version() + def resize(self, maxwidth, path_in, path_out=None, quality=0, + max_filesize=0): + """Resize using Python Imaging Library (PIL). Return the output path + of resized image. + """ + path_out = path_out or temp_file_for(path_in) + from PIL import Image -def pil_resize(backend, maxwidth, path_in, path_out=None, quality=0, - max_filesize=0): - """Resize using Python Imaging Library (PIL). Return the output path - of resized image. - """ - path_out = path_out or temp_file_for(path_in) - from PIL import Image + log.debug('artresizer: PIL resizing {0} to {1}', + displayable_path(path_in), displayable_path(path_out)) - log.debug('artresizer: PIL resizing {0} to {1}', - displayable_path(path_in), displayable_path(path_out)) + try: + im = Image.open(syspath(path_in)) + size = maxwidth, maxwidth + im.thumbnail(size, Image.ANTIALIAS) - try: - im = Image.open(syspath(path_in)) - size = maxwidth, maxwidth - im.thumbnail(size, Image.ANTIALIAS) + if quality == 0: + # Use PIL's default quality. + quality = -1 - if quality == 0: - # Use PIL's default quality. - quality = -1 + # progressive=False only affects JPEGs and is the default, + # but we include it here for explicitness. + im.save(py3_path(path_out), quality=quality, progressive=False) - # progressive=False only affects JPEGs and is the default, - # but we include it here for explicitness. - im.save(py3_path(path_out), quality=quality, progressive=False) + if max_filesize > 0: + # If maximum filesize is set, we attempt to lower the quality of + # jpeg conversion by a proportional amount, up to 3 attempts + # First, set the maximum quality to either provided, or 95 + if quality > 0: + lower_qual = quality + else: + lower_qual = 95 + for i in range(5): + # 5 attempts is an abitrary choice + filesize = os.stat(syspath(path_out)).st_size + log.debug("PIL Pass {0} : Output size: {1}B", i, filesize) + if filesize <= max_filesize: + return path_out + # The relationship between filesize & quality will be + # image dependent. + lower_qual -= 10 + # Restrict quality dropping below 10 + if lower_qual < 10: + lower_qual = 10 + # Use optimize flag to improve filesize decrease + im.save(py3_path(path_out), quality=lower_qual, + optimize=True, progressive=False) + log.warning("PIL Failed to resize file to below {0}B", + max_filesize) + return path_out - if max_filesize > 0: - # If maximum filesize is set, we attempt to lower the quality of - # jpeg conversion by a proportional amount, up to 3 attempts - # First, set the maximum quality to either provided, or 95 - if quality > 0: - lower_qual = quality else: - lower_qual = 95 - for i in range(5): - # 5 attempts is an abitrary choice - filesize = os.stat(syspath(path_out)).st_size - log.debug("PIL Pass {0} : Output size: {1}B", i, filesize) - if filesize <= max_filesize: - return path_out - # The relationship between filesize & quality will be - # image dependent. - lower_qual -= 10 - # Restrict quality dropping below 10 - if lower_qual < 10: - lower_qual = 10 - # Use optimize flag to improve filesize decrease - im.save(py3_path(path_out), quality=lower_qual, - optimize=True, progressive=False) - log.warning("PIL Failed to resize file to below {0}B", - max_filesize) - return path_out - - else: - return path_out - except OSError: - log.error("PIL cannot create thumbnail for '{0}'", - displayable_path(path_in)) - return path_in - - -def im_resize(backend, maxwidth, path_in, path_out=None, quality=0, - max_filesize=0): - """Resize using ImageMagick. - - Use the ``magick`` program or ``convert`` on older versions. Return - the output path of resized image. - """ - path_out = path_out or temp_file_for(path_in) - log.debug('artresizer: ImageMagick resizing {0} to {1}', - displayable_path(path_in), displayable_path(path_out)) - - # "-resize WIDTHx>" shrinks images with the width larger - # than the given width while maintaining the aspect ratio - # with regards to the height. - # ImageMagick already seems to default to no interlace, but we include it - # here for the sake of explicitness. - cmd = backend.convert_cmd + [ - syspath(path_in, prefix=False), - '-resize', f'{maxwidth}x>', - '-interlace', 'none', - ] - - if quality > 0: - cmd += ['-quality', f'{quality}'] - - # "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to - # SIZE in bytes. - if max_filesize > 0: - cmd += ['-define', f'jpeg:extent={max_filesize}b'] - - cmd.append(syspath(path_out, prefix=False)) - - try: - util.command_output(cmd) - except subprocess.CalledProcessError: - log.warning('artresizer: IM convert failed for {0}', - displayable_path(path_in)) - return path_in - - return path_out - - -BACKEND_FUNCS = { - PIL: pil_resize, - IMAGEMAGICK: im_resize, -} + return path_out + except OSError: + log.error("PIL cannot create thumbnail for '{0}'", + displayable_path(path_in)) + return path_in def pil_getsize(backend, path_in): @@ -515,8 +507,7 @@ class ArtResizer(metaclass=Shareable): For WEBPROXY, returns `path_in` unmodified. """ if self.local: - func = BACKEND_FUNCS[self.local_method] - return func(self.local_method, maxwidth, path_in, path_out, + return self.local_method.resize(maxwidth, path_in, path_out, quality=quality, max_filesize=max_filesize) else: # Handled by `proxy_url` already. diff --git a/test/test_art_resize.py b/test/test_art_resize.py index 9d3be19e7..80604ba76 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -24,8 +24,6 @@ from beets.util import command_output, syspath from beets.util.artresizer import ( IMBackend, PILBackend, - pil_resize, - im_resize, pil_deinterlace, im_deinterlace, ) @@ -64,11 +62,10 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): """Called after each test, unloading all plugins.""" self.teardown_beets() - def _test_img_resize(self, backend, resize_func): + def _test_img_resize(self, backend): """Test resizing based on file size, given a resize_func.""" # Check quality setting unaffected by new parameter - im_95_qual = resize_func( - backend, + im_95_qual = backend.resize( 225, self.IMG_225x225, quality=95, @@ -78,8 +75,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): self.assertExists(im_95_qual) # Attempt a lower filesize with same quality - im_a = resize_func( - backend, + im_a = backend.resize( 225, self.IMG_225x225, quality=95, @@ -91,8 +87,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): os.stat(syspath(im_95_qual)).st_size) # Attempt with lower initial quality - im_75_qual = resize_func( - backend, + im_75_qual = backend.resize( 225, self.IMG_225x225, quality=75, @@ -100,8 +95,7 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): ) self.assertExists(im_75_qual) - im_b = resize_func( - backend, + im_b = backend.resize( 225, self.IMG_225x225, quality=95, @@ -115,12 +109,12 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): @unittest.skipUnless(PILBackend.available(), "PIL not available") def test_pil_file_resize(self): """Test PIL resize function is lowering file size.""" - self._test_img_resize(PILBackend(), pil_resize) + self._test_img_resize(PILBackend()) @unittest.skipUnless(IMBackend.available(), "ImageMagick not available") def test_im_file_resize(self): """Test IM resize function is lowering file size.""" - self._test_img_resize(IMBackend(), im_resize) + self._test_img_resize(IMBackend()) @unittest.skipUnless(PILBackend.available(), "PIL not available") def test_pil_file_deinterlace(self): From 279c081f23c5962be36243c399a5eb7c89f764e9 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:27:14 +0100 Subject: [PATCH 181/357] artresizer: move get_size functions to backend classes also rename getsize -> get_size for consistency with the ArtResizer method --- beets/util/artresizer.py | 70 ++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 801f2fdb5..566087568 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -173,6 +173,27 @@ class IMBackend(LocalBackend): return path_out + def get_size(self, path_in): + cmd = self.identify_cmd + [ + '-format', '%w %h', syspath(path_in, prefix=False) + ] + + try: + out = util.command_output(cmd).stdout + except subprocess.CalledProcessError as exc: + log.warning('ImageMagick size query failed') + log.debug( + '`convert` exited with (status {}) when ' + 'getting size with command {}:\n{}', + exc.returncode, cmd, exc.output.strip() + ) + return None + try: + return tuple(map(int, out.split(b' '))) + except IndexError: + log.warning('Could not understand IM output: {0!r}', out) + return None + class PILBackend(LocalBackend): NAME="PIL" @@ -250,44 +271,16 @@ class PILBackend(LocalBackend): displayable_path(path_in)) return path_in + def get_size(self, path_in): + from PIL import Image -def pil_getsize(backend, path_in): - from PIL import Image - - try: - im = Image.open(syspath(path_in)) - return im.size - except OSError as exc: - log.error("PIL could not read file {}: {}", - displayable_path(path_in), exc) - return None - - -def im_getsize(backend, path_in): - cmd = backend.identify_cmd + \ - ['-format', '%w %h', syspath(path_in, prefix=False)] - - try: - out = util.command_output(cmd).stdout - except subprocess.CalledProcessError as exc: - log.warning('ImageMagick size query failed') - log.debug( - '`convert` exited with (status {}) when ' - 'getting size with command {}:\n{}', - exc.returncode, cmd, exc.output.strip() - ) - return None - try: - return tuple(map(int, out.split(b' '))) - except IndexError: - log.warning('Could not understand IM output: {0!r}', out) - return None - - -BACKEND_GET_SIZE = { - PIL: pil_getsize, - IMAGEMAGICK: im_getsize, -} + try: + im = Image.open(syspath(path_in)) + return im.size + except OSError as exc: + log.error("PIL could not read file {}: {}", + displayable_path(path_in), exc) + return None def pil_deinterlace(backend, path_in, path_out=None): @@ -550,8 +543,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = BACKEND_GET_SIZE[self.local_method.ID] - return func(self.local_method, path_in) + return self.local_method.get_size(path_in) else: # FIXME: Should probably issue a warning? return path_in From 1b92dea9955c447a90ed6cf817460c81119e4bf0 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:30:39 +0100 Subject: [PATCH 182/357] artresizer: move deinterlace functions to backend classes --- beets/util/artresizer.py | 60 +++++++++++++++++----------------------- test/test_art_resize.py | 10 +++---- 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 566087568..d7fa115e9 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -194,6 +194,22 @@ class IMBackend(LocalBackend): log.warning('Could not understand IM output: {0!r}', out) return None + def deinterlace(self, path_in, path_out=None): + path_out = path_out or temp_file_for(path_in) + + cmd = self.convert_cmd + [ + syspath(path_in, prefix=False), + '-interlace', 'none', + syspath(path_out, prefix=False), + ] + + try: + util.command_output(cmd) + return path_out + except subprocess.CalledProcessError: + # FIXME: add a warning + return path_in + class PILBackend(LocalBackend): NAME="PIL" @@ -282,39 +298,16 @@ class PILBackend(LocalBackend): displayable_path(path_in), exc) return None + def deinterlace(self, path_in, path_out=None): + path_out = path_out or temp_file_for(path_in) + from PIL import Image -def pil_deinterlace(backend, path_in, path_out=None): - path_out = path_out or temp_file_for(path_in) - from PIL import Image - - try: - im = Image.open(syspath(path_in)) - im.save(py3_path(path_out), progressive=False) - return path_out - except IOError: - return path_in - - -def im_deinterlace(backend, path_in, path_out=None): - path_out = path_out or temp_file_for(path_in) - - cmd = backend.convert_cmd + [ - syspath(path_in, prefix=False), - '-interlace', 'none', - syspath(path_out, prefix=False), - ] - - try: - util.command_output(cmd) - return path_out - except subprocess.CalledProcessError: - return path_in - - -DEINTERLACE_FUNCS = { - PIL: pil_deinterlace, - IMAGEMAGICK: im_deinterlace, -} + try: + im = Image.open(syspath(path_in)) + im.save(py3_path(path_out), progressive=False) + return path_out + except IOError: + return path_in def im_get_format(backend, filepath): @@ -512,8 +505,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = DEINTERLACE_FUNCS[self.local_method.ID] - return func(self.local_method, path_in, path_out) + return self.local_method.deinterlace(path_in, path_out) else: # FIXME: Should probably issue a warning? return path_in diff --git a/test/test_art_resize.py b/test/test_art_resize.py index 80604ba76..57a7121f7 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -24,8 +24,6 @@ from beets.util import command_output, syspath from beets.util.artresizer import ( IMBackend, PILBackend, - pil_deinterlace, - im_deinterlace, ) @@ -120,10 +118,10 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): def test_pil_file_deinterlace(self): """Test PIL deinterlace function. - Check if pil_deinterlace function returns images + Check if the `PILBackend.deinterlace()` function returns images that are non-progressive """ - path = pil_deinterlace(PILBackend(), self.IMG_225x225) + path = PILBackend().deinterlace(self.IMG_225x225) from PIL import Image with Image.open(path) as img: self.assertFalse('progression' in img.info) @@ -132,11 +130,11 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): def test_im_file_deinterlace(self): """Test ImageMagick deinterlace function. - Check if im_deinterlace function returns images + Check if the `IMBackend.deinterlace()` function returns images that are non-progressive. """ im = IMBackend() - path = im_deinterlace(im, self.IMG_225x225) + path = im.deinterlace(self.IMG_225x225) cmd = im.identify_cmd + [ '-format', '%[interlace]', syspath(path, prefix=False), ] From a781e6a9bf4870b9c93edd27619382198a890285 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:37:31 +0100 Subject: [PATCH 183/357] artresizer: move get_format functions to backend classes --- beets/util/artresizer.py | 49 ++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index d7fa115e9..d413c9906 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -210,6 +210,17 @@ class IMBackend(LocalBackend): # FIXME: add a warning return path_in + def get_format(self, filepath): + cmd = self.identify_cmd + [ + '-format', '%[magick]', + syspath(filepath) + ] + + try: + return util.command_output(cmd).stdout + except subprocess.CalledProcessError: + return None + class PILBackend(LocalBackend): NAME="PIL" @@ -309,34 +320,15 @@ class PILBackend(LocalBackend): except IOError: return path_in + def get_format(self, filepath): + from PIL import Image, UnidentifiedImageError -def im_get_format(backend, filepath): - cmd = backend.identify_cmd + [ - '-format', '%[magick]', - syspath(filepath) - ] - - try: - return util.command_output(cmd).stdout - except subprocess.CalledProcessError: - return None - - -def pil_get_format(backend, filepath): - from PIL import Image, UnidentifiedImageError - - try: - with Image.open(syspath(filepath)) as im: - return im.format - except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError): - log.exception("failed to detect image format for {}", filepath) - return None - - -BACKEND_GET_FORMAT = { - PIL: pil_get_format, - IMAGEMAGICK: im_get_format, -} + try: + with Image.open(syspath(filepath)) as im: + return im.format + except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError): + log.exception("failed to detect image format for {}", filepath) + return None def im_convert_format(backend, source, target, deinterlaced): @@ -546,8 +538,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = BACKEND_GET_FORMAT[self.local_method.ID] - return func(self.local_method, path_in) + return self.local_method.get_format(path_in) else: # FIXME: Should probably issue a warning? return None From 68e762c203ed3137eba85ee438355f3014cceea0 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:39:40 +0100 Subject: [PATCH 184/357] artresizer: move convert_format functions to backend classes --- beets/util/artresizer.py | 66 ++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index d413c9906..671a286bb 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -221,6 +221,23 @@ class IMBackend(LocalBackend): except subprocess.CalledProcessError: return None + def convert_format(self, source, target, deinterlaced): + cmd = self.convert_cmd + [ + syspath(source), + *(["-interlace", "none"] if deinterlaced else []), + syspath(target), + ] + + try: + subprocess.check_call( + cmd, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL + ) + return target + except subprocess.CalledProcessError: + return source + class PILBackend(LocalBackend): NAME="PIL" @@ -331,41 +348,17 @@ class PILBackend(LocalBackend): return None -def im_convert_format(backend, source, target, deinterlaced): - cmd = backend.convert_cmd + [ - syspath(source), - *(["-interlace", "none"] if deinterlaced else []), - syspath(target), - ] + def convert_format(self, source, target, deinterlaced): + from PIL import Image, UnidentifiedImageError - try: - subprocess.check_call( - cmd, - stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL - ) - return target - except subprocess.CalledProcessError: - return source - - -def pil_convert_format(backend, source, target, deinterlaced): - from PIL import Image, UnidentifiedImageError - - try: - with Image.open(syspath(source)) as im: - im.save(py3_path(target), progressive=not deinterlaced) - return target - except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError, - OSError): - log.exception("failed to convert image {} -> {}", source, target) - return source - - -BACKEND_CONVERT_IMAGE_FORMAT = { - PIL: pil_convert_format, - IMAGEMAGICK: im_convert_format, -} + try: + with Image.open(syspath(source)) as im: + im.save(py3_path(target), progressive=not deinterlaced) + return target + except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError, + OSError): + log.exception("failed to convert image {} -> {}", source, target) + return source def im_compare(backend, im1, im2, compare_threshold): @@ -561,13 +554,14 @@ class ArtResizer(metaclass=Shareable): fname, ext = os.path.splitext(path_in) path_new = fname + b'.' + new_format.encode('utf8') - func = BACKEND_CONVERT_IMAGE_FORMAT[self.local_method.ID] # allows the exception to propagate, while still making sure a changed # file path was removed result_path = path_in try: - result_path = func(self.local_method, path_in, path_new, deinterlaced) + result_path = self.local_method.convert_format( + path_in, path_new, deinterlaced + ) finally: if result_path != path_in: os.unlink(path_in) From cb9a765997e920a4061504426260776431ec29de Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:42:35 +0100 Subject: [PATCH 185/357] artresizer: move compare functions to backend classes --- beets/util/artresizer.py | 147 ++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 78 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 671a286bb..ac03e335d 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -238,6 +238,71 @@ class IMBackend(LocalBackend): except subprocess.CalledProcessError: return source + def compare(self, im1, im2, compare_threshold): + is_windows = platform.system() == "Windows" + + # Converting images to grayscale tends to minimize the weight + # of colors in the diff score. So we first convert both images + # to grayscale and then pipe them into the `compare` command. + # On Windows, ImageMagick doesn't support the magic \\?\ prefix + # on paths, so we pass `prefix=False` to `syspath`. + convert_cmd = self.convert_cmd + [ + syspath(im2, prefix=False), syspath(im1, prefix=False), + '-colorspace', 'gray', 'MIFF:-' + ] + compare_cmd = self.compare_cmd + [ + '-metric', 'PHASH', '-', 'null:', + ] + log.debug('comparing images with pipeline {} | {}', + convert_cmd, compare_cmd) + convert_proc = subprocess.Popen( + convert_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=not is_windows, + ) + compare_proc = subprocess.Popen( + compare_cmd, + stdin=convert_proc.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=not is_windows, + ) + + # Check the convert output. We're not interested in the + # standard output; that gets piped to the next stage. + convert_proc.stdout.close() + convert_stderr = convert_proc.stderr.read() + convert_proc.stderr.close() + convert_proc.wait() + if convert_proc.returncode: + log.debug( + 'ImageMagick convert failed with status {}: {!r}', + convert_proc.returncode, + convert_stderr, + ) + return None + + # Check the compare output. + stdout, stderr = compare_proc.communicate() + if compare_proc.returncode: + if compare_proc.returncode != 1: + log.debug('ImageMagick compare failed: {0}, {1}', + displayable_path(im2), displayable_path(im1)) + return None + out_str = stderr + else: + out_str = stdout + + try: + phash_diff = float(out_str) + except ValueError: + log.debug('IM output is not a number: {0!r}', out_str) + return None + + log.debug('ImageMagick compare score: {0}', phash_diff) + return phash_diff <= compare_threshold + class PILBackend(LocalBackend): NAME="PIL" @@ -360,82 +425,9 @@ class PILBackend(LocalBackend): log.exception("failed to convert image {} -> {}", source, target) return source - -def im_compare(backend, im1, im2, compare_threshold): - is_windows = platform.system() == "Windows" - - # Converting images to grayscale tends to minimize the weight - # of colors in the diff score. So we first convert both images - # to grayscale and then pipe them into the `compare` command. - # On Windows, ImageMagick doesn't support the magic \\?\ prefix - # on paths, so we pass `prefix=False` to `syspath`. - convert_cmd = backend.convert_cmd + [ - syspath(im2, prefix=False), syspath(im1, prefix=False), - '-colorspace', 'gray', 'MIFF:-' - ] - compare_cmd = backend.compare_cmd + [ - '-metric', 'PHASH', '-', 'null:', - ] - log.debug('comparing images with pipeline {} | {}', - convert_cmd, compare_cmd) - convert_proc = subprocess.Popen( - convert_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=not is_windows, - ) - compare_proc = subprocess.Popen( - compare_cmd, - stdin=convert_proc.stdout, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=not is_windows, - ) - - # Check the convert output. We're not interested in the - # standard output; that gets piped to the next stage. - convert_proc.stdout.close() - convert_stderr = convert_proc.stderr.read() - convert_proc.stderr.close() - convert_proc.wait() - if convert_proc.returncode: - log.debug( - 'ImageMagick convert failed with status {}: {!r}', - convert_proc.returncode, - convert_stderr, - ) - return None - - # Check the compare output. - stdout, stderr = compare_proc.communicate() - if compare_proc.returncode: - if compare_proc.returncode != 1: - log.debug('ImageMagick compare failed: {0}, {1}', - displayable_path(im2), displayable_path(im1)) - return None - out_str = stderr - else: - out_str = stdout - - try: - phash_diff = float(out_str) - except ValueError: - log.debug('IM output is not a number: {0!r}', out_str) - return None - - log.debug('ImageMagick compare score: {0}', phash_diff) - return phash_diff <= compare_threshold - - -def pil_compare(backend, im1, im2, compare_threshold): - # It is an error to call this when ArtResizer.can_compare is not True. - raise NotImplementedError() - - -BACKEND_COMPARE = { - PIL: pil_compare, - IMAGEMAGICK: im_compare, -} + def compare(self, im1, im2, compare_threshold): + # It is an error to call this when ArtResizer.can_compare is not True. + raise NotImplementedError() class Shareable(type): @@ -583,8 +575,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ if self.local: - func = BACKEND_COMPARE[self.local_method.ID] - return func(self.local_method, im1, im2, compare_threshold) + return self.local_method.compare(im1, im2, compare_threshold) else: # FIXME: Should probably issue a warning? return None From b097b1950616504cb5ba7cd60e9a2f7878b4a5a9 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:44:41 +0100 Subject: [PATCH 186/357] artresizer: move can_compare property to backend classes --- beets/util/artresizer.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index ac03e335d..5dad38dfc 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -238,6 +238,10 @@ class IMBackend(LocalBackend): except subprocess.CalledProcessError: return source + @property + def can_compare(self): + return self.version() > (6, 8, 7) + def compare(self, im1, im2, compare_threshold): is_windows = platform.system() == "Windows" @@ -425,6 +429,10 @@ class PILBackend(LocalBackend): log.exception("failed to convert image {} -> {}", source, target) return source + @property + def can_compare(self): + return False + def compare(self, im1, im2, compare_threshold): # It is an error to call this when ArtResizer.can_compare is not True. raise NotImplementedError() @@ -563,11 +571,10 @@ class ArtResizer(metaclass=Shareable): def can_compare(self): """A boolean indicating whether image comparison is available""" - return ( - self.local - and self.local_method.ID == IMAGEMAGICK - and self.local_method.version() > (6, 8, 7) - ) + if self.local: + return self.local_method.can_compare + else: + return False def compare(self, im1, im2, compare_threshold): """Return a boolean indicating whether two images are similar. From bc364bc7d42edccc4bd3628e0d1d266c1513eecb Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:51:06 +0100 Subject: [PATCH 187/357] artresizer: merge _check_method into ArtResizer.__init__ more concise since _check_method became much shorter due to introduction of backend classes --- beets/util/artresizer.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 5dad38dfc..77226612e 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -456,6 +456,11 @@ class Shareable(type): return cls._instance +BACKEND_CLASSES = [ + IMBackend, + PILBackend, +] + class ArtResizer(metaclass=Shareable): """A singleton class that performs image resizes. """ @@ -463,11 +468,18 @@ class ArtResizer(metaclass=Shareable): def __init__(self): """Create a resizer object with an inferred method. """ - self.local_method = self._check_method() - if self.local_method is None: - log.debug(f"artresizer: method is WEBPROXY") + # Check if a local backend is availabe, and store an instance of the + # backend class. Otherwise, fallback to the web proxy. + for backend_cls in BACKEND_CLASSES: + try: + self.local_method = backend_cls() + log.debug(f"artresizer: method is {self.local_method.NAME}") + break + except LocalBackendNotAvailableError: + continue else: - log.debug(f"artresizer: method is {self.local_method.NAME}") + log.debug("artresizer: method is WEBPROXY") + self.local_method = None def resize( self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 @@ -587,17 +599,3 @@ class ArtResizer(metaclass=Shareable): # FIXME: Should probably issue a warning? return None - @staticmethod - def _check_method(): - """Search availabe methods. - - If a local backend is availabe, return an instance of the backend - class. Otherwise, when fallback to the web proxy is requird, return - None. - """ - try: - return IMBackend() - return PILBackend() - except LocalBackendNotAvailableError: - return None - From f751893be23e37df26eb7b25e697105f4867dff1 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:59:25 +0100 Subject: [PATCH 188/357] artresizer: add FIXME notes where we should probably add some warnings There's a lot of places where ArtResizer fails or falls back to completely silently. We should at least log this, or maybe even raise exceptions in some cases so that the caller can take a decision on how to handle the failure. --- beets/util/artresizer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 77226612e..ea8b6add8 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -207,7 +207,7 @@ class IMBackend(LocalBackend): util.command_output(cmd) return path_out except subprocess.CalledProcessError: - # FIXME: add a warning + # FIXME: Should probably issue a warning? return path_in def get_format(self, filepath): @@ -219,6 +219,7 @@ class IMBackend(LocalBackend): try: return util.command_output(cmd).stdout except subprocess.CalledProcessError: + # FIXME: Should probably issue a warning? return None def convert_format(self, source, target, deinterlaced): @@ -236,6 +237,7 @@ class IMBackend(LocalBackend): ) return target except subprocess.CalledProcessError: + # FIXME: Should probably issue a warning? return source @property @@ -404,6 +406,7 @@ class PILBackend(LocalBackend): im.save(py3_path(path_out), progressive=False) return path_out except IOError: + # FIXME: Should probably issue a warning? return path_in def get_format(self, filepath): @@ -574,6 +577,9 @@ class ArtResizer(metaclass=Shareable): result_path = self.local_method.convert_format( path_in, path_new, deinterlaced ) + except Exception: + # FIXME: Should probably issue a warning? + pass finally: if result_path != path_in: os.unlink(path_in) From 5d07b5390c55fc052e64bea4d92dfeb5fb13b20c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 21:02:38 +0100 Subject: [PATCH 189/357] artresizer: remove unused enum --- beets/util/artresizer.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index ea8b6add8..a08a922d5 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -27,11 +27,6 @@ from beets import logging from beets import util from beets.util import bytestring_path, displayable_path, py3_path, syspath -# Resizing methods -PIL = 1 -IMAGEMAGICK = 2 -WEBPROXY = 3 - PROXY_URL = 'https://images.weserv.nl/' log = logging.getLogger('beets') @@ -80,7 +75,6 @@ class LocalBackend: class IMBackend(LocalBackend): NAME="ImageMagick" - ID=IMAGEMAGICK _version = None _legacy = None @@ -312,7 +306,6 @@ class IMBackend(LocalBackend): class PILBackend(LocalBackend): NAME="PIL" - ID=PIL @classmethod def version(cls): From e44a08eeb60f23e342d4ae0ac1c51234d8b8ea45 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 21:09:23 +0100 Subject: [PATCH 190/357] artresizer: style fixes --- beets/util/artresizer.py | 40 +++++++++++++++++++++------------------- test/test_art_resize.py | 17 +++++++++-------- test/test_embedart.py | 2 +- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index a08a922d5..a5e5f99d1 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -74,7 +74,7 @@ class LocalBackend: class IMBackend(LocalBackend): - NAME="ImageMagick" + NAME = "ImageMagick" _version = None _legacy = None @@ -97,8 +97,8 @@ class IMBackend(LocalBackend): match = re.search(pattern, out) if match: cls._version = (int(match.group(1)), - int(match.group(2)), - int(match.group(3))) + int(match.group(2)), + int(match.group(3))) cls._legacy = legacy if cls._version is _NOT_AVAILABLE: @@ -127,7 +127,7 @@ class IMBackend(LocalBackend): self.compare_cmd = ['magick', 'compare'] def resize(self, maxwidth, path_in, path_out=None, quality=0, - max_filesize=0): + max_filesize=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -140,8 +140,8 @@ class IMBackend(LocalBackend): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. - # ImageMagick already seems to default to no interlace, but we include it - # here for the sake of explicitness. + # ImageMagick already seems to default to no interlace, but we include + # it here for the sake of explicitness. cmd = self.convert_cmd + [ syspath(path_in, prefix=False), '-resize', f'{maxwidth}x>', @@ -151,8 +151,8 @@ class IMBackend(LocalBackend): if quality > 0: cmd += ['-quality', f'{quality}'] - # "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to - # SIZE in bytes. + # "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick + # to SIZE in bytes. if max_filesize > 0: cmd += ['-define', f'jpeg:extent={max_filesize}b'] @@ -305,7 +305,7 @@ class IMBackend(LocalBackend): class PILBackend(LocalBackend): - NAME="PIL" + NAME = "PIL" @classmethod def version(cls): @@ -322,7 +322,7 @@ class PILBackend(LocalBackend): self.version() def resize(self, maxwidth, path_in, path_out=None, quality=0, - max_filesize=0): + max_filesize=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ @@ -346,8 +346,8 @@ class PILBackend(LocalBackend): im.save(py3_path(path_out), quality=quality, progressive=False) if max_filesize > 0: - # If maximum filesize is set, we attempt to lower the quality of - # jpeg conversion by a proportional amount, up to 3 attempts + # If maximum filesize is set, we attempt to lower the quality + # of jpeg conversion by a proportional amount, up to 3 attempts # First, set the maximum quality to either provided, or 95 if quality > 0: lower_qual = quality @@ -408,11 +408,11 @@ class PILBackend(LocalBackend): try: with Image.open(syspath(filepath)) as im: return im.format - except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError): + except (ValueError, TypeError, UnidentifiedImageError, + FileNotFoundError): log.exception("failed to detect image format for {}", filepath) return None - def convert_format(self, source, target, deinterlaced): from PIL import Image, UnidentifiedImageError @@ -420,8 +420,8 @@ class PILBackend(LocalBackend): with Image.open(syspath(source)) as im: im.save(py3_path(target), progressive=not deinterlaced) return target - except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError, - OSError): + except (ValueError, TypeError, UnidentifiedImageError, + FileNotFoundError, OSError): log.exception("failed to convert image {} -> {}", source, target) return source @@ -457,6 +457,7 @@ BACKEND_CLASSES = [ PILBackend, ] + class ArtResizer(metaclass=Shareable): """A singleton class that performs image resizes. """ @@ -486,8 +487,10 @@ class ArtResizer(metaclass=Shareable): For WEBPROXY, returns `path_in` unmodified. """ if self.local: - return self.local_method.resize(maxwidth, path_in, path_out, - quality=quality, max_filesize=max_filesize) + return self.local_method.resize( + maxwidth, path_in, path_out, + quality=quality, max_filesize=max_filesize + ) else: # Handled by `proxy_url` already. return path_in @@ -597,4 +600,3 @@ class ArtResizer(metaclass=Shareable): else: # FIXME: Should probably issue a warning? return None - diff --git a/test/test_art_resize.py b/test/test_art_resize.py index 57a7121f7..b6707ce41 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -21,17 +21,17 @@ import os from test import _common from test.helper import TestHelper from beets.util import command_output, syspath -from beets.util.artresizer import ( - IMBackend, - PILBackend, -) +from beets.util.artresizer import IMBackend, PILBackend class DummyIMBackend(IMBackend): - """An `IMBackend` which pretends that ImageMagick is available, and has - a sufficiently recent version to support image comparison. + """An `IMBackend` which pretends that ImageMagick is available. + + The version is sufficiently recent to support image comparison. """ + def __init__(self): + """Init a dummy backend class for mocked ImageMagick tests.""" self.version = (7, 0, 0) self.legacy = False self.convert_cmd = ['magick'] @@ -40,9 +40,10 @@ class DummyIMBackend(IMBackend): class DummyPILBackend(PILBackend): - """An `PILBackend` which pretends that PIL is available. - """ + """An `PILBackend` which pretends that PIL is available.""" + def __init__(self): + """Init a dummy backend class for mocked PIL tests.""" pass diff --git a/test/test_embedart.py b/test/test_embedart.py index b3f61babe..f41180ec1 100644 --- a/test/test_embedart.py +++ b/test/test_embedart.py @@ -25,7 +25,7 @@ from test.test_art_resize import DummyIMBackend from mediafile import MediaFile from beets import config, logging, ui -from beets.util import artresizer, syspath, displayable_path +from beets.util import syspath, displayable_path from beets.util.artresizer import ArtResizer from beets import art From b76a3fcaa45a1adbeff499a424895c4793809213 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sat, 12 Mar 2022 23:03:08 +0100 Subject: [PATCH 191/357] artresizer: move ImageMagick/PIL code from thumbnails plugin to ArtResizer Makes the dispatch to the chosen backend simpler in the thumbnails plugin. Given that ArtResizer is not only about resizing art anymore, these methods fit there quite nicely. --- beets/util/artresizer.py | 55 ++++++++++++++++++++++++++++++++++++++++ beetsplug/thumbnails.py | 40 +++++------------------------ test/test_art_resize.py | 14 ++++++++++ test/test_thumbnails.py | 37 +++++---------------------- 4 files changed, 82 insertions(+), 64 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index a5e5f99d1..5e619b5ca 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -16,6 +16,7 @@ public resizing proxy if neither is available. """ +from itertools import chain import subprocess import os import os.path @@ -303,6 +304,18 @@ class IMBackend(LocalBackend): log.debug('ImageMagick compare score: {0}', phash_diff) return phash_diff <= compare_threshold + @property + def can_write_metadata(self): + return True + + def write_metadata(self, file, metadata): + assignments = list(chain.from_iterable( + ('-set', k, v) for k, v in metadata.items() + )) + command = self.convert_cmd + [file, *assignments, file] + + util.command_output(command) + class PILBackend(LocalBackend): NAME = "PIL" @@ -433,6 +446,21 @@ class PILBackend(LocalBackend): # It is an error to call this when ArtResizer.can_compare is not True. raise NotImplementedError() + @property + def can_write_metadata(self): + return True + + def write_metadata(self, file, metadata): + from PIL import Image, PngImagePlugin + + # FIXME: Detect and handle other file types (currently, the only user + # is the thumbnails plugin, which generates PNG images). + im = Image.open(file) + meta = PngImagePlugin.PngInfo() + for k, v in metadata.items(): + meta.add_text(k, v, 0) + im.save(file, "PNG", pnginfo=meta) + class Shareable(type): """A pseudo-singleton metaclass that allows both shared and @@ -478,6 +506,13 @@ class ArtResizer(metaclass=Shareable): log.debug("artresizer: method is WEBPROXY") self.local_method = None + @property + def method(self): + if self.local: + return self.local_method.NAME + else: + return "WEBPROXY" + def resize( self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 ): @@ -600,3 +635,23 @@ class ArtResizer(metaclass=Shareable): else: # FIXME: Should probably issue a warning? return None + + @property + def can_write_metadata(self): + """A boolean indicating whether writing image metadata is supported.""" + + if self.local: + return self.local_method.can_write_metadata + else: + return False + + def write_metadata(self, file, metadata): + """Write key-value metadata to the image file. + + Only available locally. Currently, expects the image to be a PNG file. + """ + if self.local: + self.local_method.write_metadata(file, metadata) + else: + # FIXME: Should probably issue a warning? + pass diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index e3cf6e6a2..b81957593 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -22,7 +22,6 @@ Spec: standards.freedesktop.org/thumbnail-spec/latest/index.html from hashlib import md5 import os import shutil -from itertools import chain from pathlib import PurePosixPath import ctypes import ctypes.util @@ -32,7 +31,7 @@ from xdg import BaseDirectory from beets.plugins import BeetsPlugin from beets.ui import Subcommand, decargs from beets import util -from beets.util.artresizer import ArtResizer, IMBackend, PILBackend +from beets.util.artresizer import ArtResizer BASE_DIR = os.path.join(BaseDirectory.xdg_cache_home, "thumbnails") @@ -49,7 +48,6 @@ class ThumbnailsPlugin(BeetsPlugin): 'dolphin': False, }) - self.write_metadata = None if self.config['auto'] and self._check_local_ok(): self.register_listener('art_set', self.process_album) @@ -90,18 +88,12 @@ class ThumbnailsPlugin(BeetsPlugin): if not os.path.exists(dir): os.makedirs(dir) - # FIXME: Should we have our own backend instance? - self.backend = ArtResizer.shared.local_method - if isinstance(self.backend, IMBackend): - self.write_metadata = write_metadata_im - elif isinstance(self.backend, PILBackend): - self.write_metadata = write_metadata_pil - else: - # since we're local + if not ArtResizer.shared.can_write_metadata: raise RuntimeError( - f"Thumbnails: Unexpected ArtResizer backend {self.backend!r}." + f"Thumbnails: ArtResizer backend {ArtResizer.shared.method}" + f" unexpectedly cannot write image metadata." ) - self._log.debug(f"using {self.backend.NAME} to write metadata") + self._log.debug(f"using {ArtResizer.shared.method} to write metadata") uri_getter = GioURI() if not uri_getter.available: @@ -175,7 +167,7 @@ class ThumbnailsPlugin(BeetsPlugin): metadata = {"Thumb::URI": self.get_uri(album.artpath), "Thumb::MTime": str(mtime)} try: - self.write_metadata(self.backend, image_path, metadata) + ArtResizer.shared.write_metadata(image_path, metadata) except Exception: self._log.exception("could not write metadata to {0}", util.displayable_path(image_path)) @@ -192,26 +184,6 @@ class ThumbnailsPlugin(BeetsPlugin): self._log.debug("Wrote file {0}", util.displayable_path(outfilename)) -def write_metadata_im(im_backend, file, metadata): - """Enrich the file metadata with `metadata` dict thanks to IM.""" - command = im_backend.convert_cmd + [file] + \ - list(chain.from_iterable(('-set', k, v) - for k, v in metadata.items())) + [file] - util.command_output(command) - return True - - -def write_metadata_pil(pil_backend, file, metadata): - """Enrich the file metadata with `metadata` dict thanks to PIL.""" - from PIL import Image, PngImagePlugin - im = Image.open(file) - meta = PngImagePlugin.PngInfo() - for k, v in metadata.items(): - meta.add_text(k, v, 0) - im.save(file, "PNG", pnginfo=meta) - return True - - class URIGetter: available = False name = "Abstract base" diff --git a/test/test_art_resize.py b/test/test_art_resize.py index b6707ce41..9660d96a2 100644 --- a/test/test_art_resize.py +++ b/test/test_art_resize.py @@ -16,6 +16,7 @@ import unittest +from unittest.mock import patch import os from test import _common @@ -142,6 +143,19 @@ class ArtResizerFileSizeTest(_common.TestCase, TestHelper): out = command_output(cmd).stdout self.assertTrue(out == b'None') + @patch('beets.util.artresizer.util') + def test_write_metadata_im(self, mock_util): + """Test writing image metadata.""" + metadata = {"a": "A", "b": "B"} + im = DummyIMBackend() + im.write_metadata("foo", metadata) + try: + command = im.convert_cmd + "foo -set a A -set b B foo".split() + mock_util.command_output.assert_called_once_with(command) + except AssertionError: + command = im.convert_cmd + "foo -set b B -set a A foo".split() + mock_util.command_output.assert_called_once_with(command) + def suite(): """Run this suite of tests.""" diff --git a/test/test_thumbnails.py b/test/test_thumbnails.py index 376969e93..891411535 100644 --- a/test/test_thumbnails.py +++ b/test/test_thumbnails.py @@ -20,11 +20,9 @@ from shutil import rmtree import unittest from test.helper import TestHelper -from test.test_art_resize import DummyIMBackend, DummyPILBackend from beets.util import bytestring_path from beetsplug.thumbnails import (ThumbnailsPlugin, NORMAL_DIR, LARGE_DIR, - write_metadata_im, write_metadata_pil, PathlibURI, GioURI) @@ -35,25 +33,11 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): def tearDown(self): self.teardown_beets() - @patch('beetsplug.thumbnails.util') - def test_write_metadata_im(self, mock_util): - metadata = {"a": "A", "b": "B"} - im = DummyIMBackend() - write_metadata_im(im, "foo", metadata) - try: - command = im.convert_cmd + "foo -set a A -set b B foo".split() - mock_util.command_output.assert_called_once_with(command) - except AssertionError: - command = im.convert_cmd + "foo -set b B -set a A foo".split() - mock_util.command_output.assert_called_once_with(command) - + @patch('beetsplug.thumbnails.ArtResizer') @patch('beetsplug.thumbnails.ThumbnailsPlugin._check_local_ok') @patch('beetsplug.thumbnails.os.stat') - def test_add_tags(self, mock_stat, _): + def test_add_tags(self, mock_stat, _, mock_artresizer): plugin = ThumbnailsPlugin() - # backend is not set due to _check_local_ok being mocked - plugin.backend = "DummyBackend" - plugin.write_metadata = Mock() plugin.get_uri = Mock(side_effect={b"/path/to/cover": "COVER_URI"}.__getitem__) album = Mock(artpath=b"/path/to/cover") @@ -63,8 +47,7 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): metadata = {"Thumb::URI": "COVER_URI", "Thumb::MTime": "12345"} - plugin.write_metadata.assert_called_once_with( - plugin.backend, + mock_artresizer.shared.write_metadata.assert_called_once_with( b"/path/to/thumbnail", metadata, ) @@ -76,13 +59,13 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): def test_check_local_ok(self, mock_giouri, mock_artresizer, mock_os): # test local resizing capability mock_artresizer.shared.local = False - mock_artresizer.shared.local_method = None + mock_artresizer.shared.can_write_metadata = False plugin = ThumbnailsPlugin() self.assertFalse(plugin._check_local_ok()) # test dirs creation mock_artresizer.shared.local = True - mock_artresizer.shared.local_method = DummyIMBackend() + mock_artresizer.shared.can_write_metadata = True def exists(path): if path == NORMAL_DIR: @@ -99,18 +82,12 @@ class ThumbnailsTest(unittest.TestCase, TestHelper): mock_os.path.exists = lambda _: True mock_artresizer.shared.local = True - mock_artresizer.shared.local_method = None + mock_artresizer.shared.can_write_metadata = False with self.assertRaises(RuntimeError): ThumbnailsPlugin() mock_artresizer.shared.local = True - mock_artresizer.shared.local_method = DummyPILBackend() - self.assertEqual(ThumbnailsPlugin().write_metadata, write_metadata_pil) - - mock_artresizer.shared.local = True - mock_artresizer.shared.local_method = DummyIMBackend() - self.assertEqual(ThumbnailsPlugin().write_metadata, write_metadata_im) - + mock_artresizer.shared.can_write_metadata = True self.assertTrue(ThumbnailsPlugin()._check_local_ok()) # test URI getter function From 51894d3fec9819454eb6fcf5566062bff6c6dd5c Mon Sep 17 00:00:00 2001 From: Kai Date: Sun, 13 Mar 2022 03:44:14 +0000 Subject: [PATCH 192/357] README.md: HTTP => HTTPS --- README.rst | 2 +- README_kr.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6cdcd3054..698f6ec7d 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ shockingly simple if you know a little Python. .. _writing your own plugin: https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: - http://www.w3.org/TR/html-markup/audio.html + https://www.w3.org/TR/html-markup/audio.html .. _albums that are missing tracks: https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: diff --git a/README_kr.rst b/README_kr.rst index 25dd052d8..2681bcd7a 100644 --- a/README_kr.rst +++ b/README_kr.rst @@ -54,7 +54,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 .. _writing your own plugin: https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: - http://www.w3.org/TR/html-markup/audio.html + https://www.w3.org/TR/html-markup/audio.html .. _albums that are missing tracks: https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: From 0c7d1422eece1b0f918ed8a99bc97a31df9028d8 Mon Sep 17 00:00:00 2001 From: Kai Date: Sun, 13 Mar 2022 16:14:14 +0000 Subject: [PATCH 193/357] new link for HTML5 audio --- README.rst | 2 +- README_kr.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 698f6ec7d..b894daddc 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ shockingly simple if you know a little Python. .. _writing your own plugin: https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: - https://www.w3.org/TR/html-markup/audio.html + https://html.spec.whatwg.org/multipage/media.html#the-audio-element .. _albums that are missing tracks: https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: diff --git a/README_kr.rst b/README_kr.rst index 2681bcd7a..a6a95ec5a 100644 --- a/README_kr.rst +++ b/README_kr.rst @@ -54,7 +54,7 @@ Beets는 라이브러리로 디자인 되었기 때문에, 당신이 음악들 .. _writing your own plugin: https://beets.readthedocs.org/page/dev/plugins.html .. _HTML5 Audio: - https://www.w3.org/TR/html-markup/audio.html + https://html.spec.whatwg.org/multipage/media.html#the-audio-element .. _albums that are missing tracks: https://beets.readthedocs.org/page/plugins/missing.html .. _duplicate tracks and albums: From ffe63a64f5dc6724054e22dd88d4181068c5f4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Sun, 13 Mar 2022 18:50:22 +0000 Subject: [PATCH 194/357] Nicely disable musicbrainz --- beets/autotag/mb.py | 9 +++++++++ beets/config_default.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index e6a2e277f..9a6a7e7f9 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -482,6 +482,9 @@ def match_album(artist, album, tracks=None, extra_tags=None): The query consists of an artist name, an album name, and, optionally, a number of tracks on the album and any other extra tags. """ + if not config["musicbrainz"]["enabled"].get(bool): + return None + # Build search criteria. criteria = {'release': album.lower().strip()} if artist is not None: @@ -558,6 +561,9 @@ def album_for_id(releaseid): object or None if the album is not found. May raise a MusicBrainzAPIError. """ + if not config["musicbrainz"]["enabled"].get(bool): + return None + log.debug('Requesting MusicBrainz release {}', releaseid) albumid = _parse_id(releaseid) if not albumid: @@ -579,6 +585,9 @@ def track_for_id(releaseid): """Fetches a track by its MusicBrainz ID. Returns a TrackInfo object or None if no track is found. May raise a MusicBrainzAPIError. """ + if not config["musicbrainz"]["enabled"].get(bool): + return None + trackid = _parse_id(releaseid) if not trackid: log.debug('Invalid MBID ({0}).', releaseid) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 74540891e..518e086a1 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -108,6 +108,7 @@ musicbrainz: searchlimit: 5 extra_tags: [] genres: no + enabled: yes match: strong_rec_thresh: 0.04 From 85c4310e47b0e5e4f34aa7c538f8fe68083bd5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 15 Mar 2022 21:56:33 +0000 Subject: [PATCH 195/357] Update docs --- beets/config_default.yaml | 2 +- docs/reference/config.rst | 59 ++++++++++++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 518e086a1..fd2dbf551 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -101,6 +101,7 @@ paths: statefile: state.pickle musicbrainz: + enabled: yes host: musicbrainz.org https: no ratelimit: 1 @@ -108,7 +109,6 @@ musicbrainz: searchlimit: 5 extra_tags: [] genres: no - enabled: yes match: strong_rec_thresh: 0.04 diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 642216c8f..ae04ce525 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -716,6 +716,39 @@ Default: ``{}`` (empty). MusicBrainz Options ------------------- +Default configuration:: + + musicbrainz: + enabled: yes + host: musicbrainz.org + https: no + ratelimit: 1 + ratelimit_interval: 1.0 + searchlimit: 5 + extra_tags: [] + genres: no + +.. _enabled: + +enabled +~~~~~~~ + +This option allows you to disable using MusicBrainz as a metadata source. This applies +if you use plugins that fetch data from alternative sources and should make the import +process quicker. + +Default: ``yes``. + +.. _host: + +host +~~~~ + +The ``host`` key, of course, controls the Web server hostname (and port, +optionally) that will be contacted by beets. + +Default: ``musicbrainz.org`` + You can instruct beets to use `your own MusicBrainz database`_ instead of the `main server`_. Use the ``host``, ``https`` and ``ratelimit`` options under a ``musicbrainz:`` header, like so:: @@ -725,16 +758,27 @@ under a ``musicbrainz:`` header, like so:: https: no ratelimit: 100 -The ``host`` key, of course, controls the Web server hostname (and port, -optionally) that will be contacted by beets (default: musicbrainz.org). +.. _https: + +https +~~~~~ + The ``https`` key makes the client use HTTPS instead of HTTP. This setting applies -only to custom servers. The official MusicBrainz server always uses HTTPS. (Default: no.) +only to custom servers. The official MusicBrainz server always uses HTTPS. The server must have search indices enabled (see `Building search indexes`_). -The ``ratelimit`` option, an integer, controls the number of Web service requests -per second (default: 1). **Do not change the rate limit setting** if you're -using the main MusicBrainz server---on this public server, you're `limited`_ -to one request per second. +Default: ``no`` + +.. _ratelimit and ratelimit_interval: + +ratelimit and ratelimit_interval +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``ratelimit`` many requests will be sent per ``ratelimit_interval`` by the Web service. +**Do not change these settings** if you're using the main MusicBrainz +server - on this public server, you're `limited`_ to one request per second. + +Default: ``1`` and ``1.0`` .. _your own MusicBrainz database: https://musicbrainz.org/doc/MusicBrainz_Server/Setup .. _main server: https://musicbrainz.org/ @@ -780,6 +824,7 @@ Use MusicBrainz genre tags to populate (and replace if it's already set) the ``genre`` tag. This will make it a list of all the genres tagged for the release and the release-group on MusicBrainz, separated by "; " and sorted by the total number of votes. + Default: ``no`` .. _match-config: From f10b70444c47cd3e70395aaeda941e0633b10be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 15 Mar 2022 22:13:23 +0000 Subject: [PATCH 196/357] Add a changelog entry --- docs/changelog.rst | 2 ++ docs/reference/config.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bd3ee2841..002b96c5d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,8 @@ Changelog goes here! New features: +* :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling + the MusicBrainz metadata source during the autotagging process * :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances :bug:`4101` * Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. diff --git a/docs/reference/config.rst b/docs/reference/config.rst index ae04ce525..94ec61a72 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -728,7 +728,7 @@ Default configuration:: extra_tags: [] genres: no -.. _enabled: +.. _musicbrainz.enabled: enabled ~~~~~~~ From c0e44960c6161fb30fd58142f19e03c3cab299c0 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Wed, 16 Mar 2022 13:39:18 +0100 Subject: [PATCH 197/357] expand documentation on auto and auto_keep --- docs/plugins/convert.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index f8ee245ce..07257806c 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -70,6 +70,11 @@ file. The available options are: import the non transcoded version. It uses the default format you have defined in your config file. Default: ``no``. + +.. note:: **auto** and **auto_keep** are mutually exclusive. Use either one or + the other but not both at the same time if you don't want your files to be + converted twice on import. + - **tmpdir**: The directory where temporary files will be stored during import. Default: none (system default), - **copy_album_art**: Copy album art when copying or transcoding albums matched From 3f6da68af2444b77807692023abfd607f0146750 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 16 Mar 2022 11:08:05 -0400 Subject: [PATCH 198/357] Slight docs rewording --- docs/plugins/convert.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 07257806c..ff439eb97 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -71,9 +71,9 @@ file. The available options are: defined in your config file. Default: ``no``. -.. note:: **auto** and **auto_keep** are mutually exclusive. Use either one or - the other but not both at the same time if you don't want your files to be - converted twice on import. + .. note:: You probably want to use only one of the `auto` and `auto_keep` + options, not both. Enabling both will convert your files twice on import, + which you probably don't want. - **tmpdir**: The directory where temporary files will be stored during import. Default: none (system default), From 6f23c9b41a31712c121f8b3c94247a0a7895d562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Fri, 18 Mar 2022 05:05:58 +0000 Subject: [PATCH 199/357] Move the logic to hooks.py --- beets/autotag/hooks.py | 31 +++++++++++++++---------------- beets/autotag/mb.py | 9 --------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 9cd6f2cd8..4055596be 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -598,6 +598,16 @@ def tracks_for_id(track_id): yield t +def handle_exc(call_func, *args): + if not config["musicbrainz"]["enabled"]: + return () + + try: + return call_func(*args) + except mb.MusicBrainzAPIError as exc: + exc.log(log) + + @plugins.notify_info_yielded('albuminfo_received') def album_candidates(items, artist, album, va_likely, extra_tags): """Search for album matches. ``items`` is a list of Item objects @@ -609,25 +619,17 @@ def album_candidates(items, artist, album, va_likely, extra_tags): constrain the search. """ + common_args = [album, len(items), extra_tags] # Base candidates if we have album and artist to match. if artist and album: - try: - yield from mb.match_album(artist, album, len(items), - extra_tags) - except mb.MusicBrainzAPIError as exc: - exc.log(log) + yield from handle_exc(mb.match_album, artist, *common_args) # Also add VA matches from MusicBrainz where appropriate. if va_likely and album: - try: - yield from mb.match_album(None, album, len(items), - extra_tags) - except mb.MusicBrainzAPIError as exc: - exc.log(log) + yield from handle_exc(mb.match_album, None, *common_args) # Candidates from plugins. - yield from plugins.candidates(items, artist, album, va_likely, - extra_tags) + yield from plugins.candidates(items, artist, album, va_likely, extra_tags) @plugins.notify_info_yielded('trackinfo_received') @@ -639,10 +641,7 @@ def item_candidates(item, artist, title): # MusicBrainz candidates. if artist and title: - try: - yield from mb.match_track(artist, title) - except mb.MusicBrainzAPIError as exc: - exc.log(log) + yield from handle_exc(mb.match_track, artist, title) # Plugin candidates. yield from plugins.item_candidates(item, artist, title) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 9a6a7e7f9..e6a2e277f 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -482,9 +482,6 @@ def match_album(artist, album, tracks=None, extra_tags=None): The query consists of an artist name, an album name, and, optionally, a number of tracks on the album and any other extra tags. """ - if not config["musicbrainz"]["enabled"].get(bool): - return None - # Build search criteria. criteria = {'release': album.lower().strip()} if artist is not None: @@ -561,9 +558,6 @@ def album_for_id(releaseid): object or None if the album is not found. May raise a MusicBrainzAPIError. """ - if not config["musicbrainz"]["enabled"].get(bool): - return None - log.debug('Requesting MusicBrainz release {}', releaseid) albumid = _parse_id(releaseid) if not albumid: @@ -585,9 +579,6 @@ def track_for_id(releaseid): """Fetches a track by its MusicBrainz ID. Returns a TrackInfo object or None if no track is found. May raise a MusicBrainzAPIError. """ - if not config["musicbrainz"]["enabled"].get(bool): - return None - trackid = _parse_id(releaseid) if not trackid: log.debug('Invalid MBID ({0}).', releaseid) From 2193b3749bb65648d36aa662e1f9c4b6bda8b07f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Fri, 18 Mar 2022 05:13:06 +0000 Subject: [PATCH 200/357] Remove optional docs changes --- docs/reference/config.rst | 48 ++++++--------------------------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 94ec61a72..aad9d4bdf 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -716,18 +716,6 @@ Default: ``{}`` (empty). MusicBrainz Options ------------------- -Default configuration:: - - musicbrainz: - enabled: yes - host: musicbrainz.org - https: no - ratelimit: 1 - ratelimit_interval: 1.0 - searchlimit: 5 - extra_tags: [] - genres: no - .. _musicbrainz.enabled: enabled @@ -739,16 +727,6 @@ process quicker. Default: ``yes``. -.. _host: - -host -~~~~ - -The ``host`` key, of course, controls the Web server hostname (and port, -optionally) that will be contacted by beets. - -Default: ``musicbrainz.org`` - You can instruct beets to use `your own MusicBrainz database`_ instead of the `main server`_. Use the ``host``, ``https`` and ``ratelimit`` options under a ``musicbrainz:`` header, like so:: @@ -758,27 +736,16 @@ under a ``musicbrainz:`` header, like so:: https: no ratelimit: 100 -.. _https: - -https -~~~~~ - +The ``host`` key, of course, controls the Web server hostname (and port, +optionally) that will be contacted by beets (default: musicbrainz.org). The ``https`` key makes the client use HTTPS instead of HTTP. This setting applies -only to custom servers. The official MusicBrainz server always uses HTTPS. +only to custom servers. The official MusicBrainz server always uses HTTPS. (Default: no.) The server must have search indices enabled (see `Building search indexes`_). -Default: ``no`` - -.. _ratelimit and ratelimit_interval: - -ratelimit and ratelimit_interval -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``ratelimit`` many requests will be sent per ``ratelimit_interval`` by the Web service. -**Do not change these settings** if you're using the main MusicBrainz -server - on this public server, you're `limited`_ to one request per second. - -Default: ``1`` and ``1.0`` +The ``ratelimit`` option, an integer, controls the number of Web service requests +per second (default: 1). **Do not change the rate limit setting** if you're +using the main MusicBrainz server---on this public server, you're `limited`_ +to one request per second. .. _your own MusicBrainz database: https://musicbrainz.org/doc/MusicBrainz_Server/Setup .. _main server: https://musicbrainz.org/ @@ -824,7 +791,6 @@ Use MusicBrainz genre tags to populate (and replace if it's already set) the ``genre`` tag. This will make it a list of all the genres tagged for the release and the release-group on MusicBrainz, separated by "; " and sorted by the total number of votes. - Default: ``no`` .. _match-config: From f14eac857b91311b27bd3d934a3c87dbdd9b8fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Fri, 18 Mar 2022 22:57:22 +0000 Subject: [PATCH 201/357] Rename handle_exc to invoke_mb --- beets/autotag/hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 4055596be..a385699ac 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -598,7 +598,7 @@ def tracks_for_id(track_id): yield t -def handle_exc(call_func, *args): +def invoke_mb(call_func, *args): if not config["musicbrainz"]["enabled"]: return () @@ -622,11 +622,11 @@ def album_candidates(items, artist, album, va_likely, extra_tags): common_args = [album, len(items), extra_tags] # Base candidates if we have album and artist to match. if artist and album: - yield from handle_exc(mb.match_album, artist, *common_args) + yield from invoke_mb(mb.match_album, artist, *common_args) # Also add VA matches from MusicBrainz where appropriate. if va_likely and album: - yield from handle_exc(mb.match_album, None, *common_args) + yield from invoke_mb(mb.match_album, None, *common_args) # Candidates from plugins. yield from plugins.candidates(items, artist, album, va_likely, extra_tags) @@ -641,7 +641,7 @@ def item_candidates(item, artist, title): # MusicBrainz candidates. if artist and title: - yield from handle_exc(mb.match_track, artist, title) + yield from invoke_mb(mb.match_track, artist, title) # Plugin candidates. yield from plugins.item_candidates(item, artist, title) From 40b3dd7152d4303394c9b65d9e8718a0ff55d703 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 22 Mar 2022 21:22:08 +0100 Subject: [PATCH 202/357] artresizer: add a comment to clarify the meaning of class variables --- beets/util/artresizer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 5e619b5ca..904559830 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -76,6 +76,10 @@ class LocalBackend: class IMBackend(LocalBackend): NAME = "ImageMagick" + + # These fields are used as a cache for `version()`. `_legacy` indicates + # whether the modern `magick` binary is available or whether to fall back + # to the old-style `convert`, `identify`, etc. commands. _version = None _legacy = None From 09101de397f4fcfad5e47aaef68539524b8a269f Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 22 Mar 2022 21:26:53 +0100 Subject: [PATCH 203/357] artresizer: restore incorrect change to exception handling from f751893b --- beets/util/artresizer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 904559830..8a08bf6de 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -612,9 +612,6 @@ class ArtResizer(metaclass=Shareable): result_path = self.local_method.convert_format( path_in, path_new, deinterlaced ) - except Exception: - # FIXME: Should probably issue a warning? - pass finally: if result_path != path_in: os.unlink(path_in) From af71cebeeb4c67c5471d60648ad33be81e970be5 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 22 Mar 2022 22:08:04 +0100 Subject: [PATCH 204/357] artresizer: fixup 8a969e30 this crept in from a previous version of this PR which didn't make it --- beets/util/artresizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 8a08bf6de..225280a94 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -592,7 +592,7 @@ class ArtResizer(metaclass=Shareable): Only available locally. """ - if self.local: + if not self.local: # FIXME: Should probably issue a warning? return path_in From e1ffadb2d6bbff8654c7bf593bda0dc40ac5f070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 23 Mar 2022 16:20:06 +0000 Subject: [PATCH 205/357] Return iterable from the exception handler --- beets/autotag/hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index a385699ac..2f9e2135c 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -606,6 +606,7 @@ def invoke_mb(call_func, *args): return call_func(*args) except mb.MusicBrainzAPIError as exc: exc.log(log) + return () @plugins.notify_info_yielded('albuminfo_received') From e9972491f1e4d49cbabcaec258add299e6ea0efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 23 Mar 2022 16:34:13 +0000 Subject: [PATCH 206/357] Do not assign extra variable to keep things within max line_length --- beets/autotag/hooks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 2f9e2135c..1811cbaf1 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -620,14 +620,15 @@ def album_candidates(items, artist, album, va_likely, extra_tags): constrain the search. """ - common_args = [album, len(items), extra_tags] # Base candidates if we have album and artist to match. if artist and album: - yield from invoke_mb(mb.match_album, artist, *common_args) + yield from invoke_mb(mb.match_album, artist, album, len(items), + extra_tags) # Also add VA matches from MusicBrainz where appropriate. if va_likely and album: - yield from invoke_mb(mb.match_album, None, *common_args) + yield from invoke_mb(mb.match_album, None, album, len(items), + extra_tags) # Candidates from plugins. yield from plugins.candidates(items, artist, album, va_likely, extra_tags) From a6e7105449ffc89a5cb463451527feafe93a7012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 23 Mar 2022 16:59:21 +0000 Subject: [PATCH 207/357] Move musicbrainz.enabled check out from invoke_mb --- beets/autotag/hooks.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 1811cbaf1..30904ff29 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -599,9 +599,6 @@ def tracks_for_id(track_id): def invoke_mb(call_func, *args): - if not config["musicbrainz"]["enabled"]: - return () - try: return call_func(*args) except mb.MusicBrainzAPIError as exc: @@ -620,15 +617,16 @@ def album_candidates(items, artist, album, va_likely, extra_tags): constrain the search. """ - # Base candidates if we have album and artist to match. - if artist and album: - yield from invoke_mb(mb.match_album, artist, album, len(items), - extra_tags) + if config["musicbrainz"]["enabled"]: + # Base candidates if we have album and artist to match. + if artist and album: + yield from invoke_mb(mb.match_album, artist, album, len(items), + extra_tags) - # Also add VA matches from MusicBrainz where appropriate. - if va_likely and album: - yield from invoke_mb(mb.match_album, None, album, len(items), - extra_tags) + # Also add VA matches from MusicBrainz where appropriate. + if va_likely and album: + yield from invoke_mb(mb.match_album, None, album, len(items), + extra_tags) # Candidates from plugins. yield from plugins.candidates(items, artist, album, va_likely, extra_tags) @@ -642,7 +640,7 @@ def item_candidates(item, artist, title): """ # MusicBrainz candidates. - if artist and title: + if config["musicbrainz"]["enabled"] and artist and title: yield from invoke_mb(mb.match_track, artist, title) # Plugin candidates. From 5d6f808d33492ef450dd835da6de109a8bf3b731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 23 Mar 2022 17:02:45 +0000 Subject: [PATCH 208/357] Move musicbrainz.enabled section below network config in config.rst --- docs/reference/config.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index aad9d4bdf..6e7df1b59 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -716,17 +716,6 @@ Default: ``{}`` (empty). MusicBrainz Options ------------------- -.. _musicbrainz.enabled: - -enabled -~~~~~~~ - -This option allows you to disable using MusicBrainz as a metadata source. This applies -if you use plugins that fetch data from alternative sources and should make the import -process quicker. - -Default: ``yes``. - You can instruct beets to use `your own MusicBrainz database`_ instead of the `main server`_. Use the ``host``, ``https`` and ``ratelimit`` options under a ``musicbrainz:`` header, like so:: @@ -752,6 +741,17 @@ to one request per second. .. _limited: https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting .. _Building search indexes: https://musicbrainz.org/doc/Development/Search_server_setup +.. _musicbrainz.enabled: + +enabled +~~~~~~~ + +This option allows you to disable using MusicBrainz as a metadata source. This applies +if you use plugins that fetch data from alternative sources and should make the import +process quicker. + +Default: ``yes``. + .. _searchlimit: searchlimit From 9917f2ac3aa24e2e690ffb237c5a50434c164793 Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 09:56:47 -0400 Subject: [PATCH 209/357] create db without prompt --- beets/dbcore/db.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index acd131be2..0e484d8ce 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -900,9 +900,15 @@ class Database: """The current revision of the database. To be increased whenever data is written in a transaction. """ + def _path_checker(self, path): + newpath = os.path.dirname(path) + if not os.path.isdir(newpath): + os.makedirs(newpath) + def __init__(self, path, timeout=5.0): self.path = path + self._path_checker(path) self.timeout = timeout self._connections = {} From b07b0e2f7e377d466ef4e36901396a4bb7dcfdbf Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 10:33:54 -0400 Subject: [PATCH 210/357] add prompt --- beets/dbcore/db.py | 4 +++- beets/ui/commands.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 0e484d8ce..bafd32ded 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -903,7 +903,9 @@ class Database: def _path_checker(self, path): newpath = os.path.dirname(path) if not os.path.isdir(newpath): - os.makedirs(newpath) + from beets.ui.commands import database_dir_creation + if database_dir_creation(newpath): + os.makedirs(newpath) def __init__(self, path, timeout=5.0): diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 3a3374013..bf2b184d2 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1398,6 +1398,12 @@ version_cmd = ui.Subcommand( version_cmd.func = show_version default_commands.append(version_cmd) +# database_location: return true if user wants to create the parent directories. +def database_dir_creation(path): + # Ask the user for a choice. + return ui.input_yn("{} does not exist, create it (Y/n)?" + .format(displayable_path(path))) + # modify: Declaratively change metadata. From c67245ed65a0c7f707ef9e755e32b782b276f69a Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 10:42:16 -0400 Subject: [PATCH 211/357] style checker, change log --- beets/dbcore/db.py | 3 ++- beets/ui/commands.py | 7 +++++-- docs/changelog.rst | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index bafd32ded..979c973a9 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -900,6 +900,8 @@ class Database: """The current revision of the database. To be increased whenever data is written in a transaction. """ + + # Check whether parental directories exist. def _path_checker(self, path): newpath = os.path.dirname(path) if not os.path.isdir(newpath): @@ -907,7 +909,6 @@ class Database: if database_dir_creation(newpath): os.makedirs(newpath) - def __init__(self, path, timeout=5.0): self.path = path self._path_checker(path) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index bf2b184d2..da78d682a 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1398,11 +1398,14 @@ version_cmd = ui.Subcommand( version_cmd.func = show_version default_commands.append(version_cmd) -# database_location: return true if user wants to create the parent directories. +# database_location: return true if user +# wants to create the parent directories. + + def database_dir_creation(path): # Ask the user for a choice. return ui.input_yn("{} does not exist, create it (Y/n)?" - .format(displayable_path(path))) + .format(displayable_path(path))) # modify: Declaratively change metadata. diff --git a/docs/changelog.rst b/docs/changelog.rst index 002b96c5d..fe4afa810 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changelog Changelog goes here! New features: +* Create the parental directories for database if they do not exist. * :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling the MusicBrainz metadata source during the autotagging process From d3d9318c18ca5b40ad8e41f20b5ebe0ea49856d7 Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 10:58:09 -0400 Subject: [PATCH 212/357] change doc --- docs/guides/main.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 8dbb113c4..feb918570 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -142,7 +142,8 @@ place to start:: Change that first path to a directory where you'd like to keep your music. Then, for ``library``, choose a good place to keep a database file that keeps an index -of your music. (The config's format is `YAML`_. You'll want to configure your +of your music. Beets will prompt you if the parental directories for database do +not exist. (The config's format is `YAML`_. You'll want to configure your text editor to use spaces, not real tabs, for indentation. Also, ``~`` means your home directory in these paths, even on Windows.) From 879ed7f5c2de73b44aa849b45b6915fecfca1380 Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 15:06:00 -0400 Subject: [PATCH 213/357] fix test cases, support in memory db --- beets/dbcore/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 979c973a9..84fa1173d 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -903,6 +903,8 @@ class Database: # Check whether parental directories exist. def _path_checker(self, path): + if path == ":memory:": # For testing + return newpath = os.path.dirname(path) if not os.path.isdir(newpath): from beets.ui.commands import database_dir_creation From c0d05f854508bd452e0c0b2927bca4d639b4577b Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 17:09:30 -0400 Subject: [PATCH 214/357] add test cases --- beets/dbcore/db.py | 2 +- test/test_dbcore.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 84fa1173d..243308c0d 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -903,7 +903,7 @@ class Database: # Check whether parental directories exist. def _path_checker(self, path): - if path == ":memory:": # For testing + if path == ":memory:": # For testing return newpath = os.path.dirname(path) if not os.path.isdir(newpath): diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 603d85bad..5dbff5223 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -19,6 +19,8 @@ import os import shutil import sqlite3 import unittest +from random import random +from unittest import mock from test import _common from beets import dbcore @@ -760,8 +762,31 @@ class ResultsIteratorTest(unittest.TestCase): ModelFixture1, dbcore.query.FalseQuery()).get()) +class ParentalDirCreation(unittest.TestCase): + @mock.patch('builtins.input', side_effect=['y', ]) + def test_create_yes(self, _): + non_exist_path = "ParentalDirCreationTest/nonexist/" + str(random()) + try: + dbcore.Database(non_exist_path) + except OSError as e: + raise e + shutil.rmtree("ParentalDirCreationTest") + + @mock.patch('builtins.input', side_effect=['n', ]) + def test_create_no(self, _): + non_exist_path = "ParentalDirCreationTest/nonexist/" + str(random()) + try: + dbcore.Database(non_exist_path) + except OSError as e: + raise e + if os.path.exists("ParentalDirCreationTest/nonexist/"): + shutil.rmtree("ParentalDirCreationTest") + raise OSError("Should not create dir") + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) + if __name__ == '__main__': unittest.main(defaultTest='suite') From 9029a8a6b39cc5ad6746cd8840ddaefe9bffd020 Mon Sep 17 00:00:00 2001 From: alicezou Date: Sun, 27 Mar 2022 23:05:23 -0400 Subject: [PATCH 215/357] check bytes --- beets/dbcore/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 243308c0d..397dbcedf 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -903,7 +903,7 @@ class Database: # Check whether parental directories exist. def _path_checker(self, path): - if path == ":memory:": # For testing + if not isinstance(path, bytes) and path == ':memory:': # in memory db return newpath = os.path.dirname(path) if not os.path.isdir(newpath): From a0b0028874989b0bc1cbbed00be4b05a55b1db52 Mon Sep 17 00:00:00 2001 From: alicezou Date: Tue, 29 Mar 2022 19:58:15 -0400 Subject: [PATCH 216/357] working db --- beets/ui/commands.py | 2 +- test/test_dbcore.py | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index da78d682a..394a2831a 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1404,7 +1404,7 @@ default_commands.append(version_cmd) def database_dir_creation(path): # Ask the user for a choice. - return ui.input_yn("{} does not exist, create it (Y/n)?" + return ui.input_yn("The database directory {} does not exists, create it (Y/n)?" .format(displayable_path(path))) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 5dbff5223..c92f31c1f 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -762,25 +762,22 @@ class ResultsIteratorTest(unittest.TestCase): ModelFixture1, dbcore.query.FalseQuery()).get()) -class ParentalDirCreation(unittest.TestCase): +class ParentalDirCreation(_common.TestCase): @mock.patch('builtins.input', side_effect=['y', ]) def test_create_yes(self, _): - non_exist_path = "ParentalDirCreationTest/nonexist/" + str(random()) - try: - dbcore.Database(non_exist_path) - except OSError as e: - raise e - shutil.rmtree("ParentalDirCreationTest") + non_exist_path = _common.util.py3_path(os.path.join( + self.temp_dir, b'nonexist', str(random()).encode())) + dbcore.Database(non_exist_path) @mock.patch('builtins.input', side_effect=['n', ]) def test_create_no(self, _): - non_exist_path = "ParentalDirCreationTest/nonexist/" + str(random()) - try: - dbcore.Database(non_exist_path) - except OSError as e: - raise e - if os.path.exists("ParentalDirCreationTest/nonexist/"): - shutil.rmtree("ParentalDirCreationTest") + non_exist_path_parent = _common.util.py3_path( + os.path.join(self.temp_dir, b'nonexist')) + non_exist_path = _common.util.py3_path(os.path.join( + non_exist_path_parent.encode(), str(random()).encode())) + dbcore.Database(non_exist_path) + if os.path.exists(non_exist_path_parent): + shutil.rmtree(non_exist_path_parent) raise OSError("Should not create dir") From 67e778fec6a4bd130ef0fc61714dcdaf95f874c4 Mon Sep 17 00:00:00 2001 From: alicezou Date: Tue, 29 Mar 2022 20:04:56 -0400 Subject: [PATCH 217/357] switch to control_stdin --- test/test_dbcore.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index c92f31c1f..95c52196f 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -23,6 +23,7 @@ from random import random from unittest import mock from test import _common +from test.helper import control_stdin from beets import dbcore from tempfile import mkstemp @@ -763,19 +764,19 @@ class ResultsIteratorTest(unittest.TestCase): class ParentalDirCreation(_common.TestCase): - @mock.patch('builtins.input', side_effect=['y', ]) - def test_create_yes(self, _): + def test_create_yes(self): non_exist_path = _common.util.py3_path(os.path.join( self.temp_dir, b'nonexist', str(random()).encode())) - dbcore.Database(non_exist_path) + with control_stdin('y'): + dbcore.Database(non_exist_path) - @mock.patch('builtins.input', side_effect=['n', ]) - def test_create_no(self, _): + def test_create_no(self): non_exist_path_parent = _common.util.py3_path( os.path.join(self.temp_dir, b'nonexist')) non_exist_path = _common.util.py3_path(os.path.join( non_exist_path_parent.encode(), str(random()).encode())) - dbcore.Database(non_exist_path) + with control_stdin('n'): + dbcore.Database(non_exist_path) if os.path.exists(non_exist_path_parent): shutil.rmtree(non_exist_path_parent) raise OSError("Should not create dir") From 2886296c86eba136effff50a5149e76ed94f1d0f Mon Sep 17 00:00:00 2001 From: alicezou Date: Tue, 29 Mar 2022 21:24:13 -0400 Subject: [PATCH 218/357] fix code review comments --- beets/dbcore/db.py | 11 ----------- beets/ui/__init__.py | 13 +++++++++++++ beets/ui/commands.py | 3 ++- test/test_dbcore.py | 22 ---------------------- test/test_ui_init.py | 40 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 53 insertions(+), 36 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 397dbcedf..acd131be2 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -901,19 +901,8 @@ class Database: data is written in a transaction. """ - # Check whether parental directories exist. - def _path_checker(self, path): - if not isinstance(path, bytes) and path == ':memory:': # in memory db - return - newpath = os.path.dirname(path) - if not os.path.isdir(newpath): - from beets.ui.commands import database_dir_creation - if database_dir_creation(newpath): - os.makedirs(newpath) - def __init__(self, path, timeout=5.0): self.path = path - self._path_checker(path) self.timeout = timeout self._connections = {} diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 121cb5dc0..b724a963a 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1206,11 +1206,24 @@ def _configure(options): util.displayable_path(config.config_dir())) return config +# Check whether parental directories exist. + + +def _check_db_directory_exists(path): + if path == b':memory:': # in memory db + return + newpath = os.path.dirname(path) + if not os.path.isdir(newpath): + from beets.ui.commands import database_dir_creation + if database_dir_creation(newpath): + os.makedirs(newpath) + def _open_library(config): """Create a new library instance from the configuration. """ dbpath = util.bytestring_path(config['library'].as_filename()) + _check_db_directory_exists(dbpath) try: lib = library.Library( dbpath, diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 394a2831a..1261b1776 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1404,7 +1404,8 @@ default_commands.append(version_cmd) def database_dir_creation(path): # Ask the user for a choice. - return ui.input_yn("The database directory {} does not exists, create it (Y/n)?" + return ui.input_yn("The database directory {} does not \ + exists, create it (Y/n)?" .format(displayable_path(path))) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 95c52196f..80d85c3bb 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -19,11 +19,8 @@ import os import shutil import sqlite3 import unittest -from random import random -from unittest import mock from test import _common -from test.helper import control_stdin from beets import dbcore from tempfile import mkstemp @@ -763,25 +760,6 @@ class ResultsIteratorTest(unittest.TestCase): ModelFixture1, dbcore.query.FalseQuery()).get()) -class ParentalDirCreation(_common.TestCase): - def test_create_yes(self): - non_exist_path = _common.util.py3_path(os.path.join( - self.temp_dir, b'nonexist', str(random()).encode())) - with control_stdin('y'): - dbcore.Database(non_exist_path) - - def test_create_no(self): - non_exist_path_parent = _common.util.py3_path( - os.path.join(self.temp_dir, b'nonexist')) - non_exist_path = _common.util.py3_path(os.path.join( - non_exist_path_parent.encode(), str(random()).encode())) - with control_stdin('n'): - dbcore.Database(non_exist_path) - if os.path.exists(non_exist_path_parent): - shutil.rmtree(non_exist_path_parent) - raise OSError("Should not create dir") - - def suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/test/test_ui_init.py b/test/test_ui_init.py index bb9a922a5..9f9487a6a 100644 --- a/test/test_ui_init.py +++ b/test/test_ui_init.py @@ -15,11 +15,16 @@ """Test module for file ui/__init__.py """ - +import os +import shutil import unittest -from test import _common +from random import random +from copy import deepcopy from beets import ui +from test import _common +from test.helper import control_stdin +from beets import config class InputMethodsTest(_common.TestCase): @@ -121,8 +126,39 @@ class InitTest(_common.LibTestCase): self.assertEqual(h, ui.human_seconds(i)) +class ParentalDirCreation(_common.TestCase): + def test_create_yes(self): + non_exist_path = _common.util.py3_path(os.path.join( + self.temp_dir, b'nonexist', str(random()).encode())) + # Deepcopy instead of recovering because exceptions might + # occcur; wish I can use a golang defer here. + test_config = deepcopy(config) + test_config['library'] = non_exist_path + with control_stdin('y'): + ui._open_library(test_config) + + def test_create_no(self): + non_exist_path_parent = _common.util.py3_path( + os.path.join(self.temp_dir, b'nonexist')) + non_exist_path = _common.util.py3_path(os.path.join( + non_exist_path_parent.encode(), str(random()).encode())) + test_config = deepcopy(config) + test_config['library'] = non_exist_path + + with control_stdin('n'): + try: + ui._open_library(test_config) + except ui.UserError: + if os.path.exists(non_exist_path_parent): + shutil.rmtree(non_exist_path_parent) + raise OSError("Parent directories should not be created.") + else: + raise OSError("Parent directories should not be created.") + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) + if __name__ == '__main__': unittest.main(defaultTest='suite') From b609cae1116e4bc9c73ac3748bc75f32eed7f682 Mon Sep 17 00:00:00 2001 From: alicezou Date: Wed, 30 Mar 2022 12:56:38 -0400 Subject: [PATCH 219/357] change location for database_dir_creation, change docs --- beets/ui/__init__.py | 8 +++++++- beets/ui/commands.py | 7 ------- docs/changelog.rst | 3 ++- docs/guides/main.rst | 3 +-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index b724a963a..a50c3464f 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1209,12 +1209,18 @@ def _configure(options): # Check whether parental directories exist. +def database_dir_creation(path): + # Ask the user for a choice. + return input_yn("The database directory {} does not \ + exists. Create it (Y/n)?" + .format(util.displayable_path(path))) + + def _check_db_directory_exists(path): if path == b':memory:': # in memory db return newpath = os.path.dirname(path) if not os.path.isdir(newpath): - from beets.ui.commands import database_dir_creation if database_dir_creation(newpath): os.makedirs(newpath) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 1261b1776..a18b09f7d 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1402,13 +1402,6 @@ default_commands.append(version_cmd) # wants to create the parent directories. -def database_dir_creation(path): - # Ask the user for a choice. - return ui.input_yn("The database directory {} does not \ - exists, create it (Y/n)?" - .format(displayable_path(path))) - - # modify: Declaratively change metadata. def modify_items(lib, mods, dels, query, write, move, album, confirm): diff --git a/docs/changelog.rst b/docs/changelog.rst index fe4afa810..4126a8cce 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,8 +7,9 @@ Changelog Changelog goes here! New features: -* Create the parental directories for database if they do not exist. +* Create the parental directories for database if they do not exist. + :bug:`3808` :bug:`4327` * :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling the MusicBrainz metadata source during the autotagging process * :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances diff --git a/docs/guides/main.rst b/docs/guides/main.rst index feb918570..8dbb113c4 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -142,8 +142,7 @@ place to start:: Change that first path to a directory where you'd like to keep your music. Then, for ``library``, choose a good place to keep a database file that keeps an index -of your music. Beets will prompt you if the parental directories for database do -not exist. (The config's format is `YAML`_. You'll want to configure your +of your music. (The config's format is `YAML`_. You'll want to configure your text editor to use spaces, not real tabs, for indentation. Also, ``~`` means your home directory in these paths, even on Windows.) From fa5862d7d55c387accfbdc5b29e2d2dea6c7dbfc Mon Sep 17 00:00:00 2001 From: alicezou Date: Thu, 7 Apr 2022 15:21:34 -0400 Subject: [PATCH 220/357] change func name, inline function, fix typo --- beets/ui/__init__.py | 17 +++++------------ beets/ui/commands.py | 3 --- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index a50c3464f..365a0d529 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1206,22 +1206,15 @@ def _configure(options): util.displayable_path(config.config_dir())) return config -# Check whether parental directories exist. - -def database_dir_creation(path): - # Ask the user for a choice. - return input_yn("The database directory {} does not \ - exists. Create it (Y/n)?" - .format(util.displayable_path(path))) - - -def _check_db_directory_exists(path): +def _ensure_db_directory_exists(path): if path == b':memory:': # in memory db return newpath = os.path.dirname(path) if not os.path.isdir(newpath): - if database_dir_creation(newpath): + if input_yn("The database directory {} does not \ + exist. Create it (Y/n)?" + .format(util.displayable_path(newpath))): os.makedirs(newpath) @@ -1229,7 +1222,7 @@ def _open_library(config): """Create a new library instance from the configuration. """ dbpath = util.bytestring_path(config['library'].as_filename()) - _check_db_directory_exists(dbpath) + _ensure_db_directory_exists(dbpath) try: lib = library.Library( dbpath, diff --git a/beets/ui/commands.py b/beets/ui/commands.py index a18b09f7d..3a3374013 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1398,9 +1398,6 @@ version_cmd = ui.Subcommand( version_cmd.func = show_version default_commands.append(version_cmd) -# database_location: return true if user -# wants to create the parent directories. - # modify: Declaratively change metadata. From 9e753c5c2867eb4ab85f92ac0e42be566727234c Mon Sep 17 00:00:00 2001 From: arbanhossain Date: Fri, 15 Apr 2022 20:26:31 +0600 Subject: [PATCH 221/357] added flag to exclude/disable plugins --- beets/ui/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 365a0d529..19abb37bd 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1127,6 +1127,10 @@ def _load_plugins(options, config): if len(options.plugins) > 0 else []) else: plugin_list = config['plugins'].as_str_seq() + + # Exclude any plugins that were specified on the command line + if options.exclude is not None: + plugin_list = [p for p in plugin_list if p not in options.exclude.split(',')] plugins.load_plugins(plugin_list) return plugins @@ -1261,6 +1265,8 @@ def _raw_main(args, lib=None): help='path to configuration file') parser.add_option('-p', '--plugins', dest='plugins', help='a comma-separated list of plugins to load') + parser.add_option('-x', '--exclude', dest='exclude', + help='a comma-separated list of plugins to disable') parser.add_option('-h', '--help', dest='help', action='store_true', help='show this help message and exit') parser.add_option('--version', dest='version', action='store_true', From 9dfb80b661525e38e32b8e851b40a2492f743f9c Mon Sep 17 00:00:00 2001 From: arbanhossain Date: Fri, 15 Apr 2022 20:26:54 +0600 Subject: [PATCH 222/357] documentation and changelog for --exclude flag --- docs/changelog.rst | 2 ++ docs/reference/cli.rst | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4126a8cce..a86c975fb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,8 @@ New features: * :doc:`/plugins/convert`: Add a new `auto_keep` option that automatically converts files but keeps the *originals* in the library. :bug:`1840` :bug:`4302` +* Added a ``-x`` (or ``--exclude``) flag to specify one/multiple plugin(s) to be + disabled at startup. Bug fixes: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 214956873..4731dd0e7 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -445,6 +445,10 @@ import ...``. specified, the plugin list in your configuration is ignored. The long form of this argument also allows specifying no plugins, effectively disabling all plugins: ``--plugins=``. +* ``-x EXCLUDE``: specify a comma-separated list of plugins to disable in a + specific beets run. If specified, it will exclude plugins from your configuration + and/or plugins specified using the ``-p`` flag. To disable ALL plugins, use + ``--plugins=`` instead. Beets also uses the ``BEETSDIR`` environment variable to look for configuration and data. From c0bb2ff2a175a4a5cce2db590d1d065c14475cd1 Mon Sep 17 00:00:00 2001 From: olgarrahan Date: Fri, 15 Apr 2022 16:37:05 -0400 Subject: [PATCH 223/357] Genius lyrics header bug fixed and updated test case for lyrics plugin --- beetsplug/lyrics.py | 8 +- .../lyrics/geniuscom/Ttngchinchillalyrics.txt | 864 ++++++++++++++++++ test/test_lyrics.py | 2 +- 3 files changed, 871 insertions(+), 3 deletions(-) create mode 100644 test/rsrc/lyrics/geniuscom/Ttngchinchillalyrics.txt diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 1f215df45..e73c3bc8a 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -401,7 +401,12 @@ class Genius(Backend): # all of the lyrics can be found already correctly formatted # Sometimes, though, it packages the lyrics into separate divs, most # likely for easier ad placement - lyrics_div = soup.find("div", class_="lyrics") + + lyrics_div = soup.find("div", {"data-lyrics-container": True}) + + for br in lyrics_div.find_all("br"): + br.replace_with("\n") + if not lyrics_div: self._log.debug('Received unusual song page html') verse_div = soup.find("div", @@ -429,7 +434,6 @@ class Genius(Backend): class_=re.compile("Lyrics__Footer")) for footer in footers: footer.replace_with("") - return lyrics_div.get_text() diff --git a/test/rsrc/lyrics/geniuscom/Ttngchinchillalyrics.txt b/test/rsrc/lyrics/geniuscom/Ttngchinchillalyrics.txt new file mode 100644 index 000000000..fa28a1b2a --- /dev/null +++ b/test/rsrc/lyrics/geniuscom/Ttngchinchillalyrics.txt @@ -0,0 +1,864 @@ + + + + TTNG – Chinchilla Lyrics | Genius Lyrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

🚧  The new song page is now the default experience! We need your help to continue improving contributor features.  🚧

FeaturedChartsVideosForumsAdd A SongPromote Your Music
Forums
Feed
Me
1
Messages
1
Earn IQ
100 IQ
Cover art for Chinchilla by TTNG

Chinchilla

TTNG
Track 1 on 
Animals 
This song is about those relationships with a lot of fights and reconciliations. The singer and his couple are aruging/reconciliating, telling themselves everything is going to… Read More 

Produced by

Max Read

Release Date

October 13, 2008
View All Credits 
115K
+
Click here to go to the old song page + +
So far we've lost focus
Let's just concentrate on words that could mean everything

On nights like this
We drink ourselves dry
And make promises
Without intention

So fortunate that this was brought up
The last time. As I recall
I can’t hold up your every expectation

On nights like this
We drink ourselves dry
And make promises
Without intention

My God, is this what we’ve become?
Living parodies of love and loss
Can we really be all that lost?

So fortunate that this was brought up
The last time. As I recall
I can’t hold up your every expectation

One moment to another I am restless
Seems making love forever can often risk your heart
And I cannot remember when I was this messed up
In service of another I am beautiful
How to Format Lyrics:
  • Type out all lyrics, even if it’s a chorus that’s repeated throughout the song
  • The Section Header button breaks up song sections. Highlight the text then click the link
  • Use Bold and Italics only to distinguish between different singers in the same verse.
    • E.g. “Verse 1: Kanye West, Jay-Z, Both”
  • Capitalize each line
  • To move an annotation to different lyrics in the song, use the [...] menu to switch to referent editing mode

About

This song bio is unreviewed
Genius Annotation

This song is about those relationships with a lot of fights and reconciliations. The singer and his couple are aruging/reconciliating, telling themselves everything is going to be better and things will change for good, specially when they get drunk, just to fight and reconciliate over and over again.

Ask us a question about this song
No questions asked yet
Animals (2008)
TTNG
  1. 1.
    Chinchilla
  2. 2.
    Baboon
  3. 3.
    Lemur
  4. 4.
    Badger
  5. 5.
    Quetzal
  6. 6.
    Panda
  7. 7.
    Elk
  8. 8.
    Pig
  9. 9.
    Gibbon
  10. 10.
    Dog
  11. 11.
    Crocodile
  12. 12.
    Rabbit
  13. 13.
    Zebra
Credits
Written By
Stuart Smith
Release Date
October 13, 2008
Tags
RockIndie RockMath Rock
Comments
Add a comment
Get the conversation started
Be the first to comment

Genius is the world’s biggest collection of song lyrics and musical knowledge

About GeniusContributor GuidelinesPressShopAdvertiseEvent SpacePrivacy PolicyDelete Account
LicensingJobsDevelopersCopyright PolicyContact Us
Sign Out
Do Not Sell My Personal Information
© 2022 Genius Media Group Inc.
Terms of Use
Verified Artists
All Artists:
abcdefghijklmnopqrstuvwxyz#
Hot Songs:
As It WasFlowersFirst ClassMemoriesSun and MoonView All
+ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 3adf6e359..57f5ce13d 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -457,7 +457,7 @@ class GeniusScrapeLyricsFromHtmlTest(GeniusBaseTest): def test_good_lyrics(self): """Ensure we are able to scrape a page with lyrics""" - url = 'https://genius.com/Wu-tang-clan-cream-lyrics' + url = 'https://genius.com/Ttng-chinchilla-lyrics' mock = MockFetchUrl() self.assertIsNotNone(genius._scrape_lyrics_from_html(mock(url))) From 6be2617eb1170c798b577f890a32f2953d483bb8 Mon Sep 17 00:00:00 2001 From: arbanhossain Date: Sat, 16 Apr 2022 11:56:30 +0600 Subject: [PATCH 224/357] changed -x/--exclude flag to -P/--disable-plugin --- beets/ui/__init__.py | 7 ++++--- docs/changelog.rst | 2 +- docs/reference/cli.rst | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 19abb37bd..2740c5f69 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1127,10 +1127,11 @@ def _load_plugins(options, config): if len(options.plugins) > 0 else []) else: plugin_list = config['plugins'].as_str_seq() - + # Exclude any plugins that were specified on the command line if options.exclude is not None: - plugin_list = [p for p in plugin_list if p not in options.exclude.split(',')] + plugin_list = [p for p in plugin_list + if p not in options.exclude.split(',')] plugins.load_plugins(plugin_list) return plugins @@ -1265,7 +1266,7 @@ def _raw_main(args, lib=None): help='path to configuration file') parser.add_option('-p', '--plugins', dest='plugins', help='a comma-separated list of plugins to load') - parser.add_option('-x', '--exclude', dest='exclude', + parser.add_option('-P', '--disable-plugin', dest='exclude', help='a comma-separated list of plugins to disable') parser.add_option('-h', '--help', dest='help', action='store_true', help='show this help message and exit') diff --git a/docs/changelog.rst b/docs/changelog.rst index a86c975fb..ef9f550ff 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,7 +20,7 @@ New features: * :doc:`/plugins/convert`: Add a new `auto_keep` option that automatically converts files but keeps the *originals* in the library. :bug:`1840` :bug:`4302` -* Added a ``-x`` (or ``--exclude``) flag to specify one/multiple plugin(s) to be +* Added a ``-P`` (or ``--disable-plugin``) flag to specify one/multiple plugin(s) to be disabled at startup. Bug fixes: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 4731dd0e7..319a7897d 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -445,7 +445,7 @@ import ...``. specified, the plugin list in your configuration is ignored. The long form of this argument also allows specifying no plugins, effectively disabling all plugins: ``--plugins=``. -* ``-x EXCLUDE``: specify a comma-separated list of plugins to disable in a +* ``-P EXCLUDE``: specify a comma-separated list of plugins to disable in a specific beets run. If specified, it will exclude plugins from your configuration and/or plugins specified using the ``-p`` flag. To disable ALL plugins, use ``--plugins=`` instead. From 16d74bafc3ab5c9837950718a380001f54e07fc6 Mon Sep 17 00:00:00 2001 From: olgarrahan Date: Sat, 16 Apr 2022 13:19:13 -0400 Subject: [PATCH 225/357] Genius lyrics header bug fixed and updated test case for lyrics plugin --- beetsplug/lyrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index e73c3bc8a..820d16abf 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -404,8 +404,9 @@ class Genius(Backend): lyrics_div = soup.find("div", {"data-lyrics-container": True}) - for br in lyrics_div.find_all("br"): - br.replace_with("\n") + if lyrics_div: + for br in lyrics_div.find_all("br"): + br.replace_with("\n") if not lyrics_div: self._log.debug('Received unusual song page html') From 73554acfb06913173f839b143d512a79fcb50a6a Mon Sep 17 00:00:00 2001 From: arbanhossain Date: Sun, 17 Apr 2022 11:12:27 +0600 Subject: [PATCH 226/357] changed --disable-plugin to --disable-plugins --- beets/ui/__init__.py | 4 ++-- docs/changelog.rst | 2 +- docs/reference/cli.rst | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 2740c5f69..acc1a7fc3 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1131,7 +1131,7 @@ def _load_plugins(options, config): # Exclude any plugins that were specified on the command line if options.exclude is not None: plugin_list = [p for p in plugin_list - if p not in options.exclude.split(',')] + if p not in options.exclude.split(',')] plugins.load_plugins(plugin_list) return plugins @@ -1266,7 +1266,7 @@ def _raw_main(args, lib=None): help='path to configuration file') parser.add_option('-p', '--plugins', dest='plugins', help='a comma-separated list of plugins to load') - parser.add_option('-P', '--disable-plugin', dest='exclude', + parser.add_option('-P', '--disable-plugins', dest='exclude', help='a comma-separated list of plugins to disable') parser.add_option('-h', '--help', dest='help', action='store_true', help='show this help message and exit') diff --git a/docs/changelog.rst b/docs/changelog.rst index ef9f550ff..53d859f4b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,7 +20,7 @@ New features: * :doc:`/plugins/convert`: Add a new `auto_keep` option that automatically converts files but keeps the *originals* in the library. :bug:`1840` :bug:`4302` -* Added a ``-P`` (or ``--disable-plugin``) flag to specify one/multiple plugin(s) to be +* Added a ``-P`` (or ``--disable-plugins``) flag to specify one/multiple plugin(s) to be disabled at startup. Bug fixes: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 319a7897d..a5f44f91f 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -445,9 +445,8 @@ import ...``. specified, the plugin list in your configuration is ignored. The long form of this argument also allows specifying no plugins, effectively disabling all plugins: ``--plugins=``. -* ``-P EXCLUDE``: specify a comma-separated list of plugins to disable in a - specific beets run. If specified, it will exclude plugins from your configuration - and/or plugins specified using the ``-p`` flag. To disable ALL plugins, use +* ``-P plugins``: specify a comma-separated list of plugins to disable in a + specific beets run. This will overwrite ``-p`` if used with it . To disable all plugins, use ``--plugins=`` instead. Beets also uses the ``BEETSDIR`` environment variable to look for From a26a53099623c696c389c5d9c0b25a9108019495 Mon Sep 17 00:00:00 2001 From: olgarrahan Date: Sun, 17 Apr 2022 15:25:14 -0400 Subject: [PATCH 227/357] replace_br function added to reduce code duplication --- beetsplug/lyrics.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 820d16abf..6e0439271 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -387,6 +387,10 @@ class Genius(Backend): except ValueError: return None + def replace_br(self, lyrics_div): + for br in lyrics_div.find_all("br"): + br.replace_with("\n") + def _scrape_lyrics_from_html(self, html): """Scrape lyrics from a given genius.com html""" @@ -405,8 +409,7 @@ class Genius(Backend): lyrics_div = soup.find("div", {"data-lyrics-container": True}) if lyrics_div: - for br in lyrics_div.find_all("br"): - br.replace_with("\n") + self.replace_br(lyrics_div) if not lyrics_div: self._log.debug('Received unusual song page html') @@ -423,8 +426,7 @@ class Genius(Backend): return None lyrics_div = verse_div.parent - for br in lyrics_div.find_all("br"): - br.replace_with("\n") + self.replace_br(lyrics_div) ads = lyrics_div.find_all("div", class_=re.compile("InreadAd__Container")) From 0e006f116a0578a02dac8ae73fd2d62f3cb337a0 Mon Sep 17 00:00:00 2001 From: olgarrahan Date: Sun, 17 Apr 2022 21:19:26 -0400 Subject: [PATCH 228/357] changelog updates --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4126a8cce..366981684 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -66,6 +66,8 @@ Bug fixes: * :doc:`plugins/embedart`: Fix a crash when using recent versions of ImageMagick and the ``compare_threshold`` option. :bug:`4272` +* :doc:`plugins/lyrics`: Fixed issue with Genius header being included in lyrics, + added test case of up-to-date Genius html For packagers: From 58ea48aa3b9f06055102c13bedef16733801397c Mon Sep 17 00:00:00 2001 From: arbanhossain Date: Tue, 19 Apr 2022 11:22:31 +0600 Subject: [PATCH 229/357] removed over-indentation --- beets/ui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index acc1a7fc3..900cb9305 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1131,7 +1131,7 @@ def _load_plugins(options, config): # Exclude any plugins that were specified on the command line if options.exclude is not None: plugin_list = [p for p in plugin_list - if p not in options.exclude.split(',')] + if p not in options.exclude.split(',')] plugins.load_plugins(plugin_list) return plugins From 85db3059724fcce15b20af34e7d5374bf2ef1d72 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 19 Apr 2022 09:19:23 -0400 Subject: [PATCH 230/357] Remove errant space --- docs/reference/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index a5f44f91f..dad407489 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -446,7 +446,7 @@ import ...``. of this argument also allows specifying no plugins, effectively disabling all plugins: ``--plugins=``. * ``-P plugins``: specify a comma-separated list of plugins to disable in a - specific beets run. This will overwrite ``-p`` if used with it . To disable all plugins, use + specific beets run. This will overwrite ``-p`` if used with it. To disable all plugins, use ``--plugins=`` instead. Beets also uses the ``BEETSDIR`` environment variable to look for From ca056ff7fb383b072ec70ba8c562c5fb6761d61a Mon Sep 17 00:00:00 2001 From: Tom Hu <88201630+thomasrockhu-codecov@users.noreply.github.com> Date: Wed, 20 Apr 2022 23:04:37 -0400 Subject: [PATCH 231/357] chore(ci): add informational Codecov status checks Hi, Tom from Codecov here. I noticed that you were using Codecov but weren't actually getting any notifications on pull requests. I figured it would be useful to get some idea if code being changed is being tested, but also not blocking CI/merging. Let me know if this makes sense or if we can do something that would be helpful. --- codecov.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/codecov.yml b/codecov.yml index cbbe408ba..3b702eef0 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,9 +1,13 @@ # Don't post a comment on pull requests. comment: off -# I think this disables commit statuses? +# Sets non-blocking status checks coverage: - status: - project: no - patch: no - changes: no + status: + project: + default: + informational: true + patch: + default: + informational: true + changes: no From 7cceb8cac4292967bd12f90b21fd70d29f4ede76 Mon Sep 17 00:00:00 2001 From: Tom Hu <88201630+thomasrockhu-codecov@users.noreply.github.com> Date: Sat, 23 Apr 2022 11:43:01 -0700 Subject: [PATCH 232/357] Update codecov.yml --- codecov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codecov.yml b/codecov.yml index 3b702eef0..47a8ccc21 100644 --- a/codecov.yml +++ b/codecov.yml @@ -2,6 +2,7 @@ comment: off # Sets non-blocking status checks +# https://docs.codecov.com/docs/commit-status#informational coverage: status: project: From 2ccc6bad7002ca7b0914e450425f33ee768b4380 Mon Sep 17 00:00:00 2001 From: Florian Rudin Date: Tue, 3 May 2022 21:39:08 +0800 Subject: [PATCH 233/357] Update index.rst typo --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 80a50e915..123d1b1ce 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ Then you can get a more detailed look at beets' features in the :doc:`/reference/cli/` and :doc:`/reference/config` references. You might also be interested in exploring the :doc:`plugins `. -If you still need help, your can drop by the ``#beets`` IRC channel on +If you still need help, you can drop by the ``#beets`` IRC channel on Libera.Chat, drop by `the discussion board`_, send email to `the mailing list`_, or `file a bug`_ in the issue tracker. Please let us know where you think this documentation can be improved. From a09f5e7dfee72495a18244d8568dd63b8823cbcb Mon Sep 17 00:00:00 2001 From: Quinn Casey Date: Thu, 5 May 2022 14:04:54 -0700 Subject: [PATCH 234/357] Update lyricstext.yaml --- test/rsrc/lyricstext.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/rsrc/lyricstext.yaml b/test/rsrc/lyricstext.yaml index fb698bbec..4cec7802a 100644 --- a/test/rsrc/lyricstext.yaml +++ b/test/rsrc/lyricstext.yaml @@ -57,6 +57,6 @@ Black_magic_woman: | u_n_eye: | let see cool bed for sometimes are place told in yeah or ride open hide blame knee your my borders perfect i of laying lies they love the night all out saying fast things said that on face hit hell - no low not bullets bullet fly time maybe over is roof a it know now airplane where tekst and tonight - brakes just waste we go an to you was going eye start need insane cross gotta historia mood life with - hurts too whoa me fight little every oh would thousand but high tekstu lay space do down private edycji + no low not bullets bullet fly time maybe over is roof a it know now airplane where and tonight + brakes just waste we go an to you was going eye start need insane cross gotta mood life with + hurts too whoa me fight little every oh would thousand but high lay space do down private From 1320944898ac90b9794667182d5639738e3e9987 Mon Sep 17 00:00:00 2001 From: Quinn Casey Date: Thu, 5 May 2022 15:33:32 -0700 Subject: [PATCH 235/357] Update test_lyrics.py --- test/test_lyrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 57f5ce13d..e3e3be96f 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -245,8 +245,8 @@ class LyricsAssertions: if not keywords <= words: details = ( f"{keywords!r} is not a subset of {words!r}." - f" Words only in first {keywords - words!r}," - f" Words only in second {words - keywords!r}." + f" Words only in expected set {keywords - words!r}," + f" Words only in result set {words - keywords!r}." ) self.fail(f"{details} : {msg}") From 1d43ea2dbaa86fe9ead54c028b0298675ecf4c2c Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 8 May 2022 12:21:44 -0400 Subject: [PATCH 236/357] Save Spotify album_id and track_id as flexible attributes --- beetsplug/spotify.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 931078d28..5b554bc5b 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -214,8 +214,10 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return AlbumInfo( album=album_data['name'], album_id=spotify_id, + spotify_album_id=spotify_id, artist=artist, artist_id=artist_id, + spotify_artist_id=artist_id, tracks=tracks, albumtype=album_data['album_type'], va=len(album_data['artists']) == 1 @@ -242,8 +244,10 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return TrackInfo( title=track_data['name'], track_id=track_data['id'], + spotify_track_id=track_id, artist=artist, artist_id=artist_id, + spotify_artist_id=artist_id, length=track_data['duration_ms'] / 1000, index=track_data['track_number'], medium=track_data['disc_number'], From 1b02d65112e3f31f35ea5d578d81d35d2c340b8a Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 8 May 2022 12:42:20 -0400 Subject: [PATCH 237/357] Fix return --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 5b554bc5b..eff717a9f 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -244,7 +244,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return TrackInfo( title=track_data['name'], track_id=track_data['id'], - spotify_track_id=track_id, + spotify_track_id=track_data['id'], artist=artist, artist_id=artist_id, spotify_artist_id=artist_id, From c5c34e4cdadc959553e3eae4e1575a3d3607adf9 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 8 May 2022 14:02:45 -0400 Subject: [PATCH 238/357] Update changelog --- docs/changelog.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8c4a0ed5b..02553275f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,10 +7,10 @@ Changelog Changelog goes here! New features: - +* Save Spotify `album_id`, `artist_id` and `track_id` information. Partial fix for `4347`. * Create the parental directories for database if they do not exist. :bug:`3808` :bug:`4327` -* :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling +* :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling the MusicBrainz metadata source during the autotagging process * :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances :bug:`4101` @@ -20,7 +20,7 @@ New features: * :doc:`/plugins/convert`: Add a new `auto_keep` option that automatically converts files but keeps the *originals* in the library. :bug:`1840` :bug:`4302` -* Added a ``-P`` (or ``--disable-plugins``) flag to specify one/multiple plugin(s) to be +* Added a ``-P`` (or ``--disable-plugins``) flag to specify one/multiple plugin(s) to be disabled at startup. Bug fixes: From d8de9162bf59119f9635d3d878dde9a7cd7f0ba7 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 8 May 2022 14:11:24 -0400 Subject: [PATCH 239/357] Update changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 02553275f..c2e35b627 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changelog Changelog goes here! New features: + * Save Spotify `album_id`, `artist_id` and `track_id` information. Partial fix for `4347`. * Create the parental directories for database if they do not exist. :bug:`3808` :bug:`4327` From 8881ae62f7882e935c62b67f2a8438d364cd42ff Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 8 May 2022 14:19:21 -0400 Subject: [PATCH 240/357] Update changelog.rst --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c2e35b627..8751ce0cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ Changelog goes here! New features: -* Save Spotify `album_id`, `artist_id` and `track_id` information. Partial fix for `4347`. +* Save Spotify `album_id`, `artist_id`, and `track_id` information * Create the parental directories for database if they do not exist. :bug:`3808` :bug:`4327` * :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling From e62a904f950b9805bd407266e1f4bacc78ff852d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 8 May 2022 16:03:10 -0400 Subject: [PATCH 241/357] Expand changelog entry --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8751ce0cf..e5cbb66cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,9 @@ Changelog goes here! New features: -* Save Spotify `album_id`, `artist_id`, and `track_id` information +* :doc:`/pluings/spotify`: The plugin now records Spotify-specific IDs in the + `spotify_album_id`, `spotify_artist_id`, and `spotify_track_id` fields. + :bug:`4348` * Create the parental directories for database if they do not exist. :bug:`3808` :bug:`4327` * :ref:`musicbrainz-config`: a new :ref:`musicbrainz.enabled` option allows disabling From e607763028495badfda68098f18220eb0dada9fe Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 8 May 2022 16:08:20 -0400 Subject: [PATCH 242/357] Fix typo! --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e5cbb66cc..72b1cf1fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ Changelog goes here! New features: -* :doc:`/pluings/spotify`: The plugin now records Spotify-specific IDs in the +* :doc:`/plugins/spotify`: The plugin now records Spotify-specific IDs in the `spotify_album_id`, `spotify_artist_id`, and `spotify_track_id` fields. :bug:`4348` * Create the parental directories for database if they do not exist. From b1ad49a05459f2378a9e959f7186e77849de2256 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 12 May 2022 20:09:40 -0400 Subject: [PATCH 243/357] Update spotify.py --- beetsplug/spotify.py | 105 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index eff717a9f..e0907875b 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -356,6 +356,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return response_data def commands(self): + # autotagger import command def queries(lib, opts, args): success = self._parse_opts(opts) if success: @@ -382,7 +383,23 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): ), ) spotify_cmd.func = queries - return [spotify_cmd] + + # spotifysync command + sync_cmd = ui.Subcommand('spotifysync', + help="fetch track attributes from Spotify") + sync_cmd.parser.add_option( + '-f', '--force', dest='force_refetch', + action='store_true', default=False, + help='re-download data when already present' + ) + + def func(lib, opts, args): + items = lib.items(ui.decargs(args)) + self._fetch_info(items, ui.should_write(), + opts.force_refetch or self.config['force']) + + sync_cmd.func = func + return [spotify_cmd, sync_cmd] def _parse_opts(self, opts): if opts.mode: @@ -536,3 +553,89 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self._log.warning( f'No {self.data_source} tracks found from beets query' ) + + def _fetch_info(self, items, write, force): + """Obtain track information from Spotify.""" + spotify_audio_features = { + 'acousticness': ['spotify_track_acousticness'], + 'danceability': ['spotify_track_danceability'], + 'energy': ['spotify_track_energy'], + 'instrumentalness': ['spotify_track_instrumentalness'], + 'key': ['spotify_track_key'], + 'liveness': ['spotify_track_liveness'], + 'loudness': ['spotify_track_loudness'], + 'mode': ['spotify_track_mode'], + 'speechiness': ['spotify_track_speechiness'], + 'tempo': ['spotify_track_tempo'], + 'time_signature': ['spotify_track_time_sig'], + 'valence': ['spotify_track_valence'], + } + import time + + no_items = len(items) + self._log.info('Total {} tracks', no_items) + + for index, item in enumerate(items, start=1): + time.sleep(.5) + self._log.info('Processing {}/{} tracks - {} ', + index, no_items, item) + try: + # If we're not forcing re-downloading for all tracks, check + # whether the popularity data is already present + if not force: + spotify_track_popularity = \ + item.get('spotify_track_popularity', '') + if spotify_track_popularity: + self._log.debug('Popularity already present for: {}', + item) + continue + + popularity = self.track_popularity(item.spotify_track_id) + item['spotify_track_popularity'] = popularity + audio_features = \ + self.track_audio_features(item.spotify_track_id) + for feature in audio_features.keys(): + if feature in spotify_audio_features.keys(): + item[spotify_audio_features[feature][0]] = \ + audio_features[feature] + item.store() + if write: + item.try_write() + except AttributeError: + self._log.debug('No track_id present for: {}', item) + pass + + def track_popularity(self, track_id=None): + """Fetch a track popularity by its Spotify ID. + :param track_id: (Optional) Spotify ID or URL for the track. Either + ``track_id`` or ``track_data`` must be provided. + :type track_id: str + :param track_data: (Optional) Simplified track object dict. May be + provided instead of ``track_id`` to avoid unnecessary API calls. + :type track_data: dict + :return: TrackInfo object for track + :rtype: beets.autotag.hooks.TrackInfo or None + """ + track_data = self._handle_response( + requests.get, self.track_url + track_id + ) + self._log.debug('track_data: {}', track_data['popularity']) + track_popularity = track_data['popularity'] + return track_popularity + + def track_audio_features(self, track_id=None): + """Fetch track features by its Spotify ID. + :param track_id: (Optional) Spotify ID or URL for the track. Either + ``track_id`` or ``track_data`` must be provided. + :type track_id: str + :param track_data: (Optional) Simplified track object dict. May be + provided instead of ``track_id`` to avoid unnecessary API calls. + :type track_data: dict + :return: TrackInfo object for track + :rtype: beets.autotag.hooks.TrackInfo or None + """ + track_data = self._handle_response( + requests.get, self.audio_features_url + track_id + ) + audio_features = track_data + return audio_features From e8de749eafa76091ce02caada8ffc0ecb5f22dfb Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 12 May 2022 20:24:40 -0400 Subject: [PATCH 244/357] Clean up docstrings --- beetsplug/spotify.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index e0907875b..efb5e8381 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -606,16 +606,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): pass def track_popularity(self, track_id=None): - """Fetch a track popularity by its Spotify ID. - :param track_id: (Optional) Spotify ID or URL for the track. Either - ``track_id`` or ``track_data`` must be provided. - :type track_id: str - :param track_data: (Optional) Simplified track object dict. May be - provided instead of ``track_id`` to avoid unnecessary API calls. - :type track_data: dict - :return: TrackInfo object for track - :rtype: beets.autotag.hooks.TrackInfo or None - """ + """Fetch a track popularity by its Spotify ID.""" track_data = self._handle_response( requests.get, self.track_url + track_id ) @@ -624,16 +615,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return track_popularity def track_audio_features(self, track_id=None): - """Fetch track features by its Spotify ID. - :param track_id: (Optional) Spotify ID or URL for the track. Either - ``track_id`` or ``track_data`` must be provided. - :type track_id: str - :param track_data: (Optional) Simplified track object dict. May be - provided instead of ``track_id`` to avoid unnecessary API calls. - :type track_data: dict - :return: TrackInfo object for track - :rtype: beets.autotag.hooks.TrackInfo or None - """ + """Fetch track audio features by its Spotify ID.""" track_data = self._handle_response( requests.get, self.audio_features_url + track_id ) From ba3a582483186ebe6291e5e67eb7f1318fa874f8 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 12 May 2022 20:36:04 -0400 Subject: [PATCH 245/357] Update spotify.py --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index efb5e8381..20b133257 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -41,6 +41,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): search_url = 'https://api.spotify.com/v1/search' album_url = 'https://api.spotify.com/v1/albums/' track_url = 'https://api.spotify.com/v1/tracks/' + audio_features_url = 'https://api.spotify.com/v1/audio-features/' # Spotify IDs consist of 22 alphanumeric characters # (zero-left-padded base62 representation of randomly generated UUID4) From db1c77fb25bc70465d4e258991fa5d067b909368 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 12 May 2022 20:38:01 -0400 Subject: [PATCH 246/357] Update spotify.py --- beetsplug/spotify.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 20b133257..47ef6148a 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -597,6 +597,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self.track_audio_features(item.spotify_track_id) for feature in audio_features.keys(): if feature in spotify_audio_features.keys(): + self._log.info('feature: {}', + audio_features[feature]) item[spotify_audio_features[feature][0]] = \ audio_features[feature] item.store() From 9c9f52b7e5f17565b272bb74b7473c8b6c3108e3 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 12 May 2022 20:40:03 -0400 Subject: [PATCH 247/357] remove logging --- beetsplug/spotify.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 47ef6148a..20b133257 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -597,8 +597,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self.track_audio_features(item.spotify_track_id) for feature in audio_features.keys(): if feature in spotify_audio_features.keys(): - self._log.info('feature: {}', - audio_features[feature]) item[spotify_audio_features[feature][0]] = \ audio_features[feature] item.store() From 4c4cafbf04be9f1b17643d7b73a2563db9ddba6e Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 12 May 2022 20:42:13 -0400 Subject: [PATCH 248/357] Update spotify.py --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 20b133257..58bbc89ad 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -1,5 +1,6 @@ # This file is part of beets. # Copyright 2019, Rahul Ahuja. +# Copyright 2022, Alok Saboo. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the From d465471308da7b6eb63652969d863a7970a670c5 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 13 May 2022 08:42:17 -0400 Subject: [PATCH 249/357] Add force option in config --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 58bbc89ad..d4706ba63 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -66,6 +66,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): 'client_id': '4e414367a1d14c75a5c5129a627fcab8', 'client_secret': 'f82bdc09b2254f1a8286815d02fd46dc', 'tokenfile': 'spotify_token.json', + 'force': False, } ) self.config['client_secret'].redact = True From 4eb83e8d976111fc20547ca58a7f5dfd9a6684cc Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 13 May 2022 09:58:12 -0400 Subject: [PATCH 250/357] Save track popularity during the import to save an API call --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d4706ba63..0babe749c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -251,6 +251,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): artist=artist, artist_id=artist_id, spotify_artist_id=artist_id, + spotify_track_popularity=track_data['popularity'], length=track_data['duration_ms'] / 1000, index=track_data['track_number'], medium=track_data['disc_number'], From f6c0bdac75e700bac1070570897a8fc75b3e97f1 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 13 May 2022 10:03:08 -0400 Subject: [PATCH 251/357] revert --- beetsplug/spotify.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 0babe749c..d4706ba63 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -251,7 +251,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): artist=artist, artist_id=artist_id, spotify_artist_id=artist_id, - spotify_track_popularity=track_data['popularity'], length=track_data['duration_ms'] / 1000, index=track_data['track_number'], medium=track_data['disc_number'], From 80843d772050eb34e42cf3027a0cdb819d6179af Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 13 May 2022 10:08:56 -0400 Subject: [PATCH 252/357] Update spotify.py --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d4706ba63..30ef8fafa 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -251,6 +251,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): artist=artist, artist_id=artist_id, spotify_artist_id=artist_id, + spotify_track_popularity=track_data['tracks']['popularity'], length=track_data['duration_ms'] / 1000, index=track_data['track_number'], medium=track_data['disc_number'], From 72e037f1ed3ea71b5d0d1ec302ec04b208266c67 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 13 May 2022 10:13:15 -0400 Subject: [PATCH 253/357] Update spotify.py --- beetsplug/spotify.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 30ef8fafa..d4706ba63 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -251,7 +251,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): artist=artist, artist_id=artist_id, spotify_artist_id=artist_id, - spotify_track_popularity=track_data['tracks']['popularity'], length=track_data['duration_ms'] / 1000, index=track_data['track_number'], medium=track_data['disc_number'], From e715f2d9b0d69797ba88b504b8b4aaa83c4fb561 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Tue, 17 May 2022 14:15:17 -0400 Subject: [PATCH 254/357] Disable CodeCov annotations (see #4337) As described here: https://docs.codecov.com/docs/github-checks#disabling-github-checks-patch-annotations-via-yaml These inline annotations are pretty noisy and don't add much beyond the bottom-line summaries. --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codecov.yml b/codecov.yml index 47a8ccc21..c4b333ad3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -12,3 +12,6 @@ coverage: default: informational: true changes: no + +github_checks: + annotations: false From c66225708eaf1d7944ec8ae45c89b0f4a1d3a569 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 14:50:14 -0400 Subject: [PATCH 255/357] Update beetsplug/spotify.py Co-authored-by: Adrian Sampson --- beetsplug/spotify.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d4706ba63..8ac7755ea 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -586,9 +586,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): # If we're not forcing re-downloading for all tracks, check # whether the popularity data is already present if not force: - spotify_track_popularity = \ - item.get('spotify_track_popularity', '') - if spotify_track_popularity: + if 'spotify_track_popularity' in item: self._log.debug('Popularity already present for: {}', item) continue From d313da2765888af914100fef390cc2c7dcd3597a Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 14:50:27 -0400 Subject: [PATCH 256/357] Update beetsplug/spotify.py Co-authored-by: Adrian Sampson --- beetsplug/spotify.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 8ac7755ea..868a2c0f8 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -620,5 +620,4 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): track_data = self._handle_response( requests.get, self.audio_features_url + track_id ) - audio_features = track_data - return audio_features + return track_data From 39600bcbbbb3ec143e9f36f1bfa7e1d7ab1c35e7 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 14:50:34 -0400 Subject: [PATCH 257/357] Update beetsplug/spotify.py Co-authored-by: Adrian Sampson --- beetsplug/spotify.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 868a2c0f8..460f57c1e 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -612,8 +612,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): requests.get, self.track_url + track_id ) self._log.debug('track_data: {}', track_data['popularity']) - track_popularity = track_data['popularity'] - return track_popularity + return track_data['popularity'] def track_audio_features(self, track_id=None): """Fetch track audio features by its Spotify ID.""" From 9420cf4c6c03bd6a9ceb87c8ce1a3d8b0f004bdc Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 15:04:45 -0400 Subject: [PATCH 258/357] Address comments --- beetsplug/spotify.py | 56 +++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 460f57c1e..571def87c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -20,6 +20,7 @@ Spotify playlist construction. import re import json import base64 +import time import webbrowser import collections @@ -51,6 +52,21 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): 'match_group': 2, } + spotify_audio_features = { + 'acousticness': 'spotify_track_acousticness', + 'danceability': 'spotify_track_danceability', + 'energy': 'spotify_track_energy', + 'instrumentalness': 'spotify_track_instrumentalness', + 'key': 'spotify_track_key', + 'liveness': 'spotify_track_liveness', + 'loudness': 'spotify_track_loudness', + 'mode': 'spotify_track_mode', + 'speechiness': 'spotify_track_speechiness', + 'tempo': 'spotify_track_tempo', + 'time_signature': 'spotify_track_time_sig', + 'valence': 'spotify_track_valence', + } + def __init__(self): super().__init__() self.config.add( @@ -66,7 +82,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): 'client_id': '4e414367a1d14c75a5c5129a627fcab8', 'client_secret': 'f82bdc09b2254f1a8286815d02fd46dc', 'tokenfile': 'spotify_token.json', - 'force': False, } ) self.config['client_secret'].redact = True @@ -559,45 +574,28 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): def _fetch_info(self, items, write, force): """Obtain track information from Spotify.""" - spotify_audio_features = { - 'acousticness': ['spotify_track_acousticness'], - 'danceability': ['spotify_track_danceability'], - 'energy': ['spotify_track_energy'], - 'instrumentalness': ['spotify_track_instrumentalness'], - 'key': ['spotify_track_key'], - 'liveness': ['spotify_track_liveness'], - 'loudness': ['spotify_track_loudness'], - 'mode': ['spotify_track_mode'], - 'speechiness': ['spotify_track_speechiness'], - 'tempo': ['spotify_track_tempo'], - 'time_signature': ['spotify_track_time_sig'], - 'valence': ['spotify_track_valence'], - } - import time - no_items = len(items) - self._log.info('Total {} tracks', no_items) + self._log.debug('Total {} tracks', len(items)) for index, item in enumerate(items, start=1): time.sleep(.5) self._log.info('Processing {}/{} tracks - {} ', - index, no_items, item) + index, len(items), item) + # If we're not forcing re-downloading for all tracks, check + # whether the popularity data is already present + if not force: + if 'spotify_track_popularity' in item: + self._log.debug('Popularity already present for: {}', + item) + continue try: - # If we're not forcing re-downloading for all tracks, check - # whether the popularity data is already present - if not force: - if 'spotify_track_popularity' in item: - self._log.debug('Popularity already present for: {}', - item) - continue - popularity = self.track_popularity(item.spotify_track_id) item['spotify_track_popularity'] = popularity audio_features = \ self.track_audio_features(item.spotify_track_id) for feature in audio_features.keys(): - if feature in spotify_audio_features.keys(): - item[spotify_audio_features[feature][0]] = \ + if feature in self.spotify_audio_features.keys(): + item[self.spotify_audio_features[feature]] = \ audio_features[feature] item.store() if write: From 19e2a11ea0050e4dbfe93d688c472741a816ba31 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 15:30:51 -0400 Subject: [PATCH 259/357] Updated documents and changelog. --- docs/changelog.rst | 4 ++++ docs/plugins/spotify.rst | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 72b1cf1fe..c67d57389 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,10 @@ Changelog goes here! New features: +* :doc:`/plugins/spotify`: The plugin now provides an additional command + `spotifysync` that allows getting track popularity and audio features + information from Spotify. + :bug:`4094` * :doc:`/plugins/spotify`: The plugin now records Spotify-specific IDs in the `spotify_album_id`, `spotify_artist_id`, and `spotify_track_id` fields. :bug:`4348` diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index c8e2bfb83..d29783fc6 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -19,6 +19,7 @@ Why Use This Plugin? * You have playlists or albums you'd like to make available in Spotify from Beets without having to search for each artist/album/track. * You want to check which tracks in your library are available on Spotify. * You want to autotag music with metadata from the Spotify API. +* You want to obtain track popularity and audio features (e.g., danceability) Basic Usage ----------- @@ -58,7 +59,7 @@ configuration options are provided. The default options should work as-is, but there are some options you can put in config.yaml under the ``spotify:`` section: -- **mode**: One of the following: +- **mode**: One of the following: - ``list``: Print out the playlist as a list of links. This list can then be pasted in to a new or existing Spotify playlist. @@ -105,3 +106,19 @@ Here's an example:: } ] +Obtaining Track Popularity and Audio Features from Spotify +---------------------------------------------------------- + +Spotify provides track `popularity`_ and audio `features`_ that be used to +create better playlists. + +.. _popularity: https://developer.spotify.com/documentation/web-api/reference/#/operations/get-track +.. _features: https://developer.spotify.com/documentation/web-api/reference/#/operations/get-audio-features + +The ``spotify`` plugin provides an additional command ``spotifysync`` to obtain +these track attributes from Spotify: + +* ``beet spotifysync [-f]``: obtain popularity and audio features information + for every track in the library. By default, ``spotifysync`` will skip tracks + that already have this information populated. Using the ``-f`` or ``-force`` + option will download the data even for tracks that already have it. From b2a90bf089e0c85357ad538ee2e0ba307b0de442 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 15:43:16 -0400 Subject: [PATCH 260/357] Changed spotify labels based on comment --- beetsplug/spotify.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 571def87c..e8d896f08 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -53,18 +53,18 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): } spotify_audio_features = { - 'acousticness': 'spotify_track_acousticness', - 'danceability': 'spotify_track_danceability', - 'energy': 'spotify_track_energy', - 'instrumentalness': 'spotify_track_instrumentalness', - 'key': 'spotify_track_key', - 'liveness': 'spotify_track_liveness', - 'loudness': 'spotify_track_loudness', - 'mode': 'spotify_track_mode', - 'speechiness': 'spotify_track_speechiness', - 'tempo': 'spotify_track_tempo', - 'time_signature': 'spotify_track_time_sig', - 'valence': 'spotify_track_valence', + 'acousticness': 'spotify_acousticness', + 'danceability': 'spotify_danceability', + 'energy': 'spotify_energy', + 'instrumentalness': 'spotify_instrumentalness', + 'key': 'spotify_key', + 'liveness': 'spotify_liveness', + 'loudness': 'spotify_loudness', + 'mode': 'spotify_mode', + 'speechiness': 'spotify_speechiness', + 'tempo': 'spotify_tempo', + 'time_signature': 'spotify_time_signature', + 'valence': 'spotify_valence', } def __init__(self): From 28a4e43b588a01e8cdbaec800863ce0b90f2fc8c Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 15:58:28 -0400 Subject: [PATCH 261/357] Clarified documentation --- docs/plugins/spotify.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index d29783fc6..169846adf 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -121,4 +121,8 @@ these track attributes from Spotify: * ``beet spotifysync [-f]``: obtain popularity and audio features information for every track in the library. By default, ``spotifysync`` will skip tracks that already have this information populated. Using the ``-f`` or ``-force`` - option will download the data even for tracks that already have it. + option will download the data even for tracks that already have it. Please + note that ``spotifysync`` works on tracks that have the Spotify track + identifiers. So run ``spotifysync`` only after importing your music, during + which Spotify identifiers will be added for tracks where Spotify is chosen as + the tag source. From f77a146f17c57c361d958b220c883819105466d0 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 20:52:30 -0400 Subject: [PATCH 262/357] remove force config option --- beetsplug/spotify.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index e8d896f08..90484fc59 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -413,8 +413,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): def func(lib, opts, args): items = lib.items(ui.decargs(args)) - self._fetch_info(items, ui.should_write(), - opts.force_refetch or self.config['force']) + self._fetch_info(items, ui.should_write(), opts.force_refetch) sync_cmd.func = func return [spotify_cmd, sync_cmd] From b9685a4784343288cfe7e39a0851c4163f639ce5 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 17 May 2022 21:07:57 -0400 Subject: [PATCH 263/357] Add more details to the docs --- docs/plugins/spotify.rst | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index 169846adf..6a4119984 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -109,10 +109,11 @@ Here's an example:: Obtaining Track Popularity and Audio Features from Spotify ---------------------------------------------------------- -Spotify provides track `popularity`_ and audio `features`_ that be used to -create better playlists. +Spotify provides information on track `popularity`_ and audio `features`_ that +can be used for music discovery. .. _popularity: https://developer.spotify.com/documentation/web-api/reference/#/operations/get-track + .. _features: https://developer.spotify.com/documentation/web-api/reference/#/operations/get-audio-features The ``spotify`` plugin provides an additional command ``spotifysync`` to obtain @@ -126,3 +127,19 @@ these track attributes from Spotify: identifiers. So run ``spotifysync`` only after importing your music, during which Spotify identifiers will be added for tracks where Spotify is chosen as the tag source. + + In addition to ``popularity``, the command currently sets these audio features + for all tracks with a Spotify track ID: + + * ``acousticness`` + * ``danceability`` + * ``energy`` + * ``instrumentalness`` + * ``key`` + * ``liveness`` + * ``loudness`` + * ``mode`` + * ``speechiness`` + * ``tempo`` + * ``time_signature`` + * ``valence`` From 94af794c3434ebcf372ff06831fffa519722ee73 Mon Sep 17 00:00:00 2001 From: Mark Philip Bautista Date: Wed, 25 May 2022 21:45:47 -0700 Subject: [PATCH 264/357] Update main.rst --- docs/guides/main.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 8dbb113c4..4fab4cc2d 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -94,16 +94,18 @@ Installing on Windows Installing beets on Windows can be tricky. Following these steps might help you get it right: -1. If you don't have it, `install Python`_ (you want Python 3.6). The +1. If you don't have it, `install Python`_ (you want at least Python 3.6). The installer should give you the option to "add Python to PATH." Check this box. If you do that, you can skip the next step. 2. If you haven't done so already, set your ``PATH`` environment variable to - include Python and its scripts. To do so, you have to get the "Properties" - window for "My Computer", then choose the "Advanced" tab, then hit the - "Environment Variables" button, and then look for the ``PATH`` variable in - the table. Add the following to the end of the variable's value: - ``;C:\Python36;C:\Python36\Scripts``. You may need to adjust these paths to + include Python and its scripts. To do so, open the "Settings" application, + then access the "System" screen, then access the "About" tab, and then hit + "Advanced system settings" located on the right side of the screen. This + should open the "System Properties" screen, then select the "Advanced" tab, + then hit the "Environmental Variables..." button, and then look for the PATH + variable in the table. Add the following to the end of the variable's value: + ``;C:\Python36;C:\Python36\Scripts``. You may need to adjust these paths to point to your Python installation. 3. Now install beets by running: ``pip install beets`` @@ -301,4 +303,4 @@ Please let me know what you think of beets via `the discussion board`_ or .. _the mailing list: https://groups.google.com/group/beets-users .. _the discussion board: https://discourse.beets.io -.. _twitter: https://twitter.com/b33ts +.. _twitter: https://twitter.com/b33ts \ No newline at end of file From d68ed1adca6e43ef158a528e24d89090bea7734a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= <16212750+snejus@users.noreply.github.com> Date: Tue, 31 May 2022 21:51:47 +0100 Subject: [PATCH 265/357] Make implicit path queries explicit and simplify their handling --- beets/library.py | 22 ++++------------------ docs/changelog.rst | 3 +++ test/test_query.py | 1 - 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/beets/library.py b/beets/library.py index 69fcd34cf..e15c3e287 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1382,7 +1382,7 @@ def parse_query_parts(parts, model_cls): `Query` and `Sort` they represent. Like `dbcore.parse_sorted_query`, with beets query prefixes and - special path query detection. + ensuring that implicit path queries are made explicit with 'path::' """ # Get query types and their prefix characters. prefixes = { @@ -1394,28 +1394,14 @@ def parse_query_parts(parts, model_cls): # Special-case path-like queries, which are non-field queries # containing path separators (/). - path_parts = [] - non_path_parts = [] - for s in parts: - if PathQuery.is_path_query(s): - path_parts.append(s) - else: - non_path_parts.append(s) + parts = [f"path::{s}" if PathQuery.is_path_query(s) else s for s in parts] case_insensitive = beets.config['sort_case_insensitive'].get(bool) - query, sort = dbcore.parse_sorted_query( - model_cls, non_path_parts, prefixes, case_insensitive + return dbcore.parse_sorted_query( + model_cls, parts, prefixes, case_insensitive ) - # Add path queries to aggregate query. - # Match field / flexattr depending on whether the model has the path field - fast_path_query = 'path' in model_cls._fields - query.subqueries += [PathQuery('path', s, fast_path_query) - for s in path_parts] - - return query, sort - def parse_query_string(s, model_cls): """Given a beets query string, return the `Query` and `Sort` they diff --git a/docs/changelog.rst b/docs/changelog.rst index 72b1cf1fe..d6c74e451 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,9 @@ New features: Bug fixes: +* Fix implicit paths OR queries (e.g. ``beet list /path/ , /other-path/``) + which have previously been returning the entire library. + :bug:`1865` * The Discogs release ID is now populated correctly to the discogs_albumid field again (it was no longer working after Discogs changed their release URL format). diff --git a/test/test_query.py b/test/test_query.py index 0be4b7d7f..42ac59822 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -529,7 +529,6 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, ['path album']) - @unittest.skip('unfixed (#1865)') def test_path_query_in_or_query(self): q = '/a/b , /a/b' results = self.lib.items(q) From ba777dda5011ed3e02aeb7d9e2b5ecf8877b0bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= <16212750+snejus@users.noreply.github.com> Date: Tue, 31 May 2022 22:34:40 +0100 Subject: [PATCH 266/357] Skip implicit paths tests for win32 --- test/test_query.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 42ac59822..4035b2b7b 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -33,6 +33,9 @@ from beets.library import Library, Item from beets import util import platform +# Because the absolute path begins with something like C:, we +# can't disambiguate it from an ordinary query. +WIN32_NO_IMPLICIT_PATHS = 'Implicit paths are not supported on Windows' class TestHelper(helper.TestHelper): @@ -521,6 +524,7 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, ['path album']) + @unittest.skipIf(sys.platform == 'win32', WIN32_NO_IMPLICIT_PATHS) def test_slashed_query_matches_path(self): q = '/a/b' results = self.lib.items(q) @@ -529,6 +533,7 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): results = self.lib.albums(q) self.assert_albums_matched(results, ['path album']) + @unittest.skipIf(sys.platform == 'win32', WIN32_NO_IMPLICIT_PATHS) def test_path_query_in_or_query(self): q = '/a/b , /a/b' results = self.lib.items(q) @@ -648,12 +653,8 @@ class PathQueryTest(_common.LibTestCase, TestHelper, AssertsMixin): self.assertFalse(is_path('foo:bar/')) self.assertFalse(is_path('foo:/bar')) + @unittest.skipIf(sys.platform == 'win32', WIN32_NO_IMPLICIT_PATHS) def test_detect_absolute_path(self): - if platform.system() == 'Windows': - # Because the absolute path begins with something like C:, we - # can't disambiguate it from an ordinary query. - self.skipTest('Windows absolute paths do not work as queries') - # Don't patch `os.path.exists`; we'll actually create a file when # it exists. self.patcher_exists.stop() From 72c530200419d788280f83fd3728073810971929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Tue, 31 May 2022 22:45:05 +0100 Subject: [PATCH 267/357] Fix lints --- test/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_query.py b/test/test_query.py index 4035b2b7b..8a9043fa3 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -31,12 +31,12 @@ from beets.dbcore.query import (NoneQuery, ParsingError, InvalidQueryArgumentValueError) from beets.library import Library, Item from beets import util -import platform # Because the absolute path begins with something like C:, we # can't disambiguate it from an ordinary query. WIN32_NO_IMPLICIT_PATHS = 'Implicit paths are not supported on Windows' + class TestHelper(helper.TestHelper): def assertInResult(self, item, results): # noqa From d65fcfbc8ea407bbd1e37d52b31ff4f9ba99441e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 1 Jun 2022 01:40:16 +0100 Subject: [PATCH 268/357] Use : instead of :: --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index e15c3e287..7be7bc79f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1394,7 +1394,7 @@ def parse_query_parts(parts, model_cls): # Special-case path-like queries, which are non-field queries # containing path separators (/). - parts = [f"path::{s}" if PathQuery.is_path_query(s) else s for s in parts] + parts = [f"path={s}" if PathQuery.is_path_query(s) else s for s in parts] case_insensitive = beets.config['sort_case_insensitive'].get(bool) From 5f4b46e3888d4b7e3c48921149a98563012699ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0ar=C5=ABnas=20Nejus?= Date: Wed, 1 Jun 2022 01:47:55 +0100 Subject: [PATCH 269/357] Actually use : --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 7be7bc79f..c8fa2b5fc 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1394,7 +1394,7 @@ def parse_query_parts(parts, model_cls): # Special-case path-like queries, which are non-field queries # containing path separators (/). - parts = [f"path={s}" if PathQuery.is_path_query(s) else s for s in parts] + parts = [f"path:{s}" if PathQuery.is_path_query(s) else s for s in parts] case_insensitive = beets.config['sort_case_insensitive'].get(bool) From d4864b7b0ce6a1316ab585e343d5dfc133fbd6be Mon Sep 17 00:00:00 2001 From: e-berman Date: Wed, 8 Jun 2022 22:11:51 -0700 Subject: [PATCH 270/357] add cl formatting to beatport docs --- docs/plugins/beatport.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index 6117c4a1f..f44fdeb34 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -32,7 +32,7 @@ from MusicBrainz and other sources. If you have a Beatport ID or a URL for a release or track you want to tag, you can just enter one of the two at the "enter Id" prompt in the importer. You can -also search for an id like so: +also search for an id like so:: beet import path/to/music/library --search-id id From 9db8d8f6a1cab7143d959ce98be7b7d4dcba35ff Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 12 Jun 2022 17:41:13 +0200 Subject: [PATCH 271/357] docs: fix broken links these failed in the weekly CI test. discogs probably remains broken, since it received a 403. --- docs/guides/main.rst | 4 ++-- docs/plugins/discogs.rst | 2 +- docs/plugins/fetchart.rst | 4 ++-- docs/plugins/spotify.rst | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 4fab4cc2d..2b573ac32 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -46,7 +46,7 @@ Beets works on Python 3.6 or later. * On **NixOS**, there's a `package `_ you can install with ``nix-env -i beets``. -.. _DNF package: https://apps.fedoraproject.org/packages/beets +.. _DNF package: https://packages.fedoraproject.org/pkgs/beets/ .. _SlackBuild: https://slackbuilds.org/repository/14.2/multimedia/beets/ .. _FreeBSD: http://portsmon.freebsd.org/portoverview.py?category=audio&portname=beets .. _AUR: https://aur.archlinux.org/packages/beets-git/ @@ -303,4 +303,4 @@ Please let me know what you think of beets via `the discussion board`_ or .. _the mailing list: https://groups.google.com/group/beets-users .. _the discussion board: https://discourse.beets.io -.. _twitter: https://twitter.com/b33ts \ No newline at end of file +.. _twitter: https://twitter.com/b33ts diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index 1ed2dfc93..53c6c2ac0 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -51,7 +51,7 @@ This plugin can be configured like other metadata source plugins as described in There is one additional option in the ``discogs:`` section, ``index_tracks``. Index tracks (see the `Discogs guidelines -`_), +`_), along with headers, mark divisions between distinct works on the same release or within works. When ``index_tracks`` is enabled:: diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 997cf2497..de3aec360 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -240,10 +240,10 @@ in your configuration. .. _registering a personal fanart.tv API key: https://fanart.tv/get-an-api-key/ -More detailed information can be found `on their blog`_. Specifically, the +More detailed information can be found `on their Wiki`_. Specifically, the personal key will give you earlier access to new art. -.. _on their blog: https://fanart.tv/2015/01/personal-api-keys/ +.. _on their blog: https://wiki.fanart.tv/General/personal%20api/ Last.fm ''''''' diff --git a/docs/plugins/spotify.rst b/docs/plugins/spotify.rst index c8e2bfb83..7340e2150 100644 --- a/docs/plugins/spotify.rst +++ b/docs/plugins/spotify.rst @@ -8,9 +8,9 @@ Also, the plugin can use the Spotify `Album`_ and `Track`_ APIs to provide metadata matches for the importer. .. _Spotify: https://www.spotify.com/ -.. _Spotify Search API: https://developer.spotify.com/documentation/web-api/reference/#category-search -.. _Album: https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-an-album -.. _Track: https://developer.spotify.com/documentation/web-api/reference/#endpoint-get-track +.. _Spotify Search API: https://developer.spotify.com/documentation/web-api/reference/#/operations/search +.. _Album: https://developer.spotify.com/documentation/web-api/reference/#/operations/get-an-album +.. _Track: https://developer.spotify.com/documentation/web-api/reference/#/operations/get-track Why Use This Plugin? -------------------- From d72498f4291d8b070bfc664eb70f8f5c2196b76b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 12 Jun 2022 17:51:00 +0200 Subject: [PATCH 272/357] fixup! docs: fix broken links --- docs/plugins/fetchart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index de3aec360..2b884c5ad 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -243,7 +243,7 @@ in your configuration. More detailed information can be found `on their Wiki`_. Specifically, the personal key will give you earlier access to new art. -.. _on their blog: https://wiki.fanart.tv/General/personal%20api/ +.. _on their Wiki: https://wiki.fanart.tv/General/personal%20api/ Last.fm ''''''' From c657fb66421f8d97c862f9e6b8d626d19739cf9c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 12 Jun 2022 17:56:53 +0200 Subject: [PATCH 273/357] mediafile: Improve deprecation warning This is for mediafile what cc8c3529fbf528da8eadadf2ff61389db9adf767 was for confit, cf. https://github.com/beetbox/beets/pull/4263 --- beets/mediafile.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/beets/mediafile.py b/beets/mediafile.py index 82bcc973d..46288a71d 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -16,11 +16,18 @@ import mediafile import warnings -warnings.warn("beets.mediafile is deprecated; use mediafile instead") +warnings.warn( + "beets.mediafile is deprecated; use mediafile instead", + # Show the location of the `import mediafile` statement as the warning's + # source, rather than this file, such that the offending module can be + # identified easily. + stacklevel=2, +) # Import everything from the mediafile module into this module. for key, value in mediafile.__dict__.items(): if key not in ['__name__']: globals()[key] = value +# Cleanup namespace. del key, value, warnings, mediafile From 3cd6fd64ca703b692d06d51ff106881e795a99ab Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 12 Jun 2022 13:30:23 -0400 Subject: [PATCH 274/357] Added comment about sleep --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 90484fc59..b39343284 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -577,6 +577,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self._log.debug('Total {} tracks', len(items)) for index, item in enumerate(items, start=1): + # Added sleep to avoid API rate limit time.sleep(.5) self._log.info('Processing {}/{} tracks - {} ', index, len(items), item) From 3d917edd67a0a13b05e0f8590a5648a0bfc5b8c9 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 12 Jun 2022 13:31:40 -0400 Subject: [PATCH 275/357] Update spotify.py --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index b39343284..66052f5fe 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -578,6 +578,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): for index, item in enumerate(items, start=1): # Added sleep to avoid API rate limit + # https://developer.spotify.com/documentation/web-api/guides/rate-limits/ time.sleep(.5) self._log.info('Processing {}/{} tracks - {} ', index, len(items), item) From 9a392f3157978c38757bcdfa77268471510df49a Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 12 Jun 2022 13:58:08 -0400 Subject: [PATCH 276/357] Address try/except comment --- beetsplug/spotify.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 66052f5fe..d7062f7d4 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -590,20 +590,22 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): item) continue try: - popularity = self.track_popularity(item.spotify_track_id) - item['spotify_track_popularity'] = popularity - audio_features = \ - self.track_audio_features(item.spotify_track_id) - for feature in audio_features.keys(): - if feature in self.spotify_audio_features.keys(): - item[self.spotify_audio_features[feature]] = \ - audio_features[feature] - item.store() - if write: - item.try_write() + spotify_track_id = item.spotify_track_id except AttributeError: self._log.debug('No track_id present for: {}', item) - pass + continue + + popularity = self.track_popularity(spotify_track_id) + item['spotify_track_popularity'] = popularity + audio_features = \ + self.track_audio_features(spotify_track_id) + for feature in audio_features.keys(): + if feature in self.spotify_audio_features.keys(): + item[self.spotify_audio_features[feature]] = \ + audio_features[feature] + item.store() + if write: + item.try_write() def track_popularity(self, track_id=None): """Fetch a track popularity by its Spotify ID.""" From 71be6d51387912547921098309ccdbc1c6965a83 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 12 Jun 2022 19:21:38 -0400 Subject: [PATCH 277/357] Add 429 API error handling --- beetsplug/spotify.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d7062f7d4..021fa8fc9 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -164,6 +164,11 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): ) self._authenticate() return self._handle_response(request_type, url, params=params) + elif response.status_code == 429: + seconds = response.headers['Retry-After'] + time.sleep(int(seconds)) + self._log.info('Too many API requests. Retrying after {} seconds.', seconds) + return self._handle_response(request_type, url, params=params) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From c4dec04dcf4d886cc10ebbbf50444285e3eeff0f Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 12 Jun 2022 19:27:15 -0400 Subject: [PATCH 278/357] Fix lint --- beetsplug/spotify.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 021fa8fc9..ea5a73fbd 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -167,7 +167,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): elif response.status_code == 429: seconds = response.headers['Retry-After'] time.sleep(int(seconds)) - self._log.info('Too many API requests. Retrying after {} seconds.', seconds) + self._log.info('Too many API requests. Retrying after {} seconds.', + seconds) return self._handle_response(request_type, url, params=params) else: raise ui.UserError( From 4bb8862b6f2f528cc9621f3ea22790026f7a9520 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 12 Jun 2022 19:28:47 -0400 Subject: [PATCH 279/357] lint --- beetsplug/spotify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index ea5a73fbd..771d2e436 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -167,8 +167,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): elif response.status_code == 429: seconds = response.headers['Retry-After'] time.sleep(int(seconds)) - self._log.info('Too many API requests. Retrying after {} seconds.', - seconds) + self._log.info('Too many API requests. Retrying after {} \ + seconds.', seconds) return self._handle_response(request_type, url, params=params) else: raise ui.UserError( From af7f29491e2c63aae359f8abcee9038f78c03c44 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Mon, 13 Jun 2022 18:26:30 +1000 Subject: [PATCH 280/357] Update pyhon versions --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 07ac5f51e..b3472e412 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest, windows-latest] - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11.0'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] env: PY_COLORS: 1 @@ -45,7 +45,7 @@ jobs: sudo apt install ffmpeg # For replaygain - name: Test older Python versions with tox - if: matrix.python-version != '3.10' && matrix.python-version != '3.11.0' + if: matrix.python-version != '3.10' && matrix.python-version != '3.11-dev' run: | tox -e py-test @@ -55,7 +55,7 @@ jobs: tox -vv -e py-cov - name: Test nightly Python version with tox - if: matrix.python-version == '3.11.0' + if: matrix.python-version == '3.11-dev' # continue-on-error is not ideal since it doesn't give a visible # warning, but there doesn't seem to be anything better: # https://github.com/actions/toolkit/issues/399 From a2e6680e2f21e84ccc81c6febb912b9838c022fe Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 13 Jun 2022 09:26:15 -0400 Subject: [PATCH 281/357] Address comments --- beetsplug/spotify.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 771d2e436..4aac5b24c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -32,6 +32,7 @@ from beets import ui from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.plugins import MetadataSourcePlugin, BeetsPlugin +DEFAULT_WAITING_TIME = 5 class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Spotify' @@ -165,10 +166,11 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self._authenticate() return self._handle_response(request_type, url, params=params) elif response.status_code == 429: - seconds = response.headers['Retry-After'] - time.sleep(int(seconds)) - self._log.info('Too many API requests. Retrying after {} \ + seconds = response.headers.get('Retry-After', + DEFAULT_WAITING_TIME) + self._log.debug('Too many API requests. Retrying after {} \ seconds.', seconds) + time.sleep(int(seconds)) return self._handle_response(request_type, url, params=params) else: raise ui.UserError( From c9f9ed3b646fcb66c0ccaafb94a338aa323c03bd Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 13 Jun 2022 09:31:19 -0400 Subject: [PATCH 282/357] lint --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 4aac5b24c..ac680d99e 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -34,6 +34,7 @@ from beets.plugins import MetadataSourcePlugin, BeetsPlugin DEFAULT_WAITING_TIME = 5 + class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Spotify' From 8ba2c015abbdd7935d8edfb97b873d44f0add256 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 13 Jun 2022 09:57:07 -0400 Subject: [PATCH 283/357] Sorted imports using iSort --- beetsplug/spotify.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index ac680d99e..21e97ef2c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -17,20 +17,19 @@ Spotify playlist construction. """ -import re -import json import base64 +import collections +import json +import re import time import webbrowser -import collections -import unidecode -import requests import confuse - +import requests +import unidecode from beets import ui from beets.autotag.hooks import AlbumInfo, TrackInfo -from beets.plugins import MetadataSourcePlugin, BeetsPlugin +from beets.plugins import BeetsPlugin, MetadataSourcePlugin DEFAULT_WAITING_TIME = 5 From 3c68f717e9a2ec820eca9f41805a61555c7925de Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 13 Jun 2022 10:16:10 -0400 Subject: [PATCH 284/357] Added an extra second (based on other libraries) --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 21e97ef2c..d7290d963 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -170,7 +170,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): DEFAULT_WAITING_TIME) self._log.debug('Too many API requests. Retrying after {} \ seconds.', seconds) - time.sleep(int(seconds)) + time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) else: raise ui.UserError( From 9f26190fa3289663ca2dc32a92f24049379464f4 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 13 Jun 2022 10:25:48 -0400 Subject: [PATCH 285/357] Added changelog --- docs/changelog.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c95287443..dc13ee781 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,7 +32,9 @@ New features: Bug fixes: -* Fix implicit paths OR queries (e.g. ``beet list /path/ , /other-path/``) +* Added Spotify 429 (too many requests) API error handling + :bug:`4370` +* Fix implicit paths OR queries (e.g. ``beet list /path/ , /other-path/``) which have previously been returning the entire library. :bug:`1865` * The Discogs release ID is now populated correctly to the discogs_albumid From 4d826168a4eec04fb49bd410cb20ef631cbd3438 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 16 Jun 2022 09:00:17 -0400 Subject: [PATCH 286/357] Remove sleep --- beetsplug/spotify.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d7290d963..4980d9c1b 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -585,9 +585,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self._log.debug('Total {} tracks', len(items)) for index, item in enumerate(items, start=1): - # Added sleep to avoid API rate limit - # https://developer.spotify.com/documentation/web-api/guides/rate-limits/ - time.sleep(.5) self._log.info('Processing {}/{} tracks - {} ', index, len(items), item) # If we're not forcing re-downloading for all tracks, check From 1cd78ad3c551aa09e836c51a1da092770c441593 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 16 Jun 2022 09:28:07 -0400 Subject: [PATCH 287/357] Change log to info --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 4980d9c1b..c4174d02b 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -168,7 +168,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): elif response.status_code == 429: seconds = response.headers.get('Retry-After', DEFAULT_WAITING_TIME) - self._log.debug('Too many API requests. Retrying after {} \ + self._log.info('Too many API requests. Retrying after {} \ seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) From b1b0926eed50e46d88809d07b806672ea0628514 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sat, 18 Jun 2022 10:45:02 -0400 Subject: [PATCH 288/357] UPdate changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dc13ee781..dbee12764 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,7 +32,7 @@ New features: Bug fixes: -* Added Spotify 429 (too many requests) API error handling +* We now respect the Spotify API's rate limiting, which avoids crashing when the API reports code 429 (too many requests). :bug:`4370` * Fix implicit paths OR queries (e.g. ``beet list /path/ , /other-path/``) which have previously been returning the entire library. From abe4f203b1f989456b1bf646f83ff2be08938129 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sat, 18 Jun 2022 11:23:22 -0400 Subject: [PATCH 289/357] Changed log to debug --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index c4174d02b..4980d9c1b 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -168,7 +168,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): elif response.status_code == 429: seconds = response.headers.get('Retry-After', DEFAULT_WAITING_TIME) - self._log.info('Too many API requests. Retrying after {} \ + self._log.debug('Too many API requests. Retrying after {} \ seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) From eed8a5bd83435545609c3ee515dc8dd7d7b61184 Mon Sep 17 00:00:00 2001 From: Dickson Date: Sun, 19 Jun 2022 18:44:54 +0800 Subject: [PATCH 290/357] docs: add link to Beets-audible plugin --- docs/plugins/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 3d8b97606..0e9a95136 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -279,6 +279,9 @@ Here are a few of the plugins written by the beets community: * `beets-autofix`_ automates repetitive tasks to keep your library in order. +* `beets-audible`_ adds Audible as a tagger data source and provides + other features for managing audiobook collections. + * `beets-barcode`_ lets you scan or enter barcodes for physical media to search for their metadata. @@ -370,3 +373,4 @@ Here are a few of the plugins written by the beets community: .. _beets-bpmanalyser: https://github.com/adamjakab/BeetsPluginBpmAnalyser .. _beets-originquery: https://github.com/x1ppy/beets-originquery .. _drop2beets: https://github.com/martinkirch/drop2beets +.. _beets-audible: https://github.com/Neurrone/beets-audible \ No newline at end of file From fc8d3fceeb5e3c0b577836c46212e7a06af0d68a Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 20 Jun 2022 16:52:05 -0400 Subject: [PATCH 291/357] Define item_types for spotify attributes --- beetsplug/spotify.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 4980d9c1b..6c4529450 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -37,6 +37,22 @@ DEFAULT_WAITING_TIME = 5 class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Spotify' + item_types = { + 'spotify_track_popularity': types.INTEGER, + 'spotify_acousticness': types.FLOAT, + 'spotify_danceability': types.FLOAT, + 'spotify_energy': types.FLOAT, + 'spotify_instrumentalness': types.FLOAT, + 'spotify_key': types.FLOAT, + 'spotify_liveness': types.FLOAT, + 'spotify_loudness': types.FLOAT, + 'spotify_mode': types.INTEGER, + 'spotify_speechiness': types.FLOAT, + 'spotify_tempo': types.FLOAT, + 'spotify_time_signature': types.INTEGER, + 'spotify_valence': types.FLOAT, + } + # Base URLs for the Spotify API # Documentation: https://developer.spotify.com/web-api oauth_token_url = 'https://accounts.spotify.com/api/token' From 8102bd2e35bb65f75aa246538fa18b04d35feae5 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 20 Jun 2022 16:59:03 -0400 Subject: [PATCH 292/357] Add import --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 6c4529450..42e20c828 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -29,6 +29,7 @@ import requests import unidecode from beets import ui from beets.autotag.hooks import AlbumInfo, TrackInfo +from beets.dbcore import types from beets.plugins import BeetsPlugin, MetadataSourcePlugin DEFAULT_WAITING_TIME = 5 From 276c55105944844e62c81d3395519e6e7f8d8ee4 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 26 Jun 2022 11:46:42 -0400 Subject: [PATCH 293/357] Update spotify.py --- beetsplug/spotify.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 42e20c828..e81c1b8aa 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -189,6 +189,9 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) + elif 'analysis not found' in response.text: + self._log.debug('No analysis found') + return None else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( @@ -621,6 +624,9 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): item['spotify_track_popularity'] = popularity audio_features = \ self.track_audio_features(spotify_track_id) + if audio_features is None: + self._log.debug('No audio features found for : {}', item) + continue for feature in audio_features.keys(): if feature in self.spotify_audio_features.keys(): item[self.spotify_audio_features[feature]] = \ From 2490737f617195ac2c8a0dca940b653e2c129254 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 26 Jun 2022 11:54:00 -0400 Subject: [PATCH 294/357] Update changelog.rst --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index dbee12764..8cb27013d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,6 +32,8 @@ New features: Bug fixes: +* Fix Spotify API error when tracks do not have audio-features. + :bug:`4385` * We now respect the Spotify API's rate limiting, which avoids crashing when the API reports code 429 (too many requests). :bug:`4370` * Fix implicit paths OR queries (e.g. ``beet list /path/ , /other-path/``) From 086de854d1d03edc1793f1ce9eb5ff129a80c57d Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 26 Jun 2022 11:56:13 -0400 Subject: [PATCH 295/357] Update spotify.py --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index e81c1b8aa..0de08ff3d 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -190,7 +190,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) elif 'analysis not found' in response.text: - self._log.debug('No analysis found') + self._log.debug('No audio analysis found') return None else: raise ui.UserError( From 74e549838cfda733d6f3be12db3829afbe1255ea Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Sun, 26 Jun 2022 18:38:13 +0200 Subject: [PATCH 296/357] feat(import): Add support for reading skipped paths from logfile Fixes #4379. --- beets/ui/commands.py | 45 +++++++++++++++++++++++++++++++++++++++--- docs/changelog.rst | 3 +++ docs/guides/tagger.rst | 5 ++++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 3a3374013..b0921728d 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -79,6 +79,26 @@ def _do_query(lib, query, album, also_items=True): return items, albums +def _paths_from_logfile(path): + """Parse the logfile and yield skipped paths to pass to the `import` + command. + """ + with open(path, mode="r", encoding="utf-8") as fp: + for line in fp: + verb, sep, paths = line.rstrip("\n").partition(" ") + if not sep: + raise ValueError("malformed line in logfile") + + # Ignore informational lines that don't need to be re-imported. + if verb in {"import", "duplicate-keep", "duplicate-replace"}: + continue + + if verb not in {"asis", "skip", "duplicate-skip"}: + raise ValueError(f"unknown verb {verb} found in logfile") + + yield os.path.commonpath(paths.split("; ")) + + # fields: Shows a list of available fields for queries and format strings. def _print_keys(query): @@ -914,11 +934,30 @@ def import_files(lib, paths, query): query. """ # Check the user-specified directories. + paths_to_import = [] for path in paths: - if not os.path.exists(syspath(normpath(path))): + normalized_path = syspath(normpath(path)) + if not os.path.exists(normalized_path): raise ui.UserError('no such file or directory: {}'.format( displayable_path(path))) + # Read additional paths from logfile. + if os.path.isfile(normalized_path): + try: + paths_to_import.extend(list( + _paths_from_logfile(normalized_path))) + except ValueError: + # We could raise an error here, but it's possible that the user + # tried to import a media file instead of a logfile. In that + # case, logging a warning would be confusing, so we skip this + # here. + pass + else: + # Don't re-add the log file to the list of paths to import. + continue + + paths_to_import.append(paths) + # Check parameter consistency. if config['import']['quiet'] and config['import']['timid']: raise ui.UserError("can't be both quiet and timid") @@ -939,11 +978,11 @@ def import_files(lib, paths, query): config['import']['quiet']: config['import']['resume'] = False - session = TerminalImportSession(lib, loghandler, paths, query) + session = TerminalImportSession(lib, loghandler, paths_to_import, query) session.run() # Emit event. - plugins.send('import', lib=lib, paths=paths) + plugins.send('import', lib=lib, paths=paths_to_import) def import_func(lib, opts, args): diff --git a/docs/changelog.rst b/docs/changelog.rst index dbee12764..a5086b6b9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,9 @@ New features: :bug:`1840` :bug:`4302` * Added a ``-P`` (or ``--disable-plugins``) flag to specify one/multiple plugin(s) to be disabled at startup. +* :ref:`import-options`: Add support for re-running the importer on paths in + log files that were created with the ``-l`` (or ``--logfile``) argument. + :bug:`4379` :bug:`4387` Bug fixes: diff --git a/docs/guides/tagger.rst b/docs/guides/tagger.rst index d890f5c08..5184f433f 100644 --- a/docs/guides/tagger.rst +++ b/docs/guides/tagger.rst @@ -80,6 +80,8 @@ all of these limitations. Now that that's out of the way, let's tag some music. +.. _import-options: + Options ------- @@ -101,7 +103,8 @@ command-line options you should know: * ``beet import -l LOGFILE``: write a message to ``LOGFILE`` every time you skip an album or choose to take its tags "as-is" (see below) or the album is skipped as a duplicate; this lets you come back later and reexamine albums - that weren't tagged successfully + that weren't tagged successfully. Run ``beet import LOGFILE`` rerun the + importer on such paths from the logfile. * ``beet import -q``: quiet mode. Never prompt for input and, instead, conservatively skip any albums that need your opinion. The ``-ql`` combination From 6d74ffb833c4edca1c0573e9728f8c31395adbc8 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Mon, 27 Jun 2022 19:00:40 +0200 Subject: [PATCH 297/357] feat(import): Move logfile import logic to `--from-logfile` argument As requested here: https://github.com/beetbox/beets/pull/4387#pullrequestreview-1020040380 --- beets/ui/commands.py | 79 +++++++++++++++++++++++++++--------------- docs/guides/tagger.rst | 4 +-- docs/reference/cli.rst | 4 ++- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index b0921728d..1bcb1de1d 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -933,31 +933,6 @@ def import_files(lib, paths, query): """Import the files in the given list of paths or matching the query. """ - # Check the user-specified directories. - paths_to_import = [] - for path in paths: - normalized_path = syspath(normpath(path)) - if not os.path.exists(normalized_path): - raise ui.UserError('no such file or directory: {}'.format( - displayable_path(path))) - - # Read additional paths from logfile. - if os.path.isfile(normalized_path): - try: - paths_to_import.extend(list( - _paths_from_logfile(normalized_path))) - except ValueError: - # We could raise an error here, but it's possible that the user - # tried to import a media file instead of a logfile. In that - # case, logging a warning would be confusing, so we skip this - # here. - pass - else: - # Don't re-add the log file to the list of paths to import. - continue - - paths_to_import.append(paths) - # Check parameter consistency. if config['import']['quiet'] and config['import']['timid']: raise ui.UserError("can't be both quiet and timid") @@ -978,11 +953,11 @@ def import_files(lib, paths, query): config['import']['quiet']: config['import']['resume'] = False - session = TerminalImportSession(lib, loghandler, paths_to_import, query) + session = TerminalImportSession(lib, loghandler, paths, query) session.run() # Emit event. - plugins.send('import', lib=lib, paths=paths_to_import) + plugins.send('import', lib=lib, paths=paths) def import_func(lib, opts, args): @@ -999,7 +974,25 @@ def import_func(lib, opts, args): else: query = None paths = args - if not paths: + + # The paths from the logfiles go into a separate list to allow handling + # errors differently from user-specified paths. + paths_from_logfiles = [] + for logfile in opts.from_logfiles or []: + try: + paths_from_logfiles.extend( + list(_paths_from_logfile(syspath(normpath(logfile)))) + ) + except ValueError as err: + raise ui.UserError('malformed logfile {}'.format( + util.displayable_path(logfile), + )) from err + except IOError as err: + raise ui.UserError('unreadable logfile {}'.format( + util.displayable_path(logfile), + )) from err + + if not paths and not paths_from_logfiles: raise ui.UserError('no path specified') # On Python 2, we used to get filenames as raw bytes, which is @@ -1008,6 +1001,31 @@ def import_func(lib, opts, args): # filename. paths = [p.encode(util.arg_encoding(), 'surrogateescape') for p in paths] + paths_from_logfiles = [p.encode(util.arg_encoding(), 'surrogateescape') + for p in paths_from_logfiles] + + # Check the user-specified directories. + for path in paths: + if not os.path.exists(syspath(normpath(path))): + raise ui.UserError('no such file or directory: {}'.format( + displayable_path(path))) + + # Check the directories from the logfiles, but don't throw an error in + # case those paths don't exist. Maybe some of those paths have already + # been imported and moved separately, so logging a warning should + # suffice. + for path in paths_from_logfiles: + if not os.path.exists(syspath(normpath(path))): + log.warning('No such file or directory: {}'.format( + displayable_path(path))) + continue + + paths.append(path) + + # If all paths were read from a logfile, and none of them exist, throw + # an error + if not paths: + raise ui.UserError('none of the paths are importable') import_files(lib, paths, query) @@ -1100,6 +1118,11 @@ import_cmd.parser.add_option( metavar='ID', help='restrict matching to a specific metadata backend ID' ) +import_cmd.parser.add_option( + '--from-logfile', dest='from_logfiles', action='append', + metavar='PATH', + help='read skipped paths from an existing logfile' +) import_cmd.parser.add_option( '--set', dest='set_fields', action='callback', callback=_store_dict, diff --git a/docs/guides/tagger.rst b/docs/guides/tagger.rst index 5184f433f..d47ee3c4a 100644 --- a/docs/guides/tagger.rst +++ b/docs/guides/tagger.rst @@ -103,8 +103,8 @@ command-line options you should know: * ``beet import -l LOGFILE``: write a message to ``LOGFILE`` every time you skip an album or choose to take its tags "as-is" (see below) or the album is skipped as a duplicate; this lets you come back later and reexamine albums - that weren't tagged successfully. Run ``beet import LOGFILE`` rerun the - importer on such paths from the logfile. + that weren't tagged successfully. Run ``beet import --from-logfile=LOGFILE`` + rerun the importer on such paths from the logfile. * ``beet import -q``: quiet mode. Never prompt for input and, instead, conservatively skip any albums that need your opinion. The ``-ql`` combination diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index dad407489..3726fb373 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -86,7 +86,9 @@ Optional command flags: that weren't tagged successfully---either because they're not in the MusicBrainz database or because something's wrong with the files. Use the ``-l`` option to specify a filename to log every time you skip an album - or import it "as-is" or an album gets skipped as a duplicate. + or import it "as-is" or an album gets skipped as a duplicate. You can later + review the file manually or import skipped paths from the logfile + automatically by using the ``--from-logfile LOGFILE`` argument. * Relatedly, the ``-q`` (quiet) option can help with large imports by autotagging without ever bothering to ask for user input. Whenever the From 3e37d0163e2575000265be2349d7da5702cfce4a Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Mon, 27 Jun 2022 19:48:22 +0200 Subject: [PATCH 298/357] test(import): Add test for `_paths_from_logfile` method --- test/test_ui.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/test_ui.py b/test/test_ui.py index dd24fce1a..0c7478d45 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -729,6 +729,40 @@ class ImportTest(_common.TestCase): self.assertRaises(ui.UserError, commands.import_files, None, [], None) + def test_parse_paths_from_logfile(self): + if os.path.__name__ == 'ntpath': + logfile_content = ( + "import started Wed Jun 15 23:08:26 2022\n" + "asis C:\\music\\Beatles, The\\The Beatles; C:\\music\\Beatles, The\\The Beatles\\CD 01; C:\\music\\Beatles, The\\The Beatles\\CD 02\n" # noqa: E501 + "duplicate-replace C:\\music\\Bill Evans\\Trio '65\n" + "skip C:\\music\\Michael Jackson\\Bad\n" + "skip C:\\music\\Soulwax\\Any Minute Now\n" + ) + expected_paths = [ + "C:\\music\\Beatles, The\\The Beatles", + "C:\\music\\Michael Jackson\\Bad", + "C:\\music\\Soulwax\\Any Minute Now", + ] + else: + logfile_content = ( + "import started Wed Jun 15 23:08:26 2022\n" + "asis /music/Beatles, The/The Beatles; /music/Beatles, The/The Beatles/CD 01; /music/Beatles, The/The Beatles/CD 02\n" # noqa: E501 + "duplicate-replace /music/Bill Evans/Trio '65\n" + "skip /music/Michael Jackson/Bad\n" + "skip /music/Soulwax/Any Minute Now\n" + ) + expected_paths = [ + "/music/Beatles, The/The Beatles", + "/music/Michael Jackson/Bad", + "/music/Soulwax/Any Minute Now", + ] + + logfile = os.path.join(self.temp_dir, b"logfile.log") + with open(logfile, mode="w") as fp: + fp.write(logfile_content) + actual_paths = list(commands._paths_from_logfile(logfile)) + self.assertEqual(actual_paths, expected_paths) + @_common.slow_test() class ConfigTest(unittest.TestCase, TestHelper, _common.Assertions): From cfc34826a20203240e1bc8f2ba0cee328dfe18df Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Tue, 28 Jun 2022 22:10:11 +0200 Subject: [PATCH 299/357] feat(import): Improve error reporting when reading paths from logfile --- beets/ui/commands.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 1bcb1de1d..cbdd8e3bf 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -84,21 +84,38 @@ def _paths_from_logfile(path): command. """ with open(path, mode="r", encoding="utf-8") as fp: - for line in fp: + for i, line in enumerate(fp, start=1): verb, sep, paths = line.rstrip("\n").partition(" ") if not sep: - raise ValueError("malformed line in logfile") + raise ValueError(f"line {i} is invalid") # Ignore informational lines that don't need to be re-imported. if verb in {"import", "duplicate-keep", "duplicate-replace"}: continue if verb not in {"asis", "skip", "duplicate-skip"}: - raise ValueError(f"unknown verb {verb} found in logfile") + raise ValueError(f"line {i} contains unknown verb {verb}") yield os.path.commonpath(paths.split("; ")) +def _parse_logfiles(logfiles): + """Parse all `logfiles` and yield paths from it.""" + for logfile in logfiles: + try: + yield from _paths_from_logfile(syspath(normpath(logfile))) + except ValueError as err: + raise ui.UserError('malformed logfile {}: {}'.format( + util.displayable_path(logfile), + str(err) + )) from err + except IOError as err: + raise ui.UserError('unreadable logfile {}: {}'.format( + util.displayable_path(logfile), + str(err) + )) from err + + # fields: Shows a list of available fields for queries and format strings. def _print_keys(query): @@ -977,20 +994,7 @@ def import_func(lib, opts, args): # The paths from the logfiles go into a separate list to allow handling # errors differently from user-specified paths. - paths_from_logfiles = [] - for logfile in opts.from_logfiles or []: - try: - paths_from_logfiles.extend( - list(_paths_from_logfile(syspath(normpath(logfile)))) - ) - except ValueError as err: - raise ui.UserError('malformed logfile {}'.format( - util.displayable_path(logfile), - )) from err - except IOError as err: - raise ui.UserError('unreadable logfile {}'.format( - util.displayable_path(logfile), - )) from err + paths_from_logfiles = list(_parse_logfiles(opts.from_logfiles or [])) if not paths and not paths_from_logfiles: raise ui.UserError('no path specified') From 91c2cd2ee50cce85b9f4d42956a96786e79ca346 Mon Sep 17 00:00:00 2001 From: toaru_yousei Date: Wed, 21 Jul 2021 21:02:39 +0900 Subject: [PATCH 300/357] Fix importadded plugin with reflink --- beetsplug/importadded.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/importadded.py b/beetsplug/importadded.py index e6665e0ff..409056c80 100644 --- a/beetsplug/importadded.py +++ b/beetsplug/importadded.py @@ -34,6 +34,7 @@ class ImportAddedPlugin(BeetsPlugin): register('item_copied', self.record_import_mtime) register('item_linked', self.record_import_mtime) register('item_hardlinked', self.record_import_mtime) + register('item_reflinked', self.record_import_mtime) register('album_imported', self.update_album_times) register('item_imported', self.update_item_times) register('after_write', self.update_after_write_time) @@ -49,7 +50,8 @@ class ImportAddedPlugin(BeetsPlugin): def record_if_inplace(self, task, session): if not (session.config['copy'] or session.config['move'] or - session.config['link'] or session.config['hardlink']): + session.config['link'] or session.config['hardlink'] or + session.config['reflink']): self._log.debug("In place import detected, recording mtimes from " "source paths") items = [task.item] \ From ea571a56f214c6fc5767946ec74f2611bb5183f5 Mon Sep 17 00:00:00 2001 From: toaru_yousei Date: Thu, 30 Jun 2022 00:50:55 +0900 Subject: [PATCH 301/357] ImportAdded reflink fix: Update changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index dbee12764..31ca6aa3f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -82,6 +82,9 @@ Bug fixes: :bug:`4272` * :doc:`plugins/lyrics`: Fixed issue with Genius header being included in lyrics, added test case of up-to-date Genius html +* :doc:`plugins/importadded`: Fix a bug with recently added reflink import option + that casues a crash when ImportAdded plugin enabled. + :bug:`4389` For packagers: From 9bde98c4405ef4486d1115b46f93e6709677645a Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Wed, 29 Jun 2022 19:47:21 -0400 Subject: [PATCH 302/357] Update beetsplug/spotify.py Co-authored-by: Adrian Sampson --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 0de08ff3d..979cc8975 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -625,7 +625,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): audio_features = \ self.track_audio_features(spotify_track_id) if audio_features is None: - self._log.debug('No audio features found for : {}', item) + self._log.debug('No audio features found for: {}', item) continue for feature in audio_features.keys(): if feature in self.spotify_audio_features.keys(): From 54af411b62054b9e045d597bf54e354a097f49fc Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Wed, 29 Jun 2022 19:54:12 -0400 Subject: [PATCH 303/357] Address comments --- beetsplug/spotify.py | 14 ++++++++------ docs/changelog.rst | 2 -- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 979cc8975..666a19c08 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -189,9 +189,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) - elif 'analysis not found' in response.text: - self._log.debug('No audio analysis found') - return None else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( @@ -645,7 +642,12 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): def track_audio_features(self, track_id=None): """Fetch track audio features by its Spotify ID.""" - track_data = self._handle_response( - requests.get, self.audio_features_url + track_id - ) + try: + track_data = self._handle_response( + requests.get, self.audio_features_url + \ + track_id) + track_data.raise_for_status() + except requests.exceptions.RequestException as e: + self._log.debug('Audio feature update failed: {0}', str(e)) + track_data = None return track_data diff --git a/docs/changelog.rst b/docs/changelog.rst index 8cb27013d..dbee12764 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,8 +32,6 @@ New features: Bug fixes: -* Fix Spotify API error when tracks do not have audio-features. - :bug:`4385` * We now respect the Spotify API's rate limiting, which avoids crashing when the API reports code 429 (too many requests). :bug:`4370` * Fix implicit paths OR queries (e.g. ``beet list /path/ , /other-path/``) From decdb16a1537f5a88acba359035fa6c71ea26d9e Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Wed, 29 Jun 2022 20:13:48 -0400 Subject: [PATCH 304/357] lint --- beetsplug/spotify.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 666a19c08..f610cd956 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -644,8 +644,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): """Fetch track audio features by its Spotify ID.""" try: track_data = self._handle_response( - requests.get, self.audio_features_url + \ - track_id) + requests.get, self.audio_features_url + track_id) track_data.raise_for_status() except requests.exceptions.RequestException as e: self._log.debug('Audio feature update failed: {0}', str(e)) From bfe008ed24255542a7cd09c3f2624fcc586e5955 Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Thu, 30 Jun 2022 14:19:58 -0400 Subject: [PATCH 305/357] Fix typo in Acousticbrainz warning log Add missing space in Acousticbrainz plugin warning log. --- beetsplug/acousticbrainz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index eabc5849f..040463375 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -321,7 +321,7 @@ class AcousticPlugin(plugins.BeetsPlugin): else: yield v, subdata[k] else: - self._log.warning('Acousticbrainz did not provide info' + self._log.warning('Acousticbrainz did not provide info ' 'about {}', k) self._log.debug('Data {} could not be mapped to scheme {} ' 'because key {} was not found', subdata, v, k) From 6a131d210837ed59f4d54f45275ad83783866e1b Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 1 Jul 2022 10:26:08 -0400 Subject: [PATCH 306/357] Address comments --- beetsplug/spotify.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index f610cd956..2e4d9b402 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -622,7 +622,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): audio_features = \ self.track_audio_features(spotify_track_id) if audio_features is None: - self._log.debug('No audio features found for: {}', item) + self._log.info('No audio features found for: {}', item) continue for feature in audio_features.keys(): if feature in self.spotify_audio_features.keys(): @@ -645,8 +645,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): try: track_data = self._handle_response( requests.get, self.audio_features_url + track_id) - track_data.raise_for_status() - except requests.exceptions.RequestException as e: + except AttributeError: self._log.debug('Audio feature update failed: {0}', str(e)) track_data = None return track_data From 016526f30e41dd00a9b33997fac1f96f59e128bb Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 1 Jul 2022 10:29:20 -0400 Subject: [PATCH 307/357] Lint --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 2e4d9b402..50cf02951 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -646,6 +646,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): track_data = self._handle_response( requests.get, self.audio_features_url + track_id) except AttributeError: - self._log.debug('Audio feature update failed: {0}', str(e)) + self._log.debug('Audio feature update failed: {}', str(track_id)) track_data = None return track_data From b7ff616611b018d670f84a79ecc0d0edba5e9da5 Mon Sep 17 00:00:00 2001 From: clach04 Date: Fri, 1 Jul 2022 17:51:54 -0700 Subject: [PATCH 308/357] Version bump to 1.6.1 Matche setup.py (package) version --- beets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/__init__.py b/beets/__init__.py index 9642a6f3c..910ed78ea 100644 --- a/beets/__init__.py +++ b/beets/__init__.py @@ -16,7 +16,7 @@ import confuse from sys import stderr -__version__ = '1.6.0' +__version__ = '1.6.1' __author__ = 'Adrian Sampson ' From 1d241b0d52681aa66260f700508144afea1a4659 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 4 Jul 2022 14:12:22 -0400 Subject: [PATCH 309/357] Update error handling --- beetsplug/spotify.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 50cf02951..6cad6986d 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -34,6 +34,8 @@ from beets.plugins import BeetsPlugin, MetadataSourcePlugin DEFAULT_WAITING_TIME = 5 +class SpotifyAPIError(Exception): + pass class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Spotify' @@ -189,6 +191,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) + elif 'analysis not found' in response.text: + raise SpotifyAPIError("Audio Analysis not found") else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( @@ -645,7 +649,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): try: track_data = self._handle_response( requests.get, self.audio_features_url + track_id) - except AttributeError: - self._log.debug('Audio feature update failed: {}', str(track_id)) + except SpotifyAPIError as e: + self._log.debug('Spotify API error: {}', e) track_data = None return track_data From 85e124bcd0e3e3b35a314564693a7d0f0ad516e3 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 4 Jul 2022 14:15:08 -0400 Subject: [PATCH 310/357] lint errors --- beetsplug/spotify.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 6cad6986d..f4d43afa4 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -34,9 +34,11 @@ from beets.plugins import BeetsPlugin, MetadataSourcePlugin DEFAULT_WAITING_TIME = 5 + class SpotifyAPIError(Exception): pass + class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): data_source = 'Spotify' From 95243019e98204b3f45922417deb125679cf05aa Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 4 Jul 2022 14:29:27 -0400 Subject: [PATCH 311/357] Update exception --- beetsplug/spotify.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index f4d43afa4..ee5a6463c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -194,7 +194,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) elif 'analysis not found' in response.text: - raise SpotifyAPIError("Audio Analysis not found") + raise SpotifyAPIError("API Error {0.status_code} for {}" + .format(response, url)) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From 12d9b1bd22e1c930df443cc2f58f0bc66af8621d Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 4 Jul 2022 14:31:28 -0400 Subject: [PATCH 312/357] Update exception --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index ee5a6463c..fb3228baa 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -194,7 +194,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) elif 'analysis not found' in response.text: - raise SpotifyAPIError("API Error {0.status_code} for {}" + raise SpotifyAPIError("API Error {0.status_code} for {1}" .format(response, url)) else: raise ui.UserError( From ca4b5bcec418854abd01ccf3db69a94c542021fa Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 4 Jul 2022 14:34:32 -0400 Subject: [PATCH 313/357] lint --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index fb3228baa..ee86f2714 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -195,7 +195,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return self._handle_response(request_type, url, params=params) elif 'analysis not found' in response.text: raise SpotifyAPIError("API Error {0.status_code} for {1}" - .format(response, url)) + .format(response, url)) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From c7f465f968082f447ed6146c6a0c52733dc109a4 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 5 Jul 2022 20:46:14 -0400 Subject: [PATCH 314/357] Address comments --- beetsplug/spotify.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index ee86f2714..75ae52891 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -193,7 +193,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) - elif 'analysis not found' in response.text: + elif response.status_code == 400: raise SpotifyAPIError("API Error {0.status_code} for {1}" .format(response, url)) else: @@ -650,9 +650,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): def track_audio_features(self, track_id=None): """Fetch track audio features by its Spotify ID.""" try: - track_data = self._handle_response( + return self._handle_response( requests.get, self.audio_features_url + track_id) except SpotifyAPIError as e: self._log.debug('Spotify API error: {}', e) - track_data = None - return track_data + return None From e0d5de4714030d6ed55cb3237b1c94eb587f63fa Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 5 Jul 2022 20:58:02 -0400 Subject: [PATCH 315/357] Add album information to spotify tracks update Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update hooks.py Update hooks.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update spotify.py Update hooks.py Update spotify.py Update spotify.py Cleanup more cleanup --- beets/autotag/hooks.py | 3 ++- beetsplug/spotify.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 30904ff29..070c98abc 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -160,7 +160,7 @@ class TrackInfo(AttrDict): artist_sort=None, disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, - track_alt=None, work=None, mb_workid=None, + track_alt=None, work=None, mb_workid=None,album=None, work_disambig=None, bpm=None, initial_key=None, genre=None, **kwargs): self.title = title @@ -172,6 +172,7 @@ class TrackInfo(AttrDict): self.index = index self.media = media self.medium = medium + self.album=album self.medium_index = medium_index self.medium_total = medium_total self.artist_sort = artist_sort diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 75ae52891..a692accc4 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -291,11 +291,19 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): :rtype: beets.autotag.hooks.TrackInfo """ artist, artist_id = self.get_artist(track_data['artists']) + + # Get album information for spotify tracks + try: + album=track_data['album']['name'] + except KeyError: + album=None + pass return TrackInfo( title=track_data['name'], track_id=track_data['id'], spotify_track_id=track_data['id'], artist=artist, + album=album, artist_id=artist_id, spotify_artist_id=artist_id, length=track_data['duration_ms'] / 1000, From a4baa742d57c5214d62eae7fb0e742051e7a5b96 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 7 Jul 2022 14:14:11 -0400 Subject: [PATCH 316/357] Update error code --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 75ae52891..a9c8368b0 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -193,7 +193,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): seconds.', seconds) time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) - elif response.status_code == 400: + elif response.status_code == 404: raise SpotifyAPIError("API Error {0.status_code} for {1}" .format(response, url)) else: From d61e79c6f87241a9640729020d14333f6ec664bb Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 7 Jul 2022 14:17:16 -0400 Subject: [PATCH 317/357] Update hooks.py --- beets/autotag/hooks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 070c98abc..30904ff29 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -160,7 +160,7 @@ class TrackInfo(AttrDict): artist_sort=None, disctitle=None, artist_credit=None, data_source=None, data_url=None, media=None, lyricist=None, composer=None, composer_sort=None, arranger=None, - track_alt=None, work=None, mb_workid=None,album=None, + track_alt=None, work=None, mb_workid=None, work_disambig=None, bpm=None, initial_key=None, genre=None, **kwargs): self.title = title @@ -172,7 +172,6 @@ class TrackInfo(AttrDict): self.index = index self.media = media self.medium = medium - self.album=album self.medium_index = medium_index self.medium_total = medium_total self.artist_sort = artist_sort From 224d31e097ea009b180bf384d090a03595055924 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 8 Jul 2022 17:03:29 -0400 Subject: [PATCH 318/357] Update changelog.rst --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3c61b3bf8..289c02385 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,8 @@ Changelog goes here! New features: +* We now import and tag the `album` information when importing singletons using Spotify source. + :bug:`4398 * :doc:`/plugins/spotify`: The plugin now provides an additional command `spotifysync` that allows getting track popularity and audio features information from Spotify. From 82e12c6b4bf05c4e1f1fa114a9463f2d9f75f934 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 8 Jul 2022 17:08:04 -0400 Subject: [PATCH 319/357] Add missing whitespaces --- beetsplug/spotify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 69f100f59..8d8862e8b 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -294,9 +294,9 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): # Get album information for spotify tracks try: - album=track_data['album']['name'] + album = track_data['album']['name'] except KeyError: - album=None + album = None pass return TrackInfo( title=track_data['name'], From a878cc2aadbe7eb6fe7bc5dae90b5498fb4c179f Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 8 Jul 2022 17:10:04 -0400 Subject: [PATCH 320/357] Update changelog.rst --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 289c02385..f42257448 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog goes here! New features: * We now import and tag the `album` information when importing singletons using Spotify source. - :bug:`4398 + :bug:`4398` * :doc:`/plugins/spotify`: The plugin now provides an additional command `spotifysync` that allows getting track popularity and audio features information from Spotify. From 2a18ab062e7d3024b71d3fc7fed9d61ee090acad Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sat, 9 Jul 2022 15:00:29 -0400 Subject: [PATCH 321/357] Remove extra pass statement --- beetsplug/spotify.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 8d8862e8b..ef7407b36 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -297,7 +297,6 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): album = track_data['album']['name'] except KeyError: album = None - pass return TrackInfo( title=track_data['name'], track_id=track_data['id'], From b64cefb0d0f3994f73f9fe354135e08b0daaa202 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 10 Jul 2022 13:11:27 -0400 Subject: [PATCH 322/357] Add params to debug --- beetsplug/spotify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index ef7407b36..0138f0b9f 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -194,8 +194,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) elif response.status_code == 404: - raise SpotifyAPIError("API Error {0.status_code} for {1}" - .format(response, url)) + raise SpotifyAPIError("API Error {0.status_code} for {1} and \ + params = {2}".format(response, url, params=params)) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From a28f8835cb888a6601f3d0b205962e3dda8ffa59 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 10 Jul 2022 13:12:38 -0400 Subject: [PATCH 323/357] Update spotify.py --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 0138f0b9f..8abf3154c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -195,7 +195,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return self._handle_response(request_type, url, params=params) elif response.status_code == 404: raise SpotifyAPIError("API Error {0.status_code} for {1} and \ - params = {2}".format(response, url, params=params)) + params = {2}".format(response, url, params)) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From 2bd61a7c48eb3750397387c8a2e1782c7d417e20 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 10 Jul 2022 13:15:25 -0400 Subject: [PATCH 324/357] Update spotify.py --- beetsplug/spotify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 8abf3154c..7071e1c0c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -194,8 +194,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) elif response.status_code == 404: - raise SpotifyAPIError("API Error {0.status_code} for {1} and \ - params = {2}".format(response, url, params)) + raise SpotifyAPIError("API Error {} for {} and params = {}". + format(response.status_code, url, params)) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From c2ad2b3d4c446e0cc4b14922104fea86da03294b Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 10 Jul 2022 13:23:10 -0400 Subject: [PATCH 325/357] Update spotify.py --- beetsplug/spotify.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 7071e1c0c..b95546dd2 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -194,8 +194,9 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): time.sleep(int(seconds) + 1) return self._handle_response(request_type, url, params=params) elif response.status_code == 404: - raise SpotifyAPIError("API Error {} for {} and params = {}". - format(response.status_code, url, params)) + raise SpotifyAPIError("API Error: {}\nURL: {}\nparams: {}". + format(response.status_code, url, + params)) else: raise ui.UserError( '{} API error:\n{}\nURL:\n{}\nparams:\n{}'.format( From 7b94bbd76499b059e945a52dfc90b2c7fc3d64e5 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 11 Jul 2022 11:36:08 -0400 Subject: [PATCH 326/357] Update spotify.py --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index b95546dd2..5ecb8e81a 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -36,7 +36,7 @@ DEFAULT_WAITING_TIME = 5 class SpotifyAPIError(Exception): - pass + continue class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): From 525e5eafd5a875a052294eac3307a1f886e7bf9a Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 11 Jul 2022 11:36:52 -0400 Subject: [PATCH 327/357] revert --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 5ecb8e81a..b95546dd2 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -36,7 +36,7 @@ DEFAULT_WAITING_TIME = 5 class SpotifyAPIError(Exception): - continue + pass class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): From d82362df3e2774ab3fc929b0276df3d39cfc54fa Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 11 Jul 2022 11:40:32 -0400 Subject: [PATCH 328/357] Update spotify.py --- beetsplug/spotify.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index b95546dd2..2d3b7c975 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -658,8 +658,9 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): def track_audio_features(self, track_id=None): """Fetch track audio features by its Spotify ID.""" try: - return self._handle_response( + track_features = self._handle_response( requests.get, self.audio_features_url + track_id) except SpotifyAPIError as e: self._log.debug('Spotify API error: {}', e) - return None + track_features = None + return track_features From e1153f7772f931fe74a49b912b4a98492097e107 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 11 Jul 2022 11:42:24 -0400 Subject: [PATCH 329/357] revert --- beetsplug/spotify.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 2d3b7c975..b95546dd2 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -658,9 +658,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): def track_audio_features(self, track_id=None): """Fetch track audio features by its Spotify ID.""" try: - track_features = self._handle_response( + return self._handle_response( requests.get, self.audio_features_url + track_id) except SpotifyAPIError as e: self._log.debug('Spotify API error: {}', e) - track_features = None - return track_features + return None From 4e63c8893b58f7d513a1ae7546dfa5084a9e5308 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 09:28:23 -0400 Subject: [PATCH 330/357] refactor handle_reponse --- beetsplug/spotify.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index b95546dd2..b7df9dcf8 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -396,15 +396,18 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): self._log.debug( f"Searching {self.data_source} for '{query}'" ) - response_data = ( - self._handle_response( + try: + response = self._handle_response( requests.get, self.search_url, params={'q': query, 'type': query_type}, ) - .get(query_type + 's', {}) - .get('items', []) - ) + except SpotifyAPIError as e: + self._log.debug('Spotify API error: {}', e) + response_data = (response + .get(query_type + 's', {}) + .get('items', []) + ) self._log.debug( "Found {} result(s) from {} for '{}'", len(response_data), From 85e58d48a26463f0f370f5f051f65bb9451431af Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 09:30:07 -0400 Subject: [PATCH 331/357] Update spotify.py --- beetsplug/spotify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index b7df9dcf8..d3910523c 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -404,6 +404,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): ) except SpotifyAPIError as e: self._log.debug('Spotify API error: {}', e) + return None response_data = (response .get(query_type + 's', {}) .get('items', []) From 500ff5135c34a2bd13d8ca35b8ac6d80c3ef73db Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 09:37:14 -0400 Subject: [PATCH 332/357] Update plugins.py --- beets/plugins.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index ed1f82d8f..6a68a3d34 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -749,7 +749,10 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta): tracks = self._search_api( query_type='track', keywords=title, filters={'artist': artist} ) - return [self.track_for_id(track_data=track) for track in tracks] + if tracks is not None: + return [self.track_for_id(track_data=track) for track in tracks] + else: + return None def album_distance(self, items, album_info, mapping): return get_distance( From 16f8b47be3010e6d8826ed102d4889bf6829fc43 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 09:49:49 -0400 Subject: [PATCH 333/357] Update plugins.py --- beets/plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 6a68a3d34..2f5ba397b 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -389,7 +389,8 @@ def item_candidates(item, artist, title): """Gets MusicBrainz candidates for an item from the plugins. """ for plugin in find_plugins(): - yield from plugin.item_candidates(item, artist, title) + if plugin.item_candidates(item, artist, title) is not None: + yield from plugin.item_candidates(item, artist, title) def album_for_id(album_id): From 28614d94dc8a78d5012d97a56406fe360aa67c67 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 10:32:55 -0400 Subject: [PATCH 334/357] lint --- beetsplug/spotify.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index d3910523c..be9a8509e 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -405,10 +405,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): except SpotifyAPIError as e: self._log.debug('Spotify API error: {}', e) return None - response_data = (response - .get(query_type + 's', {}) - .get('items', []) - ) + response_data = (response.get(query_type + 's', {}) + .get('items', [])) self._log.debug( "Found {} result(s) from {} for '{}'", len(response_data), From 3896e5cd9c6510b3e8e54728d89c4d2a07294b74 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 16:14:22 -0400 Subject: [PATCH 335/357] Update plugins.py --- beets/plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 2f5ba397b..bbf29e4be 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -731,7 +731,8 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta): if not va_likely: query_filters['artist'] = artist results = self._search_api(query_type='album', filters=query_filters) - albums = [self.album_for_id(album_id=r['id']) for r in results] + if results is not None: + albums = [self.album_for_id(album_id=r['id']) for r in results] return [a for a in albums if a is not None] def item_candidates(self, item, artist, title): From 32afd79eed7f5d89dd5e34d653ec890d571cae63 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 16:15:59 -0400 Subject: [PATCH 336/357] Update plugins.py --- beets/plugins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beets/plugins.py b/beets/plugins.py index bbf29e4be..b31c55482 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -733,6 +733,8 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta): results = self._search_api(query_type='album', filters=query_filters) if results is not None: albums = [self.album_for_id(album_id=r['id']) for r in results] + else: + albums = None return [a for a in albums if a is not None] def item_candidates(self, item, artist, title): From 5084794a94115e532982bd80f3bbab58e3187712 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 16:17:46 -0400 Subject: [PATCH 337/357] Update plugins.py --- beets/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index b31c55482..9b966cc19 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -733,9 +733,9 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta): results = self._search_api(query_type='album', filters=query_filters) if results is not None: albums = [self.album_for_id(album_id=r['id']) for r in results] + return [a for a in albums if a is not None] else: - albums = None - return [a for a in albums if a is not None] + return None def item_candidates(self, item, artist, title): """Returns a list of TrackInfo objects for Search API results From c685f196441ad1018cedc2d0935b8c2d6fcc44cf Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 16:21:06 -0400 Subject: [PATCH 338/357] Update plugins.py --- beets/plugins.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 9b966cc19..82c9ccb2b 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -381,7 +381,9 @@ def candidates(items, artist, album, va_likely, extra_tags=None): """Gets MusicBrainz candidates for an album from each plugin. """ for plugin in find_plugins(): - yield from plugin.candidates(items, artist, album, va_likely, + if plugin.candidates(items, artist, album, va_likely, + extra_tags) is not None: + yield from plugin.candidates(items, artist, album, va_likely, extra_tags) From 8fcd87ef9feb3a2e253969f70d85e1d852354668 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Tue, 12 Jul 2022 16:28:23 -0400 Subject: [PATCH 339/357] Lint --- beets/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/plugins.py b/beets/plugins.py index 82c9ccb2b..13fb50cae 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -384,7 +384,7 @@ def candidates(items, artist, album, va_likely, extra_tags=None): if plugin.candidates(items, artist, album, va_likely, extra_tags) is not None: yield from plugin.candidates(items, artist, album, va_likely, - extra_tags) + extra_tags) def item_candidates(item, artist, title): From 01fa07de06b113d534906648e8b6f2ec40d2d8d7 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Thu, 14 Jul 2022 20:56:07 -0400 Subject: [PATCH 340/357] Revert plugins.py changes --- beets/plugins.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 13fb50cae..ed1f82d8f 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -381,18 +381,15 @@ def candidates(items, artist, album, va_likely, extra_tags=None): """Gets MusicBrainz candidates for an album from each plugin. """ for plugin in find_plugins(): - if plugin.candidates(items, artist, album, va_likely, - extra_tags) is not None: - yield from plugin.candidates(items, artist, album, va_likely, - extra_tags) + yield from plugin.candidates(items, artist, album, va_likely, + extra_tags) def item_candidates(item, artist, title): """Gets MusicBrainz candidates for an item from the plugins. """ for plugin in find_plugins(): - if plugin.item_candidates(item, artist, title) is not None: - yield from plugin.item_candidates(item, artist, title) + yield from plugin.item_candidates(item, artist, title) def album_for_id(album_id): @@ -733,11 +730,8 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta): if not va_likely: query_filters['artist'] = artist results = self._search_api(query_type='album', filters=query_filters) - if results is not None: - albums = [self.album_for_id(album_id=r['id']) for r in results] - return [a for a in albums if a is not None] - else: - return None + albums = [self.album_for_id(album_id=r['id']) for r in results] + return [a for a in albums if a is not None] def item_candidates(self, item, artist, title): """Returns a list of TrackInfo objects for Search API results @@ -755,10 +749,7 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta): tracks = self._search_api( query_type='track', keywords=title, filters={'artist': artist} ) - if tracks is not None: - return [self.track_for_id(track_data=track) for track in tracks] - else: - return None + return [self.track_for_id(track_data=track) for track in tracks] def album_distance(self, items, album_info, mapping): return get_distance( From 06825e0729abf1897db62f122b31e843e0b8074f Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Fri, 15 Jul 2022 09:22:23 -0400 Subject: [PATCH 341/357] Return an empty sequence --- beetsplug/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index be9a8509e..30fbabc06 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -404,7 +404,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): ) except SpotifyAPIError as e: self._log.debug('Spotify API error: {}', e) - return None + return [] response_data = (response.get(query_type + 's', {}) .get('items', [])) self._log.debug( From ac5634d592cbf26ab6d72b9b5e4db17647aa3121 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Sat, 30 Jul 2022 16:10:58 +0000 Subject: [PATCH 342/357] Fix old alias It looks like the convert option for wma used to be called windows media. We could just remove this alias, but might be good to keep for backwards compatibility. --- beetsplug/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index a8674ca79..95240dc39 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -37,7 +37,7 @@ _temp_files = [] # Keep track of temporary transcoded files for deletion. # Some convenient alternate names for formats. ALIASES = { - 'wma': 'windows media', + 'windows media': 'wma', 'vorbis': 'ogg', } From d66a513d80c09db4cceac2bd91197468aea4e281 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Sat, 30 Jul 2022 16:13:18 +0000 Subject: [PATCH 343/357] Fix documentation to refer to wma instead of wmv This format actually has a definition. --- docs/plugins/convert.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index ff439eb97..aa28ed240 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -165,7 +165,7 @@ command to use to transcode audio. The tokens ``$source`` and ``$dest`` in the command are replaced with the paths to the existing and new file. The plugin in comes with default commands for the most common audio -formats: `mp3`, `alac`, `flac`, `aac`, `opus`, `ogg`, `wmv`. For +formats: `mp3`, `alac`, `flac`, `aac`, `opus`, `ogg`, `wma`. For details have a look at the output of ``beet config -d``. For a one-command-fits-all solution use the ``convert.command`` and From fd37fec73ec125e5c2dc7b17bd20ac88a966f3b4 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Sat, 30 Jul 2022 16:15:44 +0000 Subject: [PATCH 344/357] Update changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f42257448..163e69f72 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -90,6 +90,7 @@ Bug fixes: * :doc:`plugins/importadded`: Fix a bug with recently added reflink import option that casues a crash when ImportAdded plugin enabled. :bug:`4389` +* :doc:`plugins/convert`: Fix a bug with the `wma` format alias. For packagers: From c5e68f5643a69c2820f7e9de4293d35221196f67 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 30 Jul 2022 19:54:11 -0400 Subject: [PATCH 345/357] Adapt to pycodestyle changes --- beets/importer.py | 4 ++-- beets/ui/__init__.py | 2 +- beetsplug/acousticbrainz.py | 7 ++++--- beetsplug/bpd/__init__.py | 2 +- beetsplug/fetchart.py | 5 +++-- beetsplug/replaygain.py | 4 ++-- beetsplug/spotify.py | 4 ++-- test/test_convert.py | 3 ++- test/test_importer.py | 12 ++++++------ test/test_lyrics.py | 3 ++- 10 files changed, 25 insertions(+), 21 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index 561cedd2c..14478b43d 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -726,8 +726,8 @@ class ImportTask(BaseImportTask): item.update(changes) def manipulate_files(self, operation=None, write=False, session=None): - """ Copy, move, link, hardlink or reflink (depending on `operation`) the files - as well as write metadata. + """ Copy, move, link, hardlink or reflink (depending on `operation`) + the files as well as write metadata. `operation` should be an instance of `util.MoveOperation`. diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 900cb9305..ba058148d 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -581,7 +581,7 @@ def _colordiff(a, b, highlight='text_highlight', a_out.append(colorize(color, a[a_start:a_end])) b_out.append(colorize(color, b[b_start:b_end])) else: - assert(False) + assert False return ''.join(a_out), ''.join(b_out) diff --git a/beetsplug/acousticbrainz.py b/beetsplug/acousticbrainz.py index 040463375..0cfd6e318 100644 --- a/beetsplug/acousticbrainz.py +++ b/beetsplug/acousticbrainz.py @@ -233,9 +233,10 @@ class AcousticPlugin(plugins.BeetsPlugin): item.try_write() def _map_data_to_scheme(self, data, scheme): - """Given `data` as a structure of nested dictionaries, and `scheme` as a - structure of nested dictionaries , `yield` tuples `(attr, val)` where - `attr` and `val` are corresponding leaf nodes in `scheme` and `data`. + """Given `data` as a structure of nested dictionaries, and + `scheme` as a structure of nested dictionaries , `yield` tuples + `(attr, val)` where `attr` and `val` are corresponding leaf + nodes in `scheme` and `data`. As its name indicates, `scheme` defines how the data is structured, so this function tries to find leaf nodes in `data` that correspond diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index 07198b1b4..8c02d3d44 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -510,7 +510,7 @@ class BaseServer: """Remove the song at index from the playlist.""" index = cast_arg(int, index) try: - del(self.playlist[index]) + del self.playlist[index] except IndexError: raise ArgumentIndexError() self.playlist_version += 1 diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index f2c1e5a7a..c99c7081f 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -362,8 +362,9 @@ class CoverArtArchive(RemoteArtSource): GROUP_URL = 'https://coverartarchive.org/release-group/{mbid}' def get(self, album, plugin, paths): - """Return the Cover Art Archive and Cover Art Archive release group URLs - using album MusicBrainz release ID and release group ID. + """Return the Cover Art Archive and Cover Art Archive release + group URLs using album MusicBrainz release ID and release group + ID. """ def get_image_urls(url, size_suffix=None): diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 78f146a82..c228f74b3 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -904,14 +904,14 @@ class GStreamerBackend(Backend): def _on_pad_added(self, decbin, pad): sink_pad = self._conv.get_compatible_pad(pad, None) - assert(sink_pad is not None) + assert sink_pad is not None pad.link(sink_pad) def _on_pad_removed(self, decbin, pad): # Called when the decodebin element is disconnected from the # rest of the pipeline while switching input files peer = pad.get_peer() - assert(peer is None) + assert peer is None class AudioToolsBackend(Backend): diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 30fbabc06..2cbacc92f 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -374,8 +374,8 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): return unidecode.unidecode(query) def _search_api(self, query_type, filters=None, keywords=''): - """Query the Spotify Search API for the specified ``keywords``, applying - the provided ``filters``. + """Query the Spotify Search API for the specified ``keywords``, + applying the provided ``filters``. :param query_type: Item type to search across. Valid types are: 'album', 'artist', 'playlist', and 'track'. diff --git a/test/test_convert.py b/test/test_convert.py index cd32e34b1..8786be400 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -48,7 +48,8 @@ class TestHelper(helper.TestHelper): shell_quote(stub), tag) def assertFileTag(self, path, tag): # noqa - """Assert that the path is a file and the files content ends with `tag`. + """Assert that the path is a file and the files content ends + with `tag`. """ display_tag = tag tag = tag.encode('utf-8') diff --git a/test/test_importer.py b/test/test_importer.py index 60c0b793f..2bd95315f 100644 --- a/test/test_importer.py +++ b/test/test_importer.py @@ -228,14 +228,14 @@ class ImportHelper(TestHelper): ) def assert_file_in_lib(self, *segments): - """Join the ``segments`` and assert that this path exists in the library - directory + """Join the ``segments`` and assert that this path exists in the + library directory. """ self.assertExists(os.path.join(self.libdir, *segments)) def assert_file_not_in_lib(self, *segments): - """Join the ``segments`` and assert that this path exists in the library - directory + """Join the ``segments`` and assert that this path does not + exist in the library directory. """ self.assertNotExists(os.path.join(self.libdir, *segments)) @@ -462,8 +462,8 @@ class ImportPasswordRarTest(ImportZipTest): class ImportSingletonTest(_common.TestCase, ImportHelper): - """Test ``APPLY`` and ``ASIS`` choices for an import session with singletons - config set to True. + """Test ``APPLY`` and ``ASIS`` choices for an import session with + singletons config set to True. """ def setUp(self): diff --git a/test/test_lyrics.py b/test/test_lyrics.py index e3e3be96f..f8dd0b369 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -336,7 +336,8 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest, LyricsAssertions): os.environ.get('INTEGRATION_TEST', '0') == '1', 'integration testing not enabled') def test_backend_sources_ok(self): - """Test default backends with songs known to exist in respective databases. + """Test default backends with songs known to exist in respective + databases. """ # Don't test any sources marked as skipped. sources = [s for s in self.DEFAULT_SOURCES if not s.get("skip", False)] From a8434f6c380d00776dbdcf77cdced299fe470eff Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Sun, 31 Jul 2022 10:33:16 -0400 Subject: [PATCH 346/357] Add last_update --- beetsplug/spotify.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 2cbacc92f..3181b1f41 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -19,6 +19,7 @@ Spotify playlist construction. import base64 import collections +import datetime import json import re import time @@ -30,6 +31,7 @@ import unidecode from beets import ui from beets.autotag.hooks import AlbumInfo, TrackInfo from beets.dbcore import types +from beets.library import DateType from beets.plugins import BeetsPlugin, MetadataSourcePlugin DEFAULT_WAITING_TIME = 5 @@ -56,6 +58,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): 'spotify_tempo': types.FLOAT, 'spotify_time_signature': types.INTEGER, 'spotify_valence': types.FLOAT, + 'spotify_lastupdatedat': DateType(), } # Base URLs for the Spotify API @@ -645,6 +648,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): if feature in self.spotify_audio_features.keys(): item[self.spotify_audio_features[feature]] = \ audio_features[feature] + item['spotify_lastupdatedat'] = datetime.datetime.now() item.store() if write: item.try_write() From c03537c12b9cd34d596b9e0466f8e36fa24151e0 Mon Sep 17 00:00:00 2001 From: Alok Saboo Date: Mon, 1 Aug 2022 08:06:42 -0400 Subject: [PATCH 347/357] Address comments --- beetsplug/spotify.py | 4 ++-- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/spotify.py b/beetsplug/spotify.py index 3181b1f41..f1aecb583 100644 --- a/beetsplug/spotify.py +++ b/beetsplug/spotify.py @@ -58,7 +58,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): 'spotify_tempo': types.FLOAT, 'spotify_time_signature': types.INTEGER, 'spotify_valence': types.FLOAT, - 'spotify_lastupdatedat': DateType(), + 'spotify_updated': DateType(), } # Base URLs for the Spotify API @@ -648,7 +648,7 @@ class SpotifyPlugin(MetadataSourcePlugin, BeetsPlugin): if feature in self.spotify_audio_features.keys(): item[self.spotify_audio_features[feature]] = \ audio_features[feature] - item['spotify_lastupdatedat'] = datetime.datetime.now() + item['spotify_updated'] = datetime.datetime.now() item.store() if write: item.try_write() diff --git a/docs/changelog.rst b/docs/changelog.rst index 163e69f72..e6323393f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,7 @@ Changelog goes here! New features: +* Added `spotify_updated` field to track when the information was last updated. * We now import and tag the `album` information when importing singletons using Spotify source. :bug:`4398` * :doc:`/plugins/spotify`: The plugin now provides an additional command From fde2ad3f65c76f040ef5794b1ed3e852577d7059 Mon Sep 17 00:00:00 2001 From: vicholp Date: Wed, 3 Aug 2022 01:22:35 -0400 Subject: [PATCH 348/357] fix get item file of web plugin --- beetsplug/web/__init__.py | 2 +- docs/changelog.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 63f7f92ad..b7baa93c1 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -324,7 +324,7 @@ def item_file(item_id): response = flask.send_file( item_path, as_attachment=True, - attachment_filename=safe_filename + download_name=safe_filename ) response.headers['Content-Length'] = os.path.getsize(item_path) return response diff --git a/docs/changelog.rst b/docs/changelog.rst index e6323393f..31861af24 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -92,6 +92,7 @@ Bug fixes: that casues a crash when ImportAdded plugin enabled. :bug:`4389` * :doc:`plugins/convert`: Fix a bug with the `wma` format alias. +* :doc:`/plugins/web`: Fix get file from item. For packagers: From 6803ef3b83b98fdc21e42df5cacd247e14be1b90 Mon Sep 17 00:00:00 2001 From: vicholp Date: Wed, 3 Aug 2022 01:22:45 -0400 Subject: [PATCH 349/357] add test to get item file of web plugin --- test/test_web.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_web.py b/test/test_web.py index 9a18b1dba..3c84b14ac 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -667,6 +667,16 @@ class WebPluginTest(_common.LibTestCase): # Remove the item self.lib.get_item(item_id).remove() + def test_get_item_file(self): + ipath = os.path.join(self.temp_dir, b'testfile2.mp3') + shutil.copy(os.path.join(_common.RSRC, b'full.mp3'), ipath) + self.assertTrue(os.path.exists(ipath)) + item_id = self.lib.add(Item.from_path(ipath)) + + response = self.client.get('/item/' + str(item_id) + '/file') + + self.assertEqual(response.status_code, 200) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 8d957f35f976d5bc22692088b51bbf5056051745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 12 Aug 2022 14:19:52 +0200 Subject: [PATCH 350/357] Add path template "sunique" to disambiguate between singleton tracks --- beets/config_default.yaml | 5 +++ beets/library.py | 93 +++++++++++++++++++++++++++++++++++++++ test/test_library.py | 85 +++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+) diff --git a/beets/config_default.yaml b/beets/config_default.yaml index fd2dbf551..db36ef080 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -55,6 +55,11 @@ aunique: disambiguators: albumtype year label catalognum albumdisambig releasegroupdisambig bracket: '[]' +sunique: + keys: artist title + disambiguators: year trackdisambig + bracket: '[]' + overwrite_null: album: [] track: [] diff --git a/beets/library.py b/beets/library.py index c8fa2b5fc..788dab92f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1753,6 +1753,99 @@ class DefaultTemplateFunctions: self.lib._memotable[memokey] = res return res + def tmpl_sunique(self, keys=None, disam=None, bracket=None): + """Generate a string that is guaranteed to be unique among all + singletons in the library who share the same set of keys. + + A fields from "disam" is used in the string if one is sufficient to + disambiguate the albums. Otherwise, a fallback opaque value is + used. Both "keys" and "disam" should be given as + whitespace-separated lists of field names, while "bracket" is a + pair of characters to be used as brackets surrounding the + disambiguator or empty to have no brackets. + """ + # Fast paths: no album, no item or library, or memoized value. + if not self.item or not self.lib: + return '' + + if isinstance(self.item, Item): + item_id = self.item.id + album_id = self.item.album_id + else: + raise NotImplementedError("sunique is only implemented for items") + + if item_id is None: + return '' + + memokey = ('sunique', keys, disam, item_id) + memoval = self.lib._memotable.get(memokey) + if memoval is not None: + return memoval + + keys = keys or beets.config['sunique']['keys'].as_str() + disam = disam or beets.config['sunique']['disambiguators'].as_str() + if bracket is None: + bracket = beets.config['sunique']['bracket'].as_str() + keys = keys.split() + disam = disam.split() + + # Assign a left and right bracket or leave blank if argument is empty. + if len(bracket) == 2: + bracket_l = bracket[0] + bracket_r = bracket[1] + else: + bracket_l = '' + bracket_r = '' + + if album_id is not None: + # Do nothing for non singletons. + self.lib._memotable[memokey] = '' + return '' + + # Find matching singletons to disambiguate with. + subqueries = [dbcore.query.NoneQuery('album_id', True)] + item_keys = Item.all_keys() + for key in keys: + value = self.item.get(key, '') + # Use slow queries for flexible attributes. + fast = key in item_keys + subqueries.append(dbcore.MatchQuery(key, value, fast)) + singletons = self.lib.items(dbcore.AndQuery(subqueries)) + + # If there's only one singleton to matching these details, then do + # nothing. + if len(singletons) == 1: + self.lib._memotable[memokey] = '' + return '' + + # Find the first disambiguator that distinguishes the singletons. + for disambiguator in disam: + # Get the value for each singleton for the current field. + disam_values = {s.get(disambiguator, '') for s in singletons} + + # If the set of unique values is equal to the number of + # singletons in the disambiguation set, we're done -- this is + # sufficient disambiguation. + if len(disam_values) == len(singletons): + break + else: + # No disambiguator distinguished all fields. + res = f' {bracket_l}{item_id}{bracket_r}' + self.lib._memotable[memokey] = res + return res + + # Flatten disambiguation value into a string. + disam_value = self.item.formatted(for_path=True).get(disambiguator) + + # Return empty string if disambiguator is empty. + if disam_value: + res = f' {bracket_l}{disam_value}{bracket_r}' + else: + res = '' + + self.lib._memotable[memokey] = res + return res + @staticmethod def tmpl_first(s, count=1, skip=0, sep='; ', join_str='; '): """Get the item(s) from x to y in a string separated by something diff --git a/test/test_library.py b/test/test_library.py index 6981b87f9..31ced7a2c 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -805,6 +805,91 @@ class DisambiguationTest(_common.TestCase, PathFormattingMixin): self._assert_dest(b'/base/foo/the title', self.i1) +class SingletonDisambiguationTest(_common.TestCase, PathFormattingMixin): + def setUp(self): + super().setUp() + self.lib = beets.library.Library(':memory:') + self.lib.directory = b'/base' + self.lib.path_formats = [('default', 'path')] + + self.i1 = item() + self.i1.year = 2001 + self.lib.add(self.i1) + self.i2 = item() + self.i2.year = 2002 + self.lib.add(self.i2) + self.lib._connection().commit() + + self._setf('foo/$title%sunique{artist title,year}') + + def tearDown(self): + super().tearDown() + self.lib._connection().close() + + def test_sunique_expands_to_disambiguating_year(self): + self._assert_dest(b'/base/foo/the title [2001]', self.i1) + + def test_sunique_with_default_arguments_uses_trackdisambig(self): + self.i1.trackdisambig = 'live version' + self.i1.year = self.i2.year + self.i1.store() + self._setf('foo/$title%sunique{}') + self._assert_dest(b'/base/foo/the title [live version]', self.i1) + + def test_sunique_expands_to_nothing_for_distinct_singletons(self): + self.i2.title = 'different track' + self.i2.store() + + self._assert_dest(b'/base/foo/the title', self.i1) + + def test_sunique_does_not_match_album(self): + self.lib.add_album([self.i2]) + self._assert_dest(b'/base/foo/the title', self.i1) + + def test_sunique_use_fallback_numbers_when_identical(self): + self.i2.year = self.i1.year + self.i2.store() + + self._assert_dest(b'/base/foo/the title [1]', self.i1) + self._assert_dest(b'/base/foo/the title [2]', self.i2) + + def test_sunique_falls_back_to_second_distinguishing_field(self): + self._setf('foo/$title%sunique{albumartist album,month year}') + self._assert_dest(b'/base/foo/the title [2001]', self.i1) + + def test_sunique_sanitized(self): + self.i2.year = self.i1.year + self.i1.trackdisambig = 'foo/bar' + self.i2.store() + self.i1.store() + self._setf('foo/$title%sunique{artist title,trackdisambig}') + self._assert_dest(b'/base/foo/the title [foo_bar]', self.i1) + + def test_drop_empty_disambig_string(self): + self.i1.trackdisambig = None + self.i2.trackdisambig = 'foo' + self.i1.store() + self.i2.store() + self._setf('foo/$title%sunique{albumartist album,trackdisambig}') + self._assert_dest(b'/base/foo/the title', self.i1) + + def test_change_brackets(self): + self._setf('foo/$title%sunique{artist title,year,()}') + self._assert_dest(b'/base/foo/the title (2001)', self.i1) + + def test_remove_brackets(self): + self._setf('foo/$title%sunique{artist title,year,}') + self._assert_dest(b'/base/foo/the title 2001', self.i1) + + def test_key_flexible_attribute(self): + self.i1.flex = 'flex1' + self.i2.flex = 'flex2' + self.i1.store() + self.i2.store() + self._setf('foo/$title%sunique{artist title flex,year}') + self._assert_dest(b'/base/foo/the title', self.i1) + + class PluginDestinationTest(_common.TestCase): def setUp(self): super().setUp() From f641df0748bcc83da5f224c95e523b8bbd261c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 16 Aug 2022 17:54:12 +0200 Subject: [PATCH 351/357] Encapsulate common code for the aunique and sunique templates in a single method --- beets/library.py | 151 ++++++++++++++++++++--------------------------- 1 file changed, 65 insertions(+), 86 deletions(-) diff --git a/beets/library.py b/beets/library.py index 788dab92f..3b8a85685 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1683,75 +1683,17 @@ class DefaultTemplateFunctions: if album_id is None: return '' - memokey = ('aunique', keys, disam, album_id) + memokey = self._tmpl_unique_memokey('aunique', keys, disam, album_id) memoval = self.lib._memotable.get(memokey) if memoval is not None: return memoval - keys = keys or beets.config['aunique']['keys'].as_str() - disam = disam or beets.config['aunique']['disambiguators'].as_str() - if bracket is None: - bracket = beets.config['aunique']['bracket'].as_str() - keys = keys.split() - disam = disam.split() - - # Assign a left and right bracket or leave blank if argument is empty. - if len(bracket) == 2: - bracket_l = bracket[0] - bracket_r = bracket[1] - else: - bracket_l = '' - bracket_r = '' - album = self.lib.get_album(album_id) - if not album: + + return self._tmpl_unique( + 'aunique', keys, disam, bracket, album_id, album, album.item_keys, # Do nothing for singletons. - self.lib._memotable[memokey] = '' - return '' - - # Find matching albums to disambiguate with. - subqueries = [] - for key in keys: - value = album.get(key, '') - # Use slow queries for flexible attributes. - fast = key in album.item_keys - subqueries.append(dbcore.MatchQuery(key, value, fast)) - albums = self.lib.albums(dbcore.AndQuery(subqueries)) - - # If there's only one album to matching these details, then do - # nothing. - if len(albums) == 1: - self.lib._memotable[memokey] = '' - return '' - - # Find the first disambiguator that distinguishes the albums. - for disambiguator in disam: - # Get the value for each album for the current field. - disam_values = {a.get(disambiguator, '') for a in albums} - - # If the set of unique values is equal to the number of - # albums in the disambiguation set, we're done -- this is - # sufficient disambiguation. - if len(disam_values) == len(albums): - break - - else: - # No disambiguator distinguished all fields. - res = f' {bracket_l}{album.id}{bracket_r}' - self.lib._memotable[memokey] = res - return res - - # Flatten disambiguation value into a string. - disam_value = album.formatted(for_path=True).get(disambiguator) - - # Return empty string if disambiguator is empty. - if disam_value: - res = f' {bracket_l}{disam_value}{bracket_r}' - else: - res = '' - - self.lib._memotable[memokey] = res - return res + lambda a: a is None) def tmpl_sunique(self, keys=None, disam=None, bracket=None): """Generate a string that is guaranteed to be unique among all @@ -1770,22 +1712,60 @@ class DefaultTemplateFunctions: if isinstance(self.item, Item): item_id = self.item.id - album_id = self.item.album_id else: raise NotImplementedError("sunique is only implemented for items") if item_id is None: return '' - memokey = ('sunique', keys, disam, item_id) + return self._tmpl_unique( + 'sunique', keys, disam, bracket, item_id, self.item, + Item.all_keys(), + # Do nothing for non singletons. + lambda i: i.album_id is not None, + initial_subqueries=[dbcore.query.NoneQuery('album_id', True)]) + + def _tmpl_unique_memokey(self, name, keys, disam, item_id): + """Get the memokey for the unique template named "name" for the + specific parameters. + """ + return (name, keys, disam, item_id) + + def _tmpl_unique(self, name, keys, disam, bracket, item_id, db_item, + item_keys, skip_item, initial_subqueries=None): + """Generate a string that is guaranteed to be unique among all items of + the same type as "db_item" who share the same set of keys. + + A field from "disam" is used in the string if one is sufficient to + disambiguate the items. Otherwise, a fallback opaque value is + used. Both "keys" and "disam" should be given as + whitespace-separated lists of field names, while "bracket" is a + pair of characters to be used as brackets surrounding the + disambiguator or empty to have no brackets. + + "name" is the name of the templates. It is also the name of the + configuration section where the default values of the parameters + are stored. + + "skip_item" is a function that must return True when the template + should return an empty string. + + "initial_subqueries" is a list of subqueries that should be included + in the query to find the ambigous items. + """ + memokey = self._tmpl_unique_memokey(name, keys, disam, item_id) memoval = self.lib._memotable.get(memokey) if memoval is not None: return memoval - keys = keys or beets.config['sunique']['keys'].as_str() - disam = disam or beets.config['sunique']['disambiguators'].as_str() + if skip_item(db_item): + self.lib._memotable[memokey] = '' + return '' + + keys = keys or beets.config[name]['keys'].as_str() + disam = disam or beets.config[name]['disambiguators'].as_str() if bracket is None: - bracket = beets.config['sunique']['bracket'].as_str() + bracket = beets.config[name]['bracket'].as_str() keys = keys.split() disam = disam.split() @@ -1797,36 +1777,35 @@ class DefaultTemplateFunctions: bracket_l = '' bracket_r = '' - if album_id is not None: - # Do nothing for non singletons. - self.lib._memotable[memokey] = '' - return '' - - # Find matching singletons to disambiguate with. - subqueries = [dbcore.query.NoneQuery('album_id', True)] - item_keys = Item.all_keys() + # Find matching items to disambiguate with. + subqueries = [] + if initial_subqueries is not None: + subqueries.extend(initial_subqueries) for key in keys: - value = self.item.get(key, '') + value = db_item.get(key, '') # Use slow queries for flexible attributes. fast = key in item_keys subqueries.append(dbcore.MatchQuery(key, value, fast)) - singletons = self.lib.items(dbcore.AndQuery(subqueries)) + query = dbcore.AndQuery(subqueries) + ambigous_items = (self.lib.items(query) + if isinstance(db_item, Item) + else self.lib.albums(query)) - # If there's only one singleton to matching these details, then do + # If there's only one item to matching these details, then do # nothing. - if len(singletons) == 1: + if len(ambigous_items) == 1: self.lib._memotable[memokey] = '' return '' - # Find the first disambiguator that distinguishes the singletons. + # Find the first disambiguator that distinguishes the items. for disambiguator in disam: - # Get the value for each singleton for the current field. - disam_values = {s.get(disambiguator, '') for s in singletons} + # Get the value for each item for the current field. + disam_values = {s.get(disambiguator, '') for s in ambigous_items} # If the set of unique values is equal to the number of - # singletons in the disambiguation set, we're done -- this is + # items in the disambiguation set, we're done -- this is # sufficient disambiguation. - if len(disam_values) == len(singletons): + if len(disam_values) == len(ambigous_items): break else: # No disambiguator distinguished all fields. @@ -1835,7 +1814,7 @@ class DefaultTemplateFunctions: return res # Flatten disambiguation value into a string. - disam_value = self.item.formatted(for_path=True).get(disambiguator) + disam_value = db_item.formatted(for_path=True).get(disambiguator) # Return empty string if disambiguator is empty. if disam_value: From 6aa9804c24400dd654b88cdb1bf687f652bca581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 17 Aug 2022 17:03:16 +0200 Subject: [PATCH 352/357] Document the %sunique template --- docs/changelog.rst | 2 ++ docs/reference/config.rst | 17 +++++++++++++++++ docs/reference/pathformat.rst | 14 ++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31861af24..d21a55d37 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,8 @@ New features: * :ref:`import-options`: Add support for re-running the importer on paths in log files that were created with the ``-l`` (or ``--logfile``) argument. :bug:`4379` :bug:`4387` +* Add :ref:`%sunique{} ` template to disambiguate between singletons. + :bug:`4438` Bug fixes: diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 6e7df1b59..58656256f 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -326,6 +326,23 @@ The defaults look like this:: See :ref:`aunique` for more details. +.. _config-sunique: + +sunique +~~~~~~~ + +These options are used to generate a string that is guaranteed to be unique +among all singletons in the library who share the same set of keys. + +The defaults look like this:: + + sunique: + keys: artist title + disambiguators: year trackdisambig + bracket: '[]' + +See :ref:`sunique` for more details. + .. _terminal_encoding: diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index f6f2e06cc..b52c2b32a 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -73,6 +73,8 @@ These functions are built in to beets: option. * ``%aunique{identifiers,disambiguators,brackets}``: Provides a unique string to disambiguate similar albums in the database. See :ref:`aunique`, below. +* ``%sunique{identifiers,disambiguators,brackets}``: Provides a unique string + to disambiguate similar singletons in the database. See :ref:`sunique`, below. * ``%time{date_time,format}``: Return the date and time in any format accepted by `strftime`_. For example, to get the year some music was added to your library, use ``%time{$added,%Y}``. @@ -145,6 +147,18 @@ its import time. Only the second album will receive a disambiguation string. If you want to add the disambiguation string to both albums, just run ``beet move`` (possibly restricted by a query) to update the paths for the albums. +.. _sunique: + +Singleton Disambiguation +------------------------ + +It is also possible to have singleton tracks with the same name and the same +artist. Beets provides the ``%sunique{}`` template to avoid having the same +file path. + +It has the same arguments as the :ref:`%aunique ` template, but the default +values are different. The default identifiers are ``artist title`` and the +default disambiguators are ``year trackdisambig``. Syntax Details -------------- From e995019edd68329c28647d651c7766dafeace1a4 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Aug 2022 15:55:25 -0700 Subject: [PATCH 353/357] Doc tweaks for #4438 --- docs/reference/config.rst | 6 ++++-- docs/reference/pathformat.rst | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 58656256f..26a07e643 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -326,13 +326,15 @@ The defaults look like this:: See :ref:`aunique` for more details. + .. _config-sunique: sunique ~~~~~~~ -These options are used to generate a string that is guaranteed to be unique -among all singletons in the library who share the same set of keys. +Like :ref:`config-aunique` above for albums, these options control the +generation of a unique string to disambiguate *singletons* that share similar +metadata. The defaults look like this:: diff --git a/docs/reference/pathformat.rst b/docs/reference/pathformat.rst index b52c2b32a..7c52a92eb 100644 --- a/docs/reference/pathformat.rst +++ b/docs/reference/pathformat.rst @@ -73,7 +73,7 @@ These functions are built in to beets: option. * ``%aunique{identifiers,disambiguators,brackets}``: Provides a unique string to disambiguate similar albums in the database. See :ref:`aunique`, below. -* ``%sunique{identifiers,disambiguators,brackets}``: Provides a unique string +* ``%sunique{identifiers,disambiguators,brackets}``: Similarly, a unique string to disambiguate similar singletons in the database. See :ref:`sunique`, below. * ``%time{date_time,format}``: Return the date and time in any format accepted by `strftime`_. For example, to get the year some music was added to your @@ -153,8 +153,8 @@ Singleton Disambiguation ------------------------ It is also possible to have singleton tracks with the same name and the same -artist. Beets provides the ``%sunique{}`` template to avoid having the same -file path. +artist. Beets provides the ``%sunique{}`` template to avoid giving these +tracks the same file path. It has the same arguments as the :ref:`%aunique ` template, but the default values are different. The default identifiers are ``artist title`` and the From f71e503f6cc006cf702fd30539b22f899342141d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Aug 2022 16:05:33 -0700 Subject: [PATCH 354/357] Change the prefix for exact match queries PR #4251 added exact match queries, which are great, but it was subsequently pointed out that the `~` query prefix was already in use: https://github.com/beetbox/beets/pull/4251#issuecomment-1069455483 So this changes the prefix from `~` to `=~`. A little longer, but hopefully it makes the relationship to the similarly-new `=` prefix obvious. --- beets/library.py | 2 +- docs/changelog.rst | 4 +++- docs/reference/query.rst | 14 ++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/beets/library.py b/beets/library.py index 3b8a85685..c754eaa01 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1387,7 +1387,7 @@ def parse_query_parts(parts, model_cls): # Get query types and their prefix characters. prefixes = { ':': dbcore.query.RegexpQuery, - '~': dbcore.query.StringQuery, + '=~': dbcore.query.StringQuery, '=': dbcore.query.MatchQuery, } prefixes.update(plugins.queries()) diff --git a/docs/changelog.rst b/docs/changelog.rst index d21a55d37..9363ee250 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,7 +25,9 @@ New features: * :doc:`/plugins/kodiupdate`: Now supports multiple kodi instances :bug:`4101` * Add the item fields ``bitrate_mode``, ``encoder_info`` and ``encoder_settings``. -* Add query prefixes ``=`` and ``~``. +* Add :ref:`exact match ` queries, using the prefixes ``=`` and + ``=~``. + :bug:`4251` * :doc:`/plugins/discogs`: Permit appending style to genre * :doc:`/plugins/convert`: Add a new `auto_keep` option that automatically converts files but keeps the *originals* in the library. diff --git a/docs/reference/query.rst b/docs/reference/query.rst index 75fac3015..955bdf57d 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -93,15 +93,17 @@ backslashes are not part of beets' syntax; I'm just using the escaping functionality of my shell (bash or zsh, for instance) to pass ``the rebel`` as a single argument instead of two. +.. _exact-match: + Exact Matches ------------- While ordinary queries perform *substring* matches, beets can also match whole -strings by adding either ``=`` (case-sensitive) or ``~`` (ignore case) after the -field name's colon and before the expression:: +strings by adding either ``=`` (case-sensitive) or ``=~`` (ignore case) after +the field name's colon and before the expression:: $ beet list artist:air - $ beet list artist:~air + $ beet list artist:=~air $ beet list artist:=AIR The first query is a simple substring one that returns tracks by Air, AIR, and @@ -112,16 +114,16 @@ returns tracks by AIR only. Exact matches may be performed on phrases as well:: - $ beet list artist:~"dave matthews" + $ beet list artist:=~"dave matthews" $ beet list artist:="Dave Matthews" Both of these queries return tracks by Dave Matthews, but not by Dave Matthews Band. To search for exact matches across *all* fields, just prefix the expression with -a single ``=`` or ``~``:: +a single ``=`` or ``=~``:: - $ beet list ~crash + $ beet list =~crash $ beet list ="American Football" .. _regex: From 495c8accc07041718914b085884fd11e76758f65 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Aug 2022 16:11:16 -0700 Subject: [PATCH 355/357] Update exact query prefix tests --- test/test_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 8a9043fa3..3a8a7844d 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -150,7 +150,7 @@ class GetTest(DummyDataTestCase): self.assert_items_matched(results, ['beets 4 eva']) def test_get_one_keyed_exact_nocase(self): - q = 'genre:~"hard rock"' + q = 'genre:=~"hard rock"' results = self.lib.items(q) self.assert_items_matched(results, ['beets 4 eva']) @@ -220,7 +220,7 @@ class GetTest(DummyDataTestCase): self.assert_items_matched(results, ['beets 4 eva']) def test_keyed_matches_exact_nocase(self): - q = 'genre:~rock' + q = 'genre:=~rock' results = self.lib.items(q) self.assert_items_matched(results, [ 'foo bar', From 32ce44f589e231b89323a4ff5e0847026e830d5f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Wed, 17 Aug 2022 16:25:17 -0700 Subject: [PATCH 356/357] One more test fix --- test/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_query.py b/test/test_query.py index 3a8a7844d..3c6d6f70a 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -170,7 +170,7 @@ class GetTest(DummyDataTestCase): self.assert_items_matched(results, ['foo bar']) def test_get_one_unkeyed_exact_nocase(self): - q = '~"hard rock"' + q = '=~"hard rock"' results = self.lib.items(q) self.assert_items_matched(results, ['beets 4 eva']) From 93725c454dd0b122a1d3f1e83a6a4a2e20f82847 Mon Sep 17 00:00:00 2001 From: Sacha Bron Date: Sat, 20 Aug 2022 01:30:38 +0200 Subject: [PATCH 357/357] Add Beetstream in the plugin list --- docs/plugins/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 0e9a95136..dedf8d404 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -287,6 +287,8 @@ Here are a few of the plugins written by the beets community: * `beetcamp`_ enables **bandcamp.com** autotagger with a fairly extensive amount of metadata. +* `beetstream`_ is server implementation of the `SubSonic API`_ specification, allowing you to stream your music on a multitude of clients. + * `beets-bpmanalyser`_ analyses songs and calculates their tempo (BPM). * `beets-check`_ automatically checksums your files to detect corruption. @@ -341,6 +343,8 @@ Here are a few of the plugins written by the beets community: .. _beets-barcode: https://github.com/8h2a/beets-barcode .. _beetcamp: https://github.com/snejus/beetcamp +.. _beetstream: https://github.com/BinaryBrain/Beetstream +.. _SubSonic API: http://www.subsonic.org/pages/api.jsp .. _beets-check: https://github.com/geigerzaehler/beets-check .. _beets-copyartifacts: https://github.com/adammillerio/beets-copyartifacts .. _dsedivec: https://github.com/dsedivec/beets-plugins