From 0cd102b0ddaa3244682cfaa6e20745b891b593ad Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Tue, 28 Jul 2020 21:26:12 -0500 Subject: [PATCH 01/65] Update pytest test writing restriction I think the basics sections is fairly self explanatory at this point especially with the copious amounts of examples we have. Also, if we kept it, we'd have to expand on pytest basics as well. I'd rather just point to the docs/getting started guides for each. --- CONTRIBUTING.rst | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dc8617712..e839599d4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -322,9 +322,8 @@ Writing Tests Writing tests is done by adding or modifying files in folder `test`_. Take a look at `https://github.com/beetbox/beets/blob/master/test/test_template.py#L224`_ -to get a basic view on how tests are written. Despite using ``pytest`` -as a test runner, we prefer to write tests using the standard -`unittest`_ testing framework. +to get a basic view on how tests are written. We currently allow writing +tests with either `unittest`_ or `pytest`_. Any tests that involve sending out network traffic e.g. an external API call, should be skipped normally and run under our weekly `integration @@ -342,20 +341,6 @@ If you do this, it is also advised to create a similar test that 'mocks' the network call and can be run under normal circumstances by our CI and others. See `unittest.mock`_ for more info. -Basics -^^^^^^ - -- Your file should contain a class derived from unittest.TestCase -- Each method in this class which name starts with the letters *test* - will be executed to test functionality -- Errors are raised with these methods: - - - ``self.assertEqual`` - - ``self.assertTrue`` - - ``self.assertFalse`` - - ``self.assertRaises`` - -- For detailed information see `Python unittest`_ - **AVOID** using the ``start()`` and ``stop()`` methods of ``mock.patch``, as they require manual cleanup. Use the annotation or context manager forms instead. From 019055c156de1b9d740caa306f2da0376051afe5 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Sun, 19 Jul 2020 12:17:11 -0700 Subject: [PATCH 02/65] add docstring checks to flake8 currently ignore all errors on a per-file basis --- setup.cfg | 178 +++++++++++++++++++++++++++++++++++++++++++++++++++++- setup.py | 1 + 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3f5fb0c57..c87113538 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,4 +24,180 @@ ignore = FI15, # `__future__` import "generator_stop" missing FI50, # `__future__` import "division" present FI51, # `__future__` import "absolute_import" present - FI53, # `__future__` import "print_function" present \ No newline at end of file + FI53, # `__future__` import "print_function" present +per-file-ignores = + ./beet:D + ./docs/conf.py:D + ./extra/release.py:D + ./beetsplug/duplicates.py:D + ./beetsplug/bpm.py:D + ./beetsplug/convert.py:D + ./beetsplug/info.py:D + ./beetsplug/parentwork.py:D + ./beetsplug/deezer.py:D + ./beetsplug/smartplaylist.py:D + ./beetsplug/absubmit.py:D + ./beetsplug/subsonicupdate.py:D + ./beetsplug/fromfilename.py:D + ./beetsplug/mpdstats.py:D + ./beetsplug/gmusic.py:D + ./beetsplug/subsonicplaylist.py:D + ./beetsplug/rewrite.py:D + ./beetsplug/hook.py:D + ./beetsplug/playlist.py:D + ./beetsplug/ftintitle.py:D + ./beetsplug/bpd/gstplayer.py:D + ./beetsplug/bpd/__init__.py:D + ./beetsplug/scrub.py:D + ./beetsplug/sonosupdate.py:D + ./beetsplug/embyupdate.py:D + ./beetsplug/plexupdate.py:D + ./beetsplug/mbsync.py:D + ./beetsplug/lyrics.py:D + ./beetsplug/inline.py:D + ./beetsplug/freedesktop.py:D + ./beetsplug/acousticbrainz.py:D + ./beetsplug/beatport.py:D + ./beetsplug/cue.py:D + ./beetsplug/thumbnails.py:D + ./beetsplug/random.py:D + ./beetsplug/loadext.py:D + ./beetsplug/replaygain.py:D + ./beetsplug/export.py:D + ./beetsplug/fuzzy.py:D + ./beetsplug/importadded.py:D + ./beetsplug/web/__init__.py:D + ./beetsplug/bucket.py:D + ./beetsplug/the.py:D + ./beetsplug/ihate.py:D + ./beetsplug/bench.py:D + ./beetsplug/permissions.py:D + ./beetsplug/spotify.py:D + ./beetsplug/lastgenre/__init__.py:D + ./beetsplug/mbcollection.py:D + ./beetsplug/metasync/amarok.py:D + ./beetsplug/metasync/itunes.py:D + ./beetsplug/metasync/__init__.py:D + ./beetsplug/importfeeds.py:D + ./beetsplug/kodiupdate.py:D + ./beetsplug/zero.py:D + ./beetsplug/bpsync.py:D + ./beetsplug/__init__.py:D + ./beetsplug/edit.py:D + ./beetsplug/types.py:D + ./beetsplug/embedart.py:D + ./beetsplug/mpdupdate.py:D + ./beetsplug/ipfs.py:D + ./beetsplug/discogs.py:D + ./beetsplug/chroma.py:D + ./beetsplug/fish.py:D + ./beetsplug/missing.py:D + ./beetsplug/fetchart.py:D + ./beetsplug/mbsubmit.py:D + ./beetsplug/filefilter.py:D + ./beetsplug/badfiles.py:D + ./beetsplug/play.py:D + ./beetsplug/keyfinder.py:D + ./beetsplug/unimported.py:D + ./beetsplug/lastimport.py:D + ./test/test_parentwork.py:D + ./test/test_hook.py:D + ./test/test_keyfinder.py:D + ./test/test_util.py:D + ./test/test_plexupdate.py:D + ./test/test_importfeeds.py:D + ./test/test_discogs.py:D + ./test/test_acousticbrainz.py:D + ./test/test_pipeline.py:D + ./test/test_mb.py:D + ./test/test_playlist.py:D + ./test/helper.py:D + ./test/test_player.py:D + ./test/test_template.py:D + ./test/test_web.py:D + ./test/test_replaygain.py:D + ./test/test_hidden.py:D + ./test/test_info.py:D + ./test/test_dbcore.py:D + ./test/test_vfs.py:D + ./test/test_subsonic.py:D + ./test/test_play.py:D + ./test/test_types_plugin.py:D + ./test/test_plugins.py:D + ./test/test_importer.py:D + ./test/test_smartplaylist.py:D + ./test/test_spotify.py:D + ./test/test_metasync.py:D + ./test/test_bucket.py:D + ./test/test_ftintitle.py:D + ./test/lyrics_download_samples.py:D + ./test/test_convert.py:D + ./test/test_mbsubmit.py:D + ./test/testall.py:D + ./test/test_fetchart.py:D + ./test/test_ui_importer.py:D + ./test/test_mbsync.py:D + ./test/test_art.py:D + ./test/test_permissions.py:D + ./test/test_embedart.py:D + ./test/test_the.py:D + ./test/test_export.py:D + ./test/rsrc/beetsplug/test.py:D + ./test/rsrc/convert_stub.py:D + ./test/test_ui_init.py:D + ./test/test_filefilter.py:D + ./test/test_logging.py:D + ./test/test_thumbnails.py:D + ./test/test_ipfs.py:D + ./test/test_autotag.py:D + ./test/__init__.py:D + ./test/test_plugin_mediafield.py:D + ./test/test_files.py:D + ./test/test_lastgenre.py:D + ./test/_common.py:D + ./test/test_zero.py:D + ./test/test_edit.py:D + ./test/test_ihate.py:D + ./test/test_ui.py:D + ./test/test_mpdstats.py:D + ./test/test_importadded.py:D + ./test/test_query.py:D + ./test/test_sort.py:D + ./test/test_library.py:D + ./test/test_ui_commands.py:D + ./test/test_lyrics.py:D + ./test/test_beatport.py:D + ./test/test_random.py:D + ./test/test_embyupdate.py:D + ./test/test_datequery.py:D + ./test/test_config_command.py:D + ./setup.py:D + ./beets/ui/__init__.py:D + ./beets/ui/commands.py:D + ./beets/autotag/mb.py:D + ./beets/autotag/hooks.py:D + ./beets/autotag/__init__.py:D + ./beets/autotag/match.py:D + ./beets/__main__.py:D + ./beets/importer.py:D + ./beets/plugins.py:D + ./beets/util/bluelet.py:D + ./beets/util/enumeration.py:D + ./beets/util/artresizer.py:D + ./beets/util/functemplate.py:D + ./beets/util/confit.py:D + ./beets/util/pipeline.py:D + ./beets/util/hidden.py:D + ./beets/util/__init__.py:D + ./beets/library.py:D + ./beets/random.py:D + ./beets/art.py:D + ./beets/logging.py:D + ./beets/vfs.py:D + ./beets/__init__.py:D + ./beets/dbcore/query.py:D + ./beets/dbcore/db.py:D + ./beets/dbcore/__init__.py:D + ./beets/dbcore/queryparse.py:D + ./beets/dbcore/types.py:D + ./beets/mediafile.py:D diff --git a/setup.py b/setup.py index 0e2cb332a..520fc567f 100755 --- a/setup.py +++ b/setup.py @@ -131,6 +131,7 @@ setup( 'flake8', 'flake8-blind-except', 'flake8-coding', + 'flake8-docstrings', 'flake8-future-import', 'pep8-naming', ], From 02fcbea6767fd4dfdd540f679dd4f0191fae3ef4 Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Wed, 22 Jul 2020 17:04:35 -0700 Subject: [PATCH 03/65] add google docstring convention to CONTRIBUTING.rst --- CONTRIBUTING.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e839599d4..d86c490b9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -215,9 +215,13 @@ There are a few coding conventions we use in beets: Style ----- -We follow `PEP 8 `__ for -style. You can use ``tox -e lint`` to check your code for any style -errors. + +We follow `PEP 8`_ and `google's docstring format`_. + +You can use ``tox -e lint`` to check your code for any style errors. + +.. _PEP 8: https://www.python.org/dev/peps/pep-0008/ +.. _google's docstring format: https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings Handling Paths -------------- From 4e9346942136d86b2250b9299bcb6e616a57c41f Mon Sep 17 00:00:00 2001 From: Jacob Pavlock Date: Wed, 29 Jul 2020 20:32:53 -0700 Subject: [PATCH 04/65] fix missing docstring-convention declaration --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index c87113538..761ab1de3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] min-version = 2.7 accept-encodings = utf-8 +docstring-convention = google # errors we ignore; see https://www.flake8rules.com/ for more info ignore = # pycodestyle errors From 48d39ea11ecf17d13b035e9fb975d5ce526575cc Mon Sep 17 00:00:00 2001 From: Gunther Schmidl Date: Thu, 30 Jul 2020 22:42:31 +0200 Subject: [PATCH 05/65] fix regex, add test and changelog entry --- beetsplug/the.py | 2 +- docs/changelog.rst | 3 +++ test/test_the.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/beetsplug/the.py b/beetsplug/the.py index 238aec32f..dfc58817d 100644 --- a/beetsplug/the.py +++ b/beetsplug/the.py @@ -23,7 +23,7 @@ from beets.plugins import BeetsPlugin __author__ = 'baobab@heresiarch.info' __version__ = '1.1' -PATTERN_THE = u'^[the]{3}\\s' +PATTERN_THE = u'^the\\s' PATTERN_A = u'^[a][n]?\\s' FORMAT = u'{0}, {1}' diff --git a/docs/changelog.rst b/docs/changelog.rst index 64e6ab85c..4a87f7f07 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -145,6 +145,9 @@ New features: Fixes: +* :doc:`/plugins/the`: Fixed incorrect regex for 'the' that matched any + 3-letter combination of the letters t, h, e. + :bug:`3701` * :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take environment variables such as proxy servers into account when making requests :bug:`3450` diff --git a/test/test_the.py b/test/test_the.py index 263446b92..1fc488953 100644 --- a/test/test_the.py +++ b/test/test_the.py @@ -36,6 +36,8 @@ class ThePluginTest(_common.TestCase): u'A Thing, An') self.assertEqual(ThePlugin().unthe(u'the An Arse', PATTERN_A), u'the An Arse') + self.assertEqual(ThePlugin().unthe(u'TET - Travailleur', PATTERN_THE), + u'TET - Travailleur') def test_unthe_with_strip(self): config['the']['strip'] = True From 9291d9c30417d5072d217e0bc4894348de057c11 Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Sat, 1 Aug 2020 14:35:53 -0700 Subject: [PATCH 06/65] Fix rarfile 4.0 dependency conflict (#3711) --- setup.py | 9 ++++++--- tox.ini | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 520fc567f..2c3cb2b55 100755 --- a/setup.py +++ b/setup.py @@ -120,13 +120,14 @@ setup( 'pytest', 'python-mpd2', 'pyxdg', - 'rarfile', 'responses>=0.3.0', 'requests_oauthlib', ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] - ), + ) + [ + 'rarfile<4' if sys.version_info < (3, 6, 0) else 'rarfile', + ], 'lint': [ 'flake8', 'flake8-blind-except', @@ -152,7 +153,9 @@ setup( 'mpdstats': ['python-mpd2>=0.4.2'], 'plexupdate': ['requests'], 'web': ['flask', 'flask-cors'], - 'import': ['rarfile'], + 'import': ( + ['rarfile<4' if (sys.version_info < (3, 6, 0)) else 'rarfile'] + ), 'thumbnails': ['pyxdg', 'Pillow'] + (['pathlib'] if (sys.version_info < (3, 4, 0)) else []), 'metasync': ['dbus-python'], diff --git a/tox.ini b/tox.ini index e3476db1c..bb7419dbc 100644 --- a/tox.ini +++ b/tox.ini @@ -30,4 +30,4 @@ commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} [testenv:int] deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 -commands = python -bb -m pytest {posargs} \ No newline at end of file +commands = python -bb -m pytest {posargs} From c9f9f9691d1ec7905049bbe76d93eb57e7f8e0a5 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 14:59:21 +0100 Subject: [PATCH 07/65] Notify Zulip chat on integration test failure --- .github/workflows/integration_test.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1e8b3a773..5c023e933 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -25,3 +25,20 @@ jobs: - name: Test with tox run: | tox -e int + + - name: Notify on failure + if: ${{ failure() }} + env: + ZULIP_BOT_CREDENTIALS: ${{ secrets.ZULIP_BOT_CREDENTIALS }} + run: | + if [ -z "${ZULIP_BOT_CREDENTIALS}" ]; then + echo "Skipping notify, ZULIP_BOT_CREDENTIALS is unset" + exit 0 + fi + + curl -X POST https://beets.zulipchat.com/api/v1/messages \ + -u "${ZULIP_BOT_CREDENTIALS}" \ + -d "type=stream" \ + -d "to=general" \ + -d "subject=${GITHUB_WORKFLOW} - $(date -u +%Y-%m-%d)" \ + -d "content=[${GITHUB_WORKFLOW}#${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) failed." From 5f44c3147dc3bcf67af0afc162156b7f3d3d6abd Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 15:37:01 +0100 Subject: [PATCH 08/65] Switch to #github for integration test messages --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 5c023e933..0e5efe793 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -39,6 +39,6 @@ jobs: curl -X POST https://beets.zulipchat.com/api/v1/messages \ -u "${ZULIP_BOT_CREDENTIALS}" \ -d "type=stream" \ - -d "to=general" \ + -d "to=github" \ -d "subject=${GITHUB_WORKFLOW} - $(date -u +%Y-%m-%d)" \ -d "content=[${GITHUB_WORKFLOW}#${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) failed." From c7859ca9c69bfd6fb75462dd512f3b7b4dc09380 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 15:51:07 +0100 Subject: [PATCH 09/65] Add worfklow_dispatch trigger This change allows us to trigger integration tests manually. --- .github/workflows/integration_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1e8b3a773..209be0b93 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -1,5 +1,6 @@ name: integration tests on: + workflow_dispatch: schedule: - cron: '0 0 * * SUN' # run every Sunday at midnight jobs: From 92f425628d4b8659bee42925dd7766b243538676 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 16:55:51 +0100 Subject: [PATCH 10/65] Skip Genius integration test on GitHub actions --- test/test_lyrics.py | 7 ++++++- tox.ini | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 11006348e..e0ec1e548 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -330,7 +330,12 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): """Test default backends with songs known to exist in respective databases. """ errors = [] - for s in self.DEFAULT_SOURCES: + # GitHub actions seems to be on a Cloudflare blacklist, so we can't + # contact genius. + sources = [s for s in self.DEFAULT_SOURCES if + s['backend'] != lyrics.Genius or + os.environ.get('GITHUB_ACTIONS') != 'true'] + for s in sources: res = s['backend'](self.plugin.config, self.plugin._log).fetch( s['artist'], s['title']) if not is_lyrics_content_ok(s['title'], res): diff --git a/tox.ini b/tox.ini index bb7419dbc..cbf953033 100644 --- a/tox.ini +++ b/tox.ini @@ -30,4 +30,5 @@ commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} [testenv:int] deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 +passenv = GITHUB_ACTIONS commands = python -bb -m pytest {posargs} From 20a228c21b7fdaa909343f69019c000f727507b0 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 21:54:20 +0100 Subject: [PATCH 11/65] Add flake8 problem matcher --- .github/flake8-problem-matcher.json | 17 +++++++++++++++++ .github/workflows/ci.yaml | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 .github/flake8-problem-matcher.json diff --git a/.github/flake8-problem-matcher.json b/.github/flake8-problem-matcher.json new file mode 100644 index 000000000..bb8db292c --- /dev/null +++ b/.github/flake8-problem-matcher.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "flake8", + "pattern": [ + { + "regexp": "^(.*?):(\\d+):(\\d+): ([^ ]+) (.*)$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ] + } + ] +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 08e7548f2..48d43d059 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,9 +65,6 @@ jobs: lint: runs-on: ubuntu-latest - env: - PY_COLORS: 1 - steps: - uses: actions/checkout@v2 @@ -81,5 +78,8 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx + - name: Add problem matcher + run: echo "::add-matcher::.github/flake8-problem-matcher.json" + - name: Lint with flake8 run: tox -e py-lint From 5d24cb0e1dded59960ea59a3191f21b4f0597ecc Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 2 Aug 2020 22:41:01 +0100 Subject: [PATCH 12/65] Include error code in message --- .github/flake8-problem-matcher.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/flake8-problem-matcher.json b/.github/flake8-problem-matcher.json index bb8db292c..52f94e05e 100644 --- a/.github/flake8-problem-matcher.json +++ b/.github/flake8-problem-matcher.json @@ -4,12 +4,11 @@ "owner": "flake8", "pattern": [ { - "regexp": "^(.*?):(\\d+):(\\d+): ([^ ]+) (.*)$", + "regexp": "^(.*?):(\\d+):(\\d+): (.*)$", "file": 1, "line": 2, "column": 3, - "code": 4, - "message": 5 + "message": 4 } ] } From 490fc0751600c02659dd2c11c2f174b8bdec0ad2 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 3 Aug 2020 13:10:21 +0100 Subject: [PATCH 13/65] Add sphinx problem matcher --- .github/sphinx-problem-matcher.json | 18 ++++++++++++++++++ .github/workflows/ci.yaml | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 .github/sphinx-problem-matcher.json diff --git a/.github/sphinx-problem-matcher.json b/.github/sphinx-problem-matcher.json new file mode 100644 index 000000000..d33eb09b6 --- /dev/null +++ b/.github/sphinx-problem-matcher.json @@ -0,0 +1,18 @@ +{ + "problemMatcher": [ + { + "owner": "sphinx", + "pattern": [ + { + "regexp": "^Warning, treated as error:$" + }, + { + "regexp": "^(.*?):(\\d+):(.*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48d43d059..ecb7e03dd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,6 +59,9 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx + - name: Add problem matcher + run: echo "::add-matcher::.github/sphinx-problem-matcher.json" + - name: Build and check docs using tox run: tox -e docs From b89a2650cc8728d21a8aaa48b82e9b24564e44b0 Mon Sep 17 00:00:00 2001 From: Samuel Cook Date: Mon, 3 Aug 2020 18:20:20 -0700 Subject: [PATCH 14/65] Delete after convert (#3700) * If import move is true, files will be deleted after converting. Fixes #2947 * Removed trailing whitespace to comply with W293, fixing build * Add period to the end of the comment Co-Authored-By: Adrian Sampson * Added changelog entry for this fix. * Added delete_originals option to remove source files after transcode * Added unit test, removed redundant syspath call Co-authored-by: Logan Arens Co-authored-by: Logan Arens Co-authored-by: Adrian Sampson Co-authored-by: Logan Arens --- beetsplug/convert.py | 6 ++++++ docs/changelog.rst | 4 ++++ docs/plugins/convert.rst | 2 ++ test/test_convert.py | 10 ++++++++++ 4 files changed, 22 insertions(+) diff --git a/beetsplug/convert.py b/beetsplug/convert.py index e7ac4f3ac..70363f6eb 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -148,6 +148,7 @@ class ConvertPlugin(BeetsPlugin): u'never_convert_lossy_files': False, u'copy_album_art': False, u'album_art_maxwidth': 0, + u'delete_originals': False, }) self.early_import_stages = [self.auto_convert] @@ -532,11 +533,16 @@ class ConvertPlugin(BeetsPlugin): # Change the newly-imported database entry to point to the # converted file. + source_path = item.path item.path = dest item.write() item.read() # Load new audio information data. item.store() + if self.config['delete_originals']: + self._log.info(u'Removing original file {0}', source_path) + util.remove(source_path, False) + def _cleanup(self, task, session): for path in task.old_paths: if path in _temp_files: diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a87f7f07..0dfa2c5f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -142,6 +142,10 @@ New features: * :doc:`/plugins/thumbnails`: Fix a bug where pathlib expected a string instead of bytes for a path. :bug:`3360` +* :doc:`/plugins/convert`: If ``delete_originals`` is enabled, then the source files will + be deleted after importing. + Thanks to :user:`logan-arens`. + :bug:`2947` Fixes: diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 6e9d00a11..9581e24a4 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -111,6 +111,8 @@ file. The available options are: This option overrides ``link``. Only works when converting to a directory on the same filesystem as the library. Default: ``false``. +- **delete_originals**: Transcoded files will be copied or moved to their destination, depending on the import configuration. By default, the original files are not modified by the plugin. This option deletes the original files after the transcoding step has completed. + Default: ``false``. You can also configure the format to use for transcoding (see the next section): diff --git a/test/test_convert.py b/test/test_convert.py index 33bdb3b24..b8cd56741 100644 --- a/test/test_convert.py +++ b/test/test_convert.py @@ -15,6 +15,7 @@ from __future__ import division, absolute_import, print_function +import fnmatch import sys import re import os.path @@ -121,6 +122,15 @@ class ImportConvertTest(unittest.TestCase, TestHelper): self.assertIsNotNone(item) self.assertTrue(os.path.isfile(item.path)) + def test_delete_originals(self): + self.config['convert']['delete_originals'] = True + self.importer.run() + for path in self.importer.paths: + for root, dirnames, filenames in os.walk(path): + self.assertTrue(len(fnmatch.filter(filenames, '*.mp3')) == 0, + u'Non-empty import directory {0}' + .format(util.displayable_path(path))) + class ConvertCommand(object): """A mixin providing a utility method to run the `convert`command From fce27e6fa950d1846ccc9ba99640d61c98299f62 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Sun, 9 Aug 2020 19:12:58 -0700 Subject: [PATCH 15/65] mpdstats: Don't record a skip when stopping MPD. MPD keeps the current track in the queue when stopping, so it's not really like a skip, and I use it so that I can stop the music, and later start at the beginning of a track. I do this by keeping track of the current song id, and then comparing them when we receive a stop signal. --- beetsplug/mpdstats.py | 14 +++++++++----- docs/changelog.rst | 4 ++++ test/test_mpdstats.py | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index f232d87e9..39b045f9b 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -108,8 +108,9 @@ class MPDClientWrapper(object): return self.get(command, retries=retries - 1) def currentsong(self): - """Return the path to the currently playing song. Prefixes paths with the - music_directory, to get the absolute path. + """Return the path to the currently playing song, along with its + songid. Prefixes paths with the music_directory, to get the absolute + path. """ result = None entry = self.get('currentsong') @@ -118,7 +119,7 @@ class MPDClientWrapper(object): result = os.path.join(self.music_directory, entry['file']) else: result = entry['file'] - return result + return result, entry.get('id') def status(self): """Return the current status of the MPD. @@ -240,7 +241,9 @@ class MPDStats(object): def on_stop(self, status): self._log.info(u'stop') - if self.now_playing: + # if the current song stays the same it means that we stopped on the + # current track and should not record a skip. + if self.now_playing and self.now_playing['id'] != status.get('songid'): self.handle_song_change(self.now_playing) self.now_playing = None @@ -251,7 +254,7 @@ class MPDStats(object): def on_play(self, status): - path = self.mpd.currentsong() + path, songid = self.mpd.currentsong() if not path: return @@ -286,6 +289,7 @@ class MPDStats(object): 'started': time.time(), 'remaining': remaining, 'path': path, + 'id': songid, 'beets_item': self.get_item(path), } diff --git a/docs/changelog.rst b/docs/changelog.rst index 0dfa2c5f8..47159ddc2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -245,6 +245,10 @@ Fixes: * Fix a bug that caused metadata starting with something resembling a drive letter to be incorrectly split into an extra directory after the colon. :bug:`3685` +* :doc:`/plugins/mpdstats`: Don't record a skip when stopping MPD, as MPD keeps + the current track in the queue. + Thanks to :user:`aereaux`. + :bug:`3722` For plugin developers: diff --git a/test/test_mpdstats.py b/test/test_mpdstats.py index 0117e22aa..20226927f 100644 --- a/test/test_mpdstats.py +++ b/test/test_mpdstats.py @@ -62,10 +62,11 @@ class MPDStatsTest(unittest.TestCase, TestHelper): {'state': u'stop'}] EVENTS = [["player"]] * (len(STATUSES) - 1) + [KeyboardInterrupt] item_path = util.normpath('/foo/bar.flac') + songid = 1 @patch("beetsplug.mpdstats.MPDClientWrapper", return_value=Mock(**{ "events.side_effect": EVENTS, "status.side_effect": STATUSES, - "currentsong.return_value": item_path})) + "currentsong.return_value": (item_path, songid)})) def test_run_mpdstats(self, mpd_mock): item = Item(title=u'title', path=self.item_path, id=1) item.add(self.lib) From 2039f26f964c9cae4b8a91e70fd7b387671a2403 Mon Sep 17 00:00:00 2001 From: PotcFdk Date: Mon, 24 Aug 2020 01:43:17 +0200 Subject: [PATCH 16/65] Update file metadata after generating fingerprintsthrough the `submit` command. --- beetsplug/chroma.py | 2 +- docs/changelog.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/chroma.py b/beetsplug/chroma.py index 54ae90098..20d0f5479 100644 --- a/beetsplug/chroma.py +++ b/beetsplug/chroma.py @@ -279,7 +279,7 @@ def submit_items(log, userkey, items, chunksize=64): del data[:] for item in items: - fp = fingerprint_item(log, item) + fp = fingerprint_item(log, item, write=ui.should_write()) # Construct a submission dictionary for this item. item_data = { diff --git a/docs/changelog.rst b/docs/changelog.rst index 47159ddc2..f4c3f38a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. * A new :ref:`extra_tags` configuration option allows more tagged metadata From 3c8419dbe0dbaa77b7c15a4f83b39508683659d3 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Thu, 3 Sep 2020 22:44:08 -0400 Subject: [PATCH 17/65] fix(plugin): `subsonicupdate` rest call Signed-off-by: Jef LeCompte --- beetsplug/subsonicupdate.py | 28 ++++-- docs/changelog.rst | 2 + test/test_subsonic.py | 111 --------------------- test/test_subsonicupdate.py | 188 ++++++++++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 121 deletions(-) delete mode 100644 test/test_subsonic.py create mode 100644 test/test_subsonicupdate.py diff --git a/beetsplug/subsonicupdate.py b/beetsplug/subsonicupdate.py index 45fc3a8cb..004439bac 100644 --- a/beetsplug/subsonicupdate.py +++ b/beetsplug/subsonicupdate.py @@ -100,16 +100,24 @@ class SubsonicUpdate(BeetsPlugin): 't': token, 's': salt, 'v': '1.15.0', # Subsonic 6.1 and newer. - 'c': 'beets' + 'c': 'beets', + 'f': 'json' } - response = requests.post(url, params=payload) + try: + response = requests.get(url, params=payload) + json = response.json() - if response.status_code == 403: - self._log.error(u'Server authentication failed') - elif response.status_code == 200: - self._log.debug(u'Updating Subsonic') - else: - self._log.error( - u'Generic error, please try again later [Status Code: {}]' - .format(response.status_code)) + if response.status_code == 200 and \ + json['subsonic-response']['status'] == "ok": + count = json['subsonic-response']['scanStatus']['count'] + self._log.info( + u'Updating Subsonic; scanning {0} tracks'.format(count)) + elif response.status_code == 200 and \ + json['subsonic-response']['status'] == "failed": + error_message = json['subsonic-response']['error']['message'] + self._log.error(u'Error: {0}'.format(error_message)) + else: + self._log.error(u'Error: {0}', json) + except Exception as error: + self._log.error(u'Error: {0}'.format(error)) diff --git a/docs/changelog.rst b/docs/changelog.rst index 47159ddc2..c3fe3e556 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -149,6 +149,8 @@ New features: Fixes: +* :doc:`/plugins/subsonicupdate`: REST was using `POST` method rather `GET` method. + Also includes better exception handling, response parsing, and tests. * :doc:`/plugins/the`: Fixed incorrect regex for 'the' that matched any 3-letter combination of the letters t, h, e. :bug:`3701` diff --git a/test/test_subsonic.py b/test/test_subsonic.py deleted file mode 100644 index 6d37cdf4f..000000000 --- a/test/test_subsonic.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- - -"""Tests for the 'subsonic' plugin""" - -from __future__ import division, absolute_import, print_function - -import requests -import responses -import unittest - -from test import _common -from beets import config -from beetsplug import subsonicupdate -from test.helper import TestHelper -from six.moves.urllib.parse import parse_qs, urlparse - - -class ArgumentsMock(object): - def __init__(self, mode, show_failures): - self.mode = mode - self.show_failures = show_failures - self.verbose = 1 - - -def _params(url): - """Get the query parameters from a URL.""" - return parse_qs(urlparse(url).query) - - -class SubsonicPluginTest(_common.TestCase, TestHelper): - @responses.activate - def setUp(self): - config.clear() - self.setup_beets() - - config["subsonic"]["user"] = "admin" - config["subsonic"]["pass"] = "admin" - config["subsonic"]["url"] = "http://localhost:4040" - - self.subsonicupdate = subsonicupdate.SubsonicUpdate() - - def tearDown(self): - self.teardown_beets() - - @responses.activate - def test_start_scan(self): - responses.add( - responses.POST, - 'http://localhost:4040/rest/startScan', - status=200 - ) - - self.subsonicupdate.start_scan() - - @responses.activate - def test_url_with_extra_forward_slash_url(self): - config["subsonic"]["url"] = "http://localhost:4040/contextPath" - - responses.add( - responses.POST, - 'http://localhost:4040/contextPath/rest/startScan', - status=200 - ) - - self.subsonicupdate.start_scan() - - @responses.activate - def test_url_with_context_path(self): - config["subsonic"]["url"] = "http://localhost:4040/" - - responses.add( - responses.POST, - 'http://localhost:4040/rest/startScan', - status=200 - ) - - self.subsonicupdate.start_scan() - - @responses.activate - def test_url_with_missing_port(self): - config["subsonic"]["url"] = "http://localhost/airsonic" - - responses.add( - responses.POST, - 'http://localhost:4040/rest/startScan', - status=200 - ) - - with self.assertRaises(requests.exceptions.ConnectionError): - self.subsonicupdate.start_scan() - - @responses.activate - def test_url_with_missing_schema(self): - config["subsonic"]["url"] = "localhost:4040/airsonic" - - responses.add( - responses.POST, - 'http://localhost:4040/rest/startScan', - status=200 - ) - - with self.assertRaises(requests.exceptions.InvalidSchema): - self.subsonicupdate.start_scan() - - -def suite(): - return unittest.TestLoader().loadTestsFromName(__name__) - - -if __name__ == '__main__': - unittest.main(defaultTest='suite') diff --git a/test/test_subsonicupdate.py b/test/test_subsonicupdate.py new file mode 100644 index 000000000..c47208e65 --- /dev/null +++ b/test/test_subsonicupdate.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +"""Tests for the 'subsonic' plugin.""" + +from __future__ import division, absolute_import, print_function + +import responses +import unittest + +from test import _common +from beets import config +from beetsplug import subsonicupdate +from test.helper import TestHelper +from six.moves.urllib.parse import parse_qs, urlparse + + +class ArgumentsMock(object): + """Argument mocks for tests.""" + def __init__(self, mode, show_failures): + """Constructs ArgumentsMock.""" + self.mode = mode + self.show_failures = show_failures + self.verbose = 1 + + +def _params(url): + """Get the query parameters from a URL.""" + return parse_qs(urlparse(url).query) + + +class SubsonicPluginTest(_common.TestCase, TestHelper): + """Test class for subsonicupdate.""" + @responses.activate + def setUp(self): + """Sets up config and plugin for test.""" + config.clear() + self.setup_beets() + + config["subsonic"]["user"] = "admin" + config["subsonic"]["pass"] = "admin" + config["subsonic"]["url"] = "http://localhost:4040" + + self.subsonicupdate = subsonicupdate.SubsonicUpdate() + + SUCCESS_BODY = ''' +{ + "subsonic-response": { + "status": "ok", + "version": "1.15.0", + "scanStatus": { + "scanning": true, + "count": 1000 + } + } +} +''' + + FAILED_BODY = ''' +{ + "subsonic-response": { + "status": "failed", + "version": "1.15.0", + "error": { + "code": 40, + "message": "Wrong username or password." + } + } +} +''' + + ERROR_BODY = ''' +{ + "timestamp": 1599185854498, + "status": 404, + "error": "Not Found", + "message": "No message available", + "path": "/rest/startScn" +} +''' + + def tearDown(self): + """Tears down tests.""" + self.teardown_beets() + + @responses.activate + def test_start_scan(self): + """Tests success path based on best case scenario.""" + responses.add( + responses.GET, + 'http://localhost:4040/rest/startScan', + status=200, + body=self.SUCCESS_BODY + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_start_scan_failed_bad_credentials(self): + """Tests failed path based on bad credentials.""" + responses.add( + responses.GET, + 'http://localhost:4040/rest/startScan', + status=200, + body=self.FAILED_BODY + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_start_scan_failed_not_found(self): + """Tests failed path based on resource not found.""" + responses.add( + responses.GET, + 'http://localhost:4040/rest/startScan', + status=404, + body=self.ERROR_BODY + ) + + self.subsonicupdate.start_scan() + + def test_start_scan_failed_unreachable(self): + """Tests failed path based on service not available.""" + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_context_path(self): + """Tests success for included with contextPath.""" + config["subsonic"]["url"] = "http://localhost:4040/contextPath/" + + responses.add( + responses.GET, + 'http://localhost:4040/contextPath/rest/startScan', + status=200, + body=self.SUCCESS_BODY + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_trailing_forward_slash_url(self): + """Tests success path based on trailing forward slash.""" + config["subsonic"]["url"] = "http://localhost:4040/" + + responses.add( + responses.GET, + 'http://localhost:4040/rest/startScan', + status=200, + body=self.SUCCESS_BODY + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_missing_port(self): + """Tests failed path based on missing port.""" + config["subsonic"]["url"] = "http://localhost/airsonic" + + responses.add( + responses.GET, + 'http://localhost/airsonic/rest/startScan', + status=200, + body=self.SUCCESS_BODY + ) + + self.subsonicupdate.start_scan() + + @responses.activate + def test_url_with_missing_schema(self): + """Tests failed path based on missing schema.""" + config["subsonic"]["url"] = "localhost:4040/airsonic" + + responses.add( + responses.GET, + 'http://localhost:4040/rest/startScan', + status=200, + body=self.SUCCESS_BODY + ) + + self.subsonicupdate.start_scan() + + +def suite(): + """Default test suite.""" + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From c9f59ee38dec30c9eaad2f7687654c9acaf2f5ac Mon Sep 17 00:00:00 2001 From: jtpavlock Date: Fri, 4 Sep 2020 15:42:36 -0500 Subject: [PATCH 18/65] Add broken link checker to integration test (#3703) * fix broken links * add link check command to tox * add link check to the weekly integration test --- .github/workflows/integration_test.yaml | 4 ++++ CONTRIBUTING.rst | 20 +++++++++++--------- docs/changelog.rst | 20 +++++++++----------- docs/conf.py | 7 +++++++ docs/dev/index.rst | 2 +- docs/dev/library.rst | 4 ++-- docs/dev/plugins.rst | 4 ++-- docs/faq.rst | 14 ++++++++------ docs/guides/main.rst | 2 +- docs/plugins/absubmit.rst | 2 +- docs/plugins/beatport.rst | 4 ++-- docs/plugins/bpd.rst | 9 +++------ docs/plugins/convert.rst | 4 ++-- docs/plugins/embyupdate.rst | 2 +- docs/plugins/keyfinder.rst | 2 +- docs/plugins/kodiupdate.rst | 2 +- docs/plugins/lastgenre.rst | 9 +++------ docs/plugins/lyrics.rst | 7 +++---- docs/plugins/plexupdate.rst | 2 +- docs/plugins/subsonicupdate.rst | 2 +- docs/plugins/web.rst | 12 +++++------- docs/reference/config.rst | 2 +- tox.ini | 6 ++++++ 23 files changed, 76 insertions(+), 66 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 386571e5a..633947fd3 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -27,6 +27,10 @@ jobs: run: | tox -e int + - name: Check external links in docs + run: | + tox -e links + - name: Notify on failure if: ${{ failure() }} env: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d86c490b9..9600ee966 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -28,7 +28,7 @@ Non-Programming - Promote beets! Help get the word out by telling your friends, writing a blog post, or discussing it on a forum you frequent. -- Improve the `documentation `__. It’s +- Improve the `documentation`_. It’s incredibly easy to contribute here: just find a page you want to modify and hit the “Edit on GitHub” button in the upper-right. You can automatically send us a pull request for your changes. @@ -62,7 +62,7 @@ Getting the Source ^^^^^^^^^^^^^^^^^^ The easiest way to get started with the latest beets source is to use -`pip `__ to install an “editable” package. This +`pip`_ to install an “editable” package. This can be done with one command: .. code-block:: bash @@ -147,8 +147,7 @@ request and your code will ship in no time. 5. Add a changelog entry to ``docs/changelog.rst`` near the top of the document. 6. Run the tests and style checker. The easiest way to run the tests is - to use `tox `__. For more - information on running tests, see :ref:`testing`. + to use `tox`_. For more information on running tests, see :ref:`testing`. 7. Push to your fork and open a pull request! We’ll be in touch shortly. 8. If you add commits to a pull request, please add a comment or re-request a review after you push them since GitHub doesn’t @@ -253,7 +252,7 @@ guidelines to follow: Editor Settings --------------- -Personally, I work on beets with `vim `__. Here are +Personally, I work on beets with `vim`_. Here are some ``.vimrc`` lines that might help with PEP 8-compliant Python coding:: @@ -318,7 +317,7 @@ To install the test dependencies, run ``python -m pip install .[test]``. Or, just run a test suite with ``tox`` which will install them automatically. -.. _setup.py: https://github.com/beetbox/beets/blob/master/setup.py#L99` +.. _setup.py: https://github.com/beetbox/beets/blob/master/setup.py Writing Tests ------------- @@ -352,9 +351,9 @@ others. See `unittest.mock`_ for more info. .. _Python unittest: https://docs.python.org/2/library/unittest.html .. _Codecov: https://codecov.io/github/beetbox/beets .. _pytest-random: https://github.com/klrmn/pytest-random -.. _tox: http://tox.readthedocs.org -.. _detox: https://pypi.python.org/pypi/detox/ -.. _pytest: http://pytest.org +.. _tox: https://tox.readthedocs.io/en/latest/ +.. _detox: https://pypi.org/project/detox/ +.. _pytest: https://docs.pytest.org/en/stable/ .. _Linux: https://github.com/beetbox/beets/actions .. _Windows: https://ci.appveyor.com/project/beetbox/beets/ .. _`https://github.com/beetbox/beets/blob/master/setup.py#L99`: https://github.com/beetbox/beets/blob/master/setup.py#L99 @@ -364,3 +363,6 @@ others. See `unittest.mock`_ for more info. .. _integration test: https://github.com/beetbox/beets/actions?query=workflow%3A%22integration+tests%22 .. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html .. _Python unittest: https://docs.python.org/2/library/unittest.html +.. _documentation: https://beets.readthedocs.io/en/stable/ +.. _pip: https://pip.pypa.io/en/stable/ +.. _vim: https://www.vim.org/ diff --git a/docs/changelog.rst b/docs/changelog.rst index eb1236599..bc03d772e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -196,7 +196,7 @@ Fixes: * ``beet update`` will now confirm that the user still wants to update if their library folder cannot be found, preventing the user from accidentally wiping out their beets database. - Thanks to :user:`logan-arens`. + Thanks to user: `logan-arens`. :bug:`1934` * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. Thanks to :user:`aereaux`. @@ -1273,7 +1273,7 @@ And there are a few bug fixes too: The last release, 1.3.19, also erroneously reported its version as "1.3.18" when you typed ``beet version``. This has been corrected. -.. _six: https://pythonhosted.org/six/ +.. _six: https://pypi.org/project/six/ 1.3.19 (June 25, 2016) @@ -2119,7 +2119,7 @@ As usual, there are loads of little fixes and improvements: * The :ref:`config-cmd` command can now use ``$EDITOR`` variables with arguments. -.. _API changes: https://developer.echonest.com/forums/thread/3650 +.. _API changes: https://web.archive.org/web/20160814092627/https://developer.echonest.com/forums/thread/3650 .. _Plex: https://plex.tv/ .. _musixmatch: https://www.musixmatch.com/ @@ -2344,7 +2344,7 @@ The big new features are: * A new :ref:`asciify-paths` configuration option replaces all non-ASCII characters in paths. -.. _Mutagen: https://bitbucket.org/lazka/mutagen +.. _Mutagen: https://github.com/quodlibet/mutagen .. _Spotify: https://www.spotify.com/ And the multitude of little improvements and fixes: @@ -2599,7 +2599,7 @@ Fixes: * :doc:`/plugins/convert`: Display a useful error message when the FFmpeg executable can't be found. -.. _requests: https://www.python-requests.org/ +.. _requests: https://requests.readthedocs.io/en/master/ 1.3.3 (February 26, 2014) @@ -2780,7 +2780,7 @@ As usual, there are also innumerable little fixes and improvements: Bezman. -.. _Acoustic Attributes: http://developer.echonest.com/acoustic-attributes.html +.. _Acoustic Attributes: https://web.archive.org/web/20160701063109/http://developer.echonest.com/acoustic-attributes.html .. _MPD: https://www.musicpd.org/ @@ -3130,7 +3130,7 @@ will automatically migrate your configuration to the new system. header. Thanks to Uwe L. Korn. * :doc:`/plugins/lastgenre`: Fix an error when using genre canonicalization. -.. _Tomahawk: https://tomahawk-player.org/ +.. _Tomahawk: https://github.com/tomahawk-player/tomahawk 1.1b3 (March 16, 2013) ---------------------- @@ -3473,7 +3473,7 @@ begins today on features for version 1.1. * Changed plugin loading so that modules can be imported without unintentionally loading the plugins they contain. -.. _The Echo Nest: http://the.echonest.com/ +.. _The Echo Nest: https://web.archive.org/web/20180329103558/http://the.echonest.com/ .. _Tomahawk resolver: https://beets.io/blog/tomahawk-resolver.html .. _mp3gain: http://mp3gain.sourceforge.net/download.php .. _aacgain: https://aacgain.altosdesign.com @@ -3911,7 +3911,7 @@ plugin. * The :doc:`/plugins/web` encapsulates a simple **Web-based GUI for beets**. The current iteration can browse the library and play music in browsers that - support `HTML5 Audio`_. + support HTML5 Audio. * When moving items that are part of an album, the album art implicitly moves too. @@ -3928,8 +3928,6 @@ plugin. * Fix crash when "copying" an art file that's already in place. -.. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html - 1.0b9 (July 9, 2011) -------------------- diff --git a/docs/conf.py b/docs/conf.py index bb3e3d00f..018ef5397 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,13 @@ extlinks = { 'stdlib': ('https://docs.python.org/3/library/%s.html', ''), } +linkcheck_ignore = [ + r'https://github.com/beetbox/beets/issues/', + r'https://github.com/\w+$', # ignore user pages + r'.*localhost.*', + r'https://www.musixmatch.com/', # blocks requests +] + # Options for HTML output htmlhelp_basename = 'beetsdoc' diff --git a/docs/dev/index.rst b/docs/dev/index.rst index f1465494d..63335160c 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -7,7 +7,7 @@ in hacking beets itself or creating plugins for it. See also the documentation for `MediaFile`_, the library used by beets to read and write metadata tags in media files. -.. _MediaFile: https://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/en/latest/ .. toctree:: diff --git a/docs/dev/library.rst b/docs/dev/library.rst index 77e218b93..071b780f3 100644 --- a/docs/dev/library.rst +++ b/docs/dev/library.rst @@ -45,7 +45,7 @@ responsible for handling queries to retrieve stored objects. .. automethod:: transaction -.. _SQLite: https://sqlite.org/ +.. _SQLite: https://sqlite.org/index.html .. _ORM: https://en.wikipedia.org/wiki/Object-relational_mapping @@ -118,7 +118,7 @@ To make changes to either the database or the tags on a file, you update an item's fields (e.g., ``item.title = "Let It Be"``) and then call ``item.write()``. -.. _MediaFile: https://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/en/latest/ Items also track their modification times (mtimes) to help detect when they become out of sync with on-disk metadata, mainly to speed up the diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 3328654e0..563775fd6 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -301,7 +301,7 @@ To access this value, say ``self.config['foo'].get()`` at any point in your plugin's code. The `self.config` object is a *view* as defined by the `Confuse`_ library. -.. _Confuse: https://confuse.readthedocs.org/ +.. _Confuse: https://confuse.readthedocs.io/en/latest/ If you want to access configuration values *outside* of your plugin's section, import the `config` object from the `beets` module. That is, just put ``from @@ -379,7 +379,7 @@ access to file tags. If you have created a descriptor you can add it through your plugins ``add_media_field()`` method. .. automethod:: beets.plugins.BeetsPlugin.add_media_field -.. _MediaFile: https://mediafile.readthedocs.io/ +.. _MediaFile: https://mediafile.readthedocs.io/en/latest/ Here's an example plugin that provides a meaningless new field "foo":: diff --git a/docs/faq.rst b/docs/faq.rst index 9732a4725..eeab6c1ef 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -2,10 +2,9 @@ FAQ ### Here are some answers to frequently-asked questions from IRC and elsewhere. -Got a question that isn't answered here? Try `IRC`_, the `discussion board`_, or +Got a question that isn't answered here? Try the `discussion board`_, or :ref:`filing an issue ` in the bug tracker. -.. _IRC: irc://irc.freenode.net/beets .. _mailing list: https://groups.google.com/group/beets-users .. _discussion board: https://discourse.beets.io @@ -119,7 +118,7 @@ Run a command like this:: pip install -U beets -The ``-U`` flag tells `pip `__ to upgrade +The ``-U`` flag tells `pip`_ to upgrade beets to the latest version. If you want a specific version, you can specify with using ``==`` like so:: @@ -188,7 +187,9 @@ there to report a bug. Please follow these guidelines when reporting an issue: If you've never reported a bug before, Mozilla has some well-written `general guidelines for good bug -reports `__. +reports`_. + +.. _general guidelines for good bug reports: https://developer.mozilla.org/en-US/docs/Mozilla/QA/Bug_writing_guidelines .. _find-config: @@ -300,8 +301,7 @@ a flag. There is no simple way to remedy this.) …not change my ID3 tags? ------------------------ -Beets writes `ID3v2.4 `__ tags by -default. +Beets writes `ID3v2.4`_ tags by default. Some software, including Windows (i.e., Windows Explorer and Windows Media Player) and `id3lib/id3v2 `__, don't support v2.4 tags. When using 2.4-unaware software, it might look @@ -311,6 +311,7 @@ To enable ID3v2.3 tags, enable the :ref:`id3v23` config option. .. _invalid: +.. _ID3v2.4: https://id3.org/id3v2.4.0-structure …complain that a file is "unreadable"? -------------------------------------- @@ -379,3 +380,4 @@ installed using pip, the command ``pip show -f beets`` can show you where try `this Super User answer`_. .. _this Super User answer: https://superuser.com/a/284361/4569 +.. _pip: https://pip.pypa.io/en/stable/ diff --git a/docs/guides/main.rst b/docs/guides/main.rst index 2f05634d9..f1da16f50 100644 --- a/docs/guides/main.rst +++ b/docs/guides/main.rst @@ -64,7 +64,7 @@ beets`` if you run into permissions problems). To install without pip, download beets from `its PyPI page`_ and run ``python setup.py install`` in the directory therein. -.. _its PyPI page: https://pypi.org/project/beets#downloads +.. _its PyPI page: https://pypi.org/project/beets/#files .. _pip: https://pip.pypa.io The best way to upgrade beets to a new version is by running ``pip install -U diff --git a/docs/plugins/absubmit.rst b/docs/plugins/absubmit.rst index 64c77e077..953335a14 100644 --- a/docs/plugins/absubmit.rst +++ b/docs/plugins/absubmit.rst @@ -62,6 +62,6 @@ file. The available options are: .. _streaming_extractor_music: https://acousticbrainz.org/download .. _FAQ: https://acousticbrainz.org/faq .. _pip: https://pip.pypa.io -.. _requests: https://docs.python-requests.org/en/master/ +.. _requests: https://requests.readthedocs.io/en/master/ .. _github: https://github.com/MTG/essentia .. _AcousticBrainz: https://acousticbrainz.org diff --git a/docs/plugins/beatport.rst b/docs/plugins/beatport.rst index cbf5b4312..6117c4a1f 100644 --- a/docs/plugins/beatport.rst +++ b/docs/plugins/beatport.rst @@ -41,6 +41,6 @@ Configuration This plugin can be configured like other metadata source plugins as described in :ref:`metadata-source-plugin-configuration`. -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib -.. _Beatport: https://beetport.com +.. _Beatport: https://www.beatport.com/ diff --git a/docs/plugins/bpd.rst b/docs/plugins/bpd.rst index 49563a73a..2330bea70 100644 --- a/docs/plugins/bpd.rst +++ b/docs/plugins/bpd.rst @@ -5,7 +5,7 @@ BPD is a music player using music from a beets library. It runs as a daemon and implements the MPD protocol, so it's compatible with all the great MPD clients out there. I'm using `Theremin`_, `gmpc`_, `Sonata`_, and `Ario`_ successfully. -.. _Theremin: https://theremin.sigterm.eu/ +.. _Theremin: https://github.com/TheStalwart/Theremin .. _gmpc: https://gmpc.wikia.com/wiki/Gnome_Music_Player_Client .. _Sonata: http://sonata.berlios.de/ .. _Ario: http://ario-player.sourceforge.net/ @@ -13,7 +13,7 @@ out there. I'm using `Theremin`_, `gmpc`_, `Sonata`_, and `Ario`_ successfully. Dependencies ------------ -Before you can use BPD, you'll need the media library called GStreamer (along +Before you can use BPD, you'll need the media library called `GStreamer`_ (along with its Python bindings) on your system. * On Mac OS X, you can use `Homebrew`_. Run ``brew install gstreamer @@ -22,14 +22,11 @@ with its Python bindings) on your system. * On Linux, you need to install GStreamer 1.0 and the GObject bindings for python. Under Ubuntu, they are called ``python-gi`` and ``gstreamer1.0``. -* On Windows, you may want to try `GStreamer WinBuilds`_ (caveat emptor: I - haven't tried this). - You will also need the various GStreamer plugin packages to make everything work. See the :doc:`/plugins/chroma` documentation for more information on installing GStreamer plugins. -.. _GStreamer WinBuilds: https://www.gstreamer-winbuild.ylatuya.es/ +.. _GStreamer: https://gstreamer.freedesktop.org/download .. _Homebrew: https://brew.sh Usage diff --git a/docs/plugins/convert.rst b/docs/plugins/convert.rst index 9581e24a4..d53b8dc6d 100644 --- a/docs/plugins/convert.rst +++ b/docs/plugins/convert.rst @@ -191,7 +191,7 @@ can use the :doc:`/plugins/replaygain` to do this analysis. See the LAME `documentation`_ and the `HydrogenAudio wiki`_ for other LAME configuration options and a thorough discussion of MP3 encoding. -.. _documentation: http://lame.sourceforge.net/using.php +.. _documentation: https://lame.sourceforge.io/index.php .. _HydrogenAudio wiki: https://wiki.hydrogenaud.io/index.php?title=LAME .. _gapless: https://wiki.hydrogenaud.io/index.php?title=Gapless_playback -.. _LAME: https://lame.sourceforge.net/ +.. _LAME: https://lame.sourceforge.io/index.php diff --git a/docs/plugins/embyupdate.rst b/docs/plugins/embyupdate.rst index 626fafa9d..1a8b7c7b1 100644 --- a/docs/plugins/embyupdate.rst +++ b/docs/plugins/embyupdate.rst @@ -18,7 +18,7 @@ To use the ``embyupdate`` plugin you need to install the `requests`_ library wit With that all in place, you'll see beets send the "update" command to your Emby server every time you change your beets library. .. _Emby: https://emby.media/ -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ Configuration ------------- diff --git a/docs/plugins/keyfinder.rst b/docs/plugins/keyfinder.rst index 2ed2c1cec..a5c64d39c 100644 --- a/docs/plugins/keyfinder.rst +++ b/docs/plugins/keyfinder.rst @@ -31,5 +31,5 @@ configuration file. The available options are: `initial_key` value. Default: ``no``. -.. _KeyFinder: https://www.ibrahimshaath.co.uk/keyfinder/ +.. _KeyFinder: http://www.ibrahimshaath.co.uk/keyfinder/ .. _keyfinder-cli: https://github.com/EvanPurkhiser/keyfinder-cli/ diff --git a/docs/plugins/kodiupdate.rst b/docs/plugins/kodiupdate.rst index e60f503f2..f521a8000 100644 --- a/docs/plugins/kodiupdate.rst +++ b/docs/plugins/kodiupdate.rst @@ -27,7 +27,7 @@ With that all in place, you'll see beets send the "update" command to your Kodi host every time you change your beets library. .. _Kodi: https://kodi.tv/ -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ Configuration ------------- diff --git a/docs/plugins/lastgenre.rst b/docs/plugins/lastgenre.rst index 5fcdd2254..dee4260de 100644 --- a/docs/plugins/lastgenre.rst +++ b/docs/plugins/lastgenre.rst @@ -1,13 +1,10 @@ LastGenre Plugin ================ -The MusicBrainz database `does not contain genre information`_. Therefore, when -importing and autotagging music, beets does not assign a genre. The -``lastgenre`` plugin fetches *tags* from `Last.fm`_ and assigns them as genres + +The ``lastgenre`` plugin fetches *tags* from `Last.fm`_ and assigns them as genres to your albums and items. -.. _does not contain genre information: - https://musicbrainz.org/doc/General_FAQ#Why_does_MusicBrainz_not_support_genre_information.3F .. _Last.fm: https://last.fm/ Installation @@ -72,7 +69,7 @@ nothing would ever be matched to a more generic node since all the specific subgenres are in the whitelist to begin with. -.. _YAML: https://www.yaml.org/ +.. _YAML: https://yaml.org/ .. _tree of nested genre names: https://raw.githubusercontent.com/beetbox/beets/master/beetsplug/lastgenre/genres-tree.yaml diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index fac07ad87..942497a7c 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -26,7 +26,7 @@ already have them. The lyrics will be stored in the beets database. If the ``import.write`` config option is on, then the lyrics will also be written to the files' tags. -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ Configuration @@ -180,8 +180,7 @@ You also need to register for a Microsoft Azure Marketplace free account and to the `Microsoft Translator API`_. Follow the four steps process, specifically at step 3 enter ``beets`` as *Client ID* and copy/paste the generated *Client secret* into your ``bing_client_secret`` configuration, alongside -``bing_lang_to`` target `language code`_. +``bing_lang_to`` target `language code`. .. _langdetect: https://pypi.python.org/pypi/langdetect -.. _Microsoft Translator API: https://www.microsoft.com/en-us/translator/getstarted.aspx -.. _language code: https://msdn.microsoft.com/en-us/library/hh456380.aspx +.. _Microsoft Translator API: https://docs.microsoft.com/en-us/azure/cognitive-services/translator/translator-how-to-signup diff --git a/docs/plugins/plexupdate.rst b/docs/plugins/plexupdate.rst index 92fc949d2..b6a2bf920 100644 --- a/docs/plugins/plexupdate.rst +++ b/docs/plugins/plexupdate.rst @@ -25,7 +25,7 @@ With that all in place, you'll see beets send the "update" command to your Plex server every time you change your beets library. .. _Plex: https://plex.tv/ -.. _requests: https://docs.python-requests.org/en/latest/ +.. _requests: https://requests.readthedocs.io/en/master/ .. _documentation about tokens: https://support.plex.tv/hc/en-us/articles/204059436-Finding-your-account-token-X-Plex-Token Configuration diff --git a/docs/plugins/subsonicupdate.rst b/docs/plugins/subsonicupdate.rst index 3549be091..710d21f2c 100644 --- a/docs/plugins/subsonicupdate.rst +++ b/docs/plugins/subsonicupdate.rst @@ -4,7 +4,7 @@ SubsonicUpdate Plugin ``subsonicupdate`` is a very simple plugin for beets that lets you automatically update `Subsonic`_'s index whenever you change your beets library. -.. _Subsonic: https://www.subsonic.org +.. _Subsonic: http://www.subsonic.org/pages/index.jsp To use ``subsonicupdate`` plugin, enable it in your configuration (see :ref:`using-plugins`). diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 65d4743fb..85de48dd4 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -19,8 +19,6 @@ The Web interface depends on `Flask`_. To get it, just run ``pip install flask``. Then enable the ``web`` plugin in your configuration (see :ref:`using-plugins`). -.. _Flask: https://flask.pocoo.org/ - If you need CORS (it's disabled by default---see :ref:`web-cors`, below), then you also need `flask-cors`_. Just type ``pip install flask-cors``. @@ -47,9 +45,7 @@ Usage ----- Type queries into the little search box. Double-click a track to play it with -`HTML5 Audio`_. - -.. _HTML5 Audio: http://www.w3.org/TR/html-markup/audio.html +HTML5 Audio. Configuration ------------- @@ -78,7 +74,7 @@ The Web backend is built using a simple REST+JSON API with the excellent `Flask`_ library. The frontend is a single-page application written with `Backbone.js`_. This allows future non-Web clients to use the same backend API. -.. _Flask: https://flask.pocoo.org/ + .. _Backbone.js: https://backbonejs.org Eventually, to make the Web player really viable, we should use a Flash fallback @@ -90,7 +86,7 @@ for unsupported formats/browsers. There are a number of options for this: .. _audio.js: https://kolber.github.io/audiojs/ .. _html5media: https://html5media.info/ -.. _MediaElement.js: https://mediaelementjs.com/ +.. _MediaElement.js: https://www.mediaelementjs.com/ .. _web-cors: @@ -262,3 +258,5 @@ Responds with the number of tracks and albums in the database. :: "items": 5, "albums": 3 } + +.. _Flask: https://flask.palletsprojects.com/en/1.1.x/ diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 46f14f2c5..2f8cee3c9 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -689,7 +689,7 @@ to one request per second. .. _your own MusicBrainz database: https://musicbrainz.org/doc/MusicBrainz_Server/Setup .. _main server: https://musicbrainz.org/ .. _limited: https://musicbrainz.org/doc/XML_Web_Service/Rate_Limiting -.. _Building search indexes: https://musicbrainz.org/doc/MusicBrainz_Server/Setup#Building_search_indexes +.. _Building search indexes: https://musicbrainz.org/doc/Development/Search_server_setup .. _searchlimit: diff --git a/tox.ini b/tox.ini index cbf953033..69308235d 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,12 @@ basepython = python2.7 deps = sphinx commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} +# checks all links in the docs +[testenv:links] +deps = sphinx +allowlist_externals = /bin/bash +commands = /bin/bash -c '! sphinx-build -b linkcheck docs {envtmpdir}/linkcheck | grep "broken\s"' + [testenv:int] deps = {[_test]deps} setenv = INTEGRATION_TEST = 1 From c8443332dea3dcd868c5d7d4ec1ae0ea232d210d Mon Sep 17 00:00:00 2001 From: Jamie Quigley Date: Mon, 7 Sep 2020 17:31:42 +0100 Subject: [PATCH 19/65] Added flac-specific samplerate-bitdepth reporting for duplicate imports --- beets/ui/commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 56f9ad1f5..d9fe9dcf0 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -465,9 +465,13 @@ def summarize_items(items, singleton): if items: average_bitrate = sum([item.bitrate for item in items]) / len(items) + if items[0].format == "FLAC": + sample_bits = u'{}kHz/{} bit'.format(items[0].samplerate, items[0].bitdepth) total_duration = sum([item.length for item in items]) total_filesize = sum([item.filesize for item in items]) summary_parts.append(u'{0}kbps'.format(int(average_bitrate / 1000))) + if items[0].format == "FLAC": + summary_parts.append(sample_bits) summary_parts.append(ui.human_seconds_short(total_duration)) summary_parts.append(ui.human_bytes(total_filesize)) From 1ab162743ac27be40277a9c33f88313b8751f294 Mon Sep 17 00:00:00 2001 From: Jamie Quigley Date: Mon, 7 Sep 2020 17:42:56 +0100 Subject: [PATCH 20/65] Convert to kHz and meet line limit --- beets/ui/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index d9fe9dcf0..eb85485b6 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -466,7 +466,8 @@ def summarize_items(items, singleton): if items: average_bitrate = sum([item.bitrate for item in items]) / len(items) if items[0].format == "FLAC": - sample_bits = u'{}kHz/{} bit'.format(items[0].samplerate, items[0].bitdepth) + sample_bits = u'{}kHz/{} bit'.format( + round(int(items[0].samplerate) / 1000, 1), items[0].bitdepth) total_duration = sum([item.length for item in items]) total_filesize = sum([item.filesize for item in items]) summary_parts.append(u'{0}kbps'.format(int(average_bitrate / 1000))) From e83959ab757103ab3f36d6bb8d0215d1084166ca Mon Sep 17 00:00:00 2001 From: Jamie Quigley Date: Mon, 7 Sep 2020 22:11:15 +0100 Subject: [PATCH 21/65] Add changelog item and merge if statements --- beets/ui/commands.py | 5 ++--- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index eb85485b6..f34e5578f 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -465,13 +465,12 @@ def summarize_items(items, singleton): if items: average_bitrate = sum([item.bitrate for item in items]) / len(items) - if items[0].format == "FLAC": - sample_bits = u'{}kHz/{} bit'.format( - round(int(items[0].samplerate) / 1000, 1), items[0].bitdepth) total_duration = sum([item.length for item in items]) total_filesize = sum([item.filesize for item in items]) summary_parts.append(u'{0}kbps'.format(int(average_bitrate / 1000))) if items[0].format == "FLAC": + sample_bits = u'{}kHz/{} bit'.format( + round(int(items[0].samplerate) / 1000, 1), items[0].bitdepth) summary_parts.append(sample_bits) summary_parts.append(ui.human_seconds_short(total_duration)) summary_parts.append(ui.human_bytes(total_filesize)) diff --git a/docs/changelog.rst b/docs/changelog.rst index bc03d772e..a9d2ac540 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -147,6 +147,7 @@ New features: be deleted after importing. Thanks to :user:`logan-arens`. :bug:`2947` +* Added flac-specific reporting of samplerate and bitrate when importing duplicates. Fixes: From 822bc1ce88d981502567383bb0ca583f422f5877 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 13 Jul 2020 12:42:16 +0200 Subject: [PATCH 22/65] add possibility to select individual items to the remove CLI command --- beets/ui/__init__.py | 10 ++++++---- beets/ui/commands.py | 42 ++++++++++++++++++++++++++++++++---------- docs/reference/cli.rst | 19 +++++++++++++++---- test/test_ui.py | 39 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 89 insertions(+), 21 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index aec0e80a9..09f30c109 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -389,17 +389,19 @@ def input_yn(prompt, require=False): return sel == u'y' -def input_select_objects(prompt, objs, rep): +def input_select_objects(prompt, objs, rep, prompt_all=None): """Prompt to user to choose all, none, or some of the given objects. Return the list of selected objects. `prompt` is the prompt string to use for each question (it should be - phrased as an imperative verb). `rep` is a function to call on each - object to print it out when confirming objects individually. + phrased as an imperative verb). If `prompt_all` is given, it is used + instead of `prompt` for the first (yes(/no/select) question. + `rep` is a function to call on each object to print it out when confirming + objects individually. """ choice = input_options( (u'y', u'n', u's'), False, - u'%s? (Yes/no/select)' % prompt) + u'%s? (Yes/no/select)' % (prompt_all or prompt)) print() # Blank line. if choice == u'y': # Yes. diff --git a/beets/ui/commands.py b/beets/ui/commands.py index f34e5578f..49c4b4dc6 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1232,31 +1232,53 @@ def remove_items(lib, query, album, delete, force): """ # Get the matching items. items, albums = _do_query(lib, query, album) + objs = albums if album else items # Confirm file removal if not forcing removal. if not force: # Prepare confirmation with user. - print_() + album_str = u" in {} album{}".format( + len(albums), u's' if len(albums) > 1 else u'' + ) if album else "" + if delete: fmt = u'$path - $title' - prompt = u'Really DELETE %i file%s (y/n)?' % \ - (len(items), 's' if len(items) > 1 else '') + prompt = u'Really DELETE' + prompt_all = u'Really DELETE {} file{}{}'.format( + len(items), u's' if len(items) > 1 else u'', album_str + ) else: fmt = u'' - prompt = u'Really remove %i item%s from the library (y/n)?' % \ - (len(items), 's' if len(items) > 1 else '') + prompt = u'Really remove from the library?' + prompt_all = u'Really remove {} item{}{} from the library?'.format( + len(items), u's' if len(items) > 1 else u'', album_str + ) + + # Helpers for printing affected items + def fmt_track(t): + ui.print_(format(t, fmt)) + + def fmt_album(a): + ui.print_() + for i in a.items(): + fmt_track(i) + + fmt_obj = fmt_album if album else fmt_track # Show all the items. - for item in items: - ui.print_(format(item, fmt)) + for o in objs: + fmt_obj(o) # Confirm with user. - if not ui.input_yn(prompt, True): - return + objs = ui.input_select_objects(prompt, objs, fmt_obj, + prompt_all=prompt_all) + + if not objs: + return # Remove (and possibly delete) items. with lib.transaction(): - for obj in (albums if album else items): + for obj in objs: obj.remove(delete) diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 724afc80a..2062193ab 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -230,10 +230,21 @@ remove Remove music from your library. This command uses the same :doc:`query ` syntax as the ``list`` command. -You'll be shown a list of the files that will be removed and asked to confirm. -By default, this just removes entries from the library database; it doesn't -touch the files on disk. To actually delete the files, use ``beet remove -d``. -If you do not want to be prompted to remove the files, use ``beet remove -f``. +By default, it just removes entries from the library database; it doesn't +touch the files on disk. To actually delete the files, use the ``-d`` flag. +When the ``-a`` flag is given, the command operates on albums instead of +individual tracks. + +When you run the ``remove`` command, it prints a list of all +affected items in the library and asks for your permission before removing +them. You can then choose to abort (type `n`), confirm (`y`), or interactively +choose some of the items (`s`). In the latter case, the command will prompt you +for every matching item or album and invite you to type `y` to remove the +item/album, `n` to keep it or `q` to exit and only remove the items/albums +selected up to this point. +This option lets you choose precisely which tracks/albums to remove without +spending too much time to carefully craft a query. +If you do not want to be prompted at all, use the ``-f`` option. .. _modify-cmd: diff --git a/test/test_ui.py b/test/test_ui.py index b1e7e8fad..f4ab8b16d 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -111,7 +111,7 @@ class ListTest(unittest.TestCase): self.assertNotIn(u'the album', stdout.getvalue()) -class RemoveTest(_common.TestCase): +class RemoveTest(_common.TestCase, TestHelper): def setUp(self): super(RemoveTest, self).setUp() @@ -122,8 +122,8 @@ class RemoveTest(_common.TestCase): # Copy a file into the library. self.lib = library.Library(':memory:', self.libdir) - item_path = os.path.join(_common.RSRC, b'full.mp3') - self.i = library.Item.from_path(item_path) + self.item_path = os.path.join(_common.RSRC, b'full.mp3') + self.i = library.Item.from_path(self.item_path) self.lib.add(self.i) self.i.move(operation=MoveOperation.COPY) @@ -153,6 +153,39 @@ class RemoveTest(_common.TestCase): self.assertEqual(len(list(items)), 0) self.assertFalse(os.path.exists(self.i.path)) + def test_remove_items_select_with_delete(self): + i2 = library.Item.from_path(self.item_path) + self.lib.add(i2) + i2.move(operation=MoveOperation.COPY) + + for s in ('s', 'y', 'n'): + self.io.addinput(s) + commands.remove_items(self.lib, u'', False, True, False) + items = self.lib.items() + self.assertEqual(len(list(items)), 1) + # FIXME: is the order of the items as queried by the remove command + # really deterministic? + self.assertFalse(os.path.exists(syspath(self.i.path))) + self.assertTrue(os.path.exists(syspath(i2.path))) + + def test_remove_albums_select_with_delete(self): + a1 = self.add_album_fixture() + a2 = self.add_album_fixture() + path1 = a1.items()[0].path + path2 = a2.items()[0].path + items = self.lib.items() + self.assertEqual(len(list(items)), 3) + + for s in ('s', 'y', 'n'): + self.io.addinput(s) + commands.remove_items(self.lib, u'', True, True, False) + items = self.lib.items() + self.assertEqual(len(list(items)), 2) # incl. the item from setUp() + # FIXME: is the order of the items as queried by the remove command + # really deterministic? + self.assertFalse(os.path.exists(syspath(path1))) + self.assertTrue(os.path.exists(syspath(path2))) + class ModifyTest(unittest.TestCase, TestHelper): From a8065ff3d6ce93483e8fa7f18f47fb6769c3230c Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 13 Jul 2020 13:06:47 +0200 Subject: [PATCH 23/65] update changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a9d2ac540..6a59312d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -148,6 +148,8 @@ New features: Thanks to :user:`logan-arens`. :bug:`2947` * Added flac-specific reporting of samplerate and bitrate when importing duplicates. +* ``beet remove`` now also allows interactive selection of items from the query + similar to ``beet modify`` Fixes: From 19784845041676a7cc9e176695ef7b45b40db83b Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Sun, 13 Sep 2020 15:55:09 +0200 Subject: [PATCH 24/65] don't assume items are queried in any specific order in interactive delete test The previous test worked (on my machine, and on Github CI and AppVeyor), but it is not obvious whether the order is really guaranteed (given that the full beets database stack and sqlite are involved). Thus, to prevent this from exploding at some point, only verify the number of deletions for now. --- test/test_ui.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/test/test_ui.py b/test/test_ui.py index f4ab8b16d..5cfed1fda 100644 --- a/test/test_ui.py +++ b/test/test_ui.py @@ -163,10 +163,14 @@ class RemoveTest(_common.TestCase, TestHelper): commands.remove_items(self.lib, u'', False, True, False) items = self.lib.items() self.assertEqual(len(list(items)), 1) - # FIXME: is the order of the items as queried by the remove command - # really deterministic? - self.assertFalse(os.path.exists(syspath(self.i.path))) - self.assertTrue(os.path.exists(syspath(i2.path))) + # There is probably no guarantee that the items are queried in any + # spcecific order, thus just ensure that exactly one was removed. + # To improve upon this, self.io would need to have the capability to + # generate input that depends on previous output. + num_existing = 0 + num_existing += 1 if os.path.exists(syspath(self.i.path)) else 0 + num_existing += 1 if os.path.exists(syspath(i2.path)) else 0 + self.assertEqual(num_existing, 1) def test_remove_albums_select_with_delete(self): a1 = self.add_album_fixture() @@ -181,10 +185,11 @@ class RemoveTest(_common.TestCase, TestHelper): commands.remove_items(self.lib, u'', True, True, False) items = self.lib.items() self.assertEqual(len(list(items)), 2) # incl. the item from setUp() - # FIXME: is the order of the items as queried by the remove command - # really deterministic? - self.assertFalse(os.path.exists(syspath(path1))) - self.assertTrue(os.path.exists(syspath(path2))) + # See test_remove_items_select_with_delete() + num_existing = 0 + num_existing += 1 if os.path.exists(syspath(path1)) else 0 + num_existing += 1 if os.path.exists(syspath(path2)) else 0 + self.assertEqual(num_existing, 1) class ModifyTest(unittest.TestCase, TestHelper): From 33b10d60fbb4823c01ac8b781ea33b50867aaa4f Mon Sep 17 00:00:00 2001 From: djl Date: Sun, 13 Sep 2020 20:27:12 +0100 Subject: [PATCH 25/65] fetchart: Improve Cover Art Archive source. (#3748) * fetchart: Improve Cover Art Archive source. Instead of blindly selecting the first image, we now treat all "front" images as candidates. This is useful where some digital releases have both an animated cover and a still image and the animated image is the first image returned from the API. --- beetsplug/fetchart.py | 53 +++++++++++----- docs/changelog.rst | 2 + test/test_art.py | 136 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 163 insertions(+), 28 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 86c5b958f..1bf8ad428 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -308,16 +308,44 @@ class CoverArtArchive(RemoteArtSource): VALID_THUMBNAIL_SIZES = [250, 500, 1200] if util.SNI_SUPPORTED: - URL = 'https://coverartarchive.org/release/{mbid}/front' - GROUP_URL = 'https://coverartarchive.org/release-group/{mbid}/front' + URL = 'https://coverartarchive.org/release/{mbid}' + GROUP_URL = 'https://coverartarchive.org/release-group/{mbid}' else: - URL = 'http://coverartarchive.org/release/{mbid}/front' - GROUP_URL = 'http://coverartarchive.org/release-group/{mbid}/front' + URL = 'http://coverartarchive.org/release/{mbid}' + GROUP_URL = 'http://coverartarchive.org/release-group/{mbid}' def get(self, album, plugin, paths): """Return the Cover Art Archive and Cover Art Archive release group URLs using album MusicBrainz release ID and release group ID. """ + + def get_image_urls(url, size_suffix=None): + try: + response = self.request(url) + except requests.RequestException: + self._log.debug(u'{0}: error receiving response' + .format(self.NAME)) + return + + try: + data = response.json() + except ValueError: + self._log.debug(u'{0}: error loading response: {1}' + .format(self.NAME, response.text)) + return + + for item in data.get('images', []): + try: + if 'Front' not in item['types']: + continue + + if size_suffix: + yield item['thumbnails'][size_suffix] + else: + yield item['image'] + except KeyError: + pass + release_url = self.URL.format(mbid=album.mb_albumid) release_group_url = self.GROUP_URL.format(mbid=album.mb_releasegroupid) @@ -330,19 +358,12 @@ class CoverArtArchive(RemoteArtSource): size_suffix = "-" + str(plugin.maxwidth) if 'release' in self.match_by and album.mb_albumid: - if size_suffix: - release_thumbnail_url = release_url + size_suffix - yield self._candidate(url=release_thumbnail_url, - match=Candidate.MATCH_EXACT) - yield self._candidate(url=release_url, - match=Candidate.MATCH_EXACT) + for url in get_image_urls(release_url, size_suffix): + yield self._candidate(url=url, match=Candidate.MATCH_EXACT) + if 'releasegroup' in self.match_by and album.mb_releasegroupid: - if size_suffix: - release_group_thumbnail_url = release_group_url + size_suffix - yield self._candidate(url=release_group_thumbnail_url, - match=Candidate.MATCH_FALLBACK) - yield self._candidate(url=release_group_url, - match=Candidate.MATCH_FALLBACK) + for url in get_image_urls(release_group_url): + yield self._candidate(url=url, match=Candidate.MATCH_FALLBACK) class Amazon(RemoteArtSource): diff --git a/docs/changelog.rst b/docs/changelog.rst index a9d2ac540..c89bdb681 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -148,6 +148,8 @@ New features: Thanks to :user:`logan-arens`. :bug:`2947` * Added flac-specific reporting of samplerate and bitrate when importing duplicates. +* :doc:`/plugins/fetchart`: Cover Art Archive source now iterates over + all front images instead of blindly selecting the first one. Fixes: diff --git a/test/test_art.py b/test/test_art.py index f4b3a6e62..51e5a9fe8 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -76,6 +76,96 @@ class FetchImageHelper(_common.TestCase): file_type, b'').ljust(32, b'\x00')) +class CAAHelper(): + """Helper mixin for mocking requests to the Cover Art Archive.""" + MBID_RELASE = 'rid' + MBID_GROUP = 'rgid' + + RELEASE_URL = 'coverartarchive.org/release/{0}' \ + .format(MBID_RELASE) + GROUP_URL = 'coverartarchive.org/release-group/{0}' \ + .format(MBID_GROUP) + + if util.SNI_SUPPORTED: + RELEASE_URL = "https://" + RELEASE_URL + GROUP_URL = "https://" + GROUP_URL + else: + RELEASE_URL = "http://" + RELEASE_URL + GROUP_URL = "http://" + GROUP_URL + + RESPONSE_RELEASE = """{ + "images": [ + { + "approved": false, + "back": false, + "comment": "GIF", + "edit": 12345, + "front": true, + "id": 12345, + "image": "http://coverartarchive.org/release/rid/12345.gif", + "thumbnails": { + "1200": "http://coverartarchive.org/release/rid/12345-1200.jpg", + "250": "http://coverartarchive.org/release/rid/12345-250.jpg", + "500": "http://coverartarchive.org/release/rid/12345-500.jpg", + "large": "http://coverartarchive.org/release/rid/12345-500.jpg", + "small": "http://coverartarchive.org/release/rid/12345-250.jpg" + }, + "types": [ + "Front" + ] + }, + { + "approved": false, + "back": false, + "comment": "", + "edit": 12345, + "front": false, + "id": 12345, + "image": "http://coverartarchive.org/release/rid/12345.jpg", + "thumbnails": { + "1200": "http://coverartarchive.org/release/rid/12345-1200.jpg", + "250": "http://coverartarchive.org/release/rid/12345-250.jpg", + "500": "http://coverartarchive.org/release/rid/12345-500.jpg", + "large": "http://coverartarchive.org/release/rid/12345-500.jpg", + "small": "http://coverartarchive.org/release/rid/12345-250.jpg" + }, + "types": [ + "Front" + ] + } + ], + "release": "https://musicbrainz.org/release/releaseid" +}""" + RESPONSE_GROUP = """{ + "images": [ + { + "approved": false, + "back": false, + "comment": "", + "edit": 12345, + "front": true, + "id": 12345, + "image": "http://coverartarchive.org/release/releaseid/12345.jpg", + "thumbnails": { + "1200": "http://coverartarchive.org/release/rgid/12345-1200.jpg", + "250": "http://coverartarchive.org/release/rgid/12345-250.jpg", + "500": "http://coverartarchive.org/release/rgid/12345-500.jpg", + "large": "http://coverartarchive.org/release/rgid/12345-500.jpg", + "small": "http://coverartarchive.org/release/rgid/12345-250.jpg" + }, + "types": [ + "Front" + ] + } + ], + "release": "https://musicbrainz.org/release/release-id" + }""" + + def mock_caa_response(self, url, json): + responses.add(responses.GET, url, body=json, + content_type='application/json') + + class FetchImageTest(FetchImageHelper, UseThePlugin): URL = 'http://example.com/test.jpg' @@ -156,15 +246,13 @@ class FSArtTest(UseThePlugin): self.assertEqual(candidates, paths) -class CombinedTest(FetchImageHelper, UseThePlugin): +class CombinedTest(FetchImageHelper, UseThePlugin, CAAHelper): ASIN = 'xxxx' MBID = 'releaseid' AMAZON_URL = 'https://images.amazon.com/images/P/{0}.01.LZZZZZZZ.jpg' \ .format(ASIN) AAO_URL = 'https://www.albumart.org/index_detail.php?asin={0}' \ .format(ASIN) - CAA_URL = 'coverartarchive.org/release/{0}/front' \ - .format(MBID) def setUp(self): super(CombinedTest, self).setUp() @@ -211,17 +299,19 @@ class CombinedTest(FetchImageHelper, UseThePlugin): self.assertEqual(responses.calls[-1].request.url, self.AAO_URL) def test_main_interface_uses_caa_when_mbid_available(self): - self.mock_response("http://" + self.CAA_URL) - self.mock_response("https://" + self.CAA_URL) - album = _common.Bag(mb_albumid=self.MBID, asin=self.ASIN) + self.mock_caa_response(self.RELEASE_URL, self.RESPONSE_RELEASE) + self.mock_caa_response(self.GROUP_URL, self.RESPONSE_GROUP) + self.mock_response('http://coverartarchive.org/release/rid/12345.gif', + content_type='image/gif') + self.mock_response('http://coverartarchive.org/release/rid/12345.jpg', + content_type='image/jpeg') + album = _common.Bag(mb_albumid=self.MBID_RELASE, + mb_releasegroupid=self.MBID_GROUP, + asin=self.ASIN) candidate = self.plugin.art_for_album(album, None) self.assertIsNotNone(candidate) - self.assertEqual(len(responses.calls), 1) - if util.SNI_SUPPORTED: - url = "https://" + self.CAA_URL - else: - url = "http://" + self.CAA_URL - self.assertEqual(responses.calls[0].request.url, url) + self.assertEqual(len(responses.calls), 3) + self.assertEqual(responses.calls[0].request.url, self.RELEASE_URL) def test_local_only_does_not_access_network(self): album = _common.Bag(mb_albumid=self.MBID, asin=self.ASIN) @@ -416,6 +506,28 @@ class GoogleImageTest(UseThePlugin): next(self.source.get(album, self.settings, [])) +class CoverArtArchiveTest(UseThePlugin, CAAHelper): + + def setUp(self): + super(CoverArtArchiveTest, self).setUp() + self.source = fetchart.CoverArtArchive(logger, self.plugin.config) + self.settings = Settings(maxwidth=0) + + @responses.activate + def run(self, *args, **kwargs): + super(CoverArtArchiveTest, self).run(*args, **kwargs) + + def test_caa_finds_image(self): + album = _common.Bag(mb_albumid=self.MBID_RELASE, + mb_releasegroupid=self.MBID_GROUP) + self.mock_caa_response(self.RELEASE_URL, self.RESPONSE_RELEASE) + self.mock_caa_response(self.GROUP_URL, self.RESPONSE_GROUP) + candidates = list(self.source.get(album, self.settings, [])) + self.assertEqual(len(candidates), 3) + self.assertEqual(len(responses.calls), 2) + self.assertEqual(responses.calls[0].request.url, self.RELEASE_URL) + + class FanartTVTest(UseThePlugin): RESPONSE_MULTIPLE = u"""{ "name": "artistname", From 76220fb1486b854804c4fcdaa9c9f22c65e8c77b Mon Sep 17 00:00:00 2001 From: Cadel Watson Date: Sun, 20 Sep 2020 10:18:07 +1000 Subject: [PATCH 26/65] Add DELETE method for items and albums --- beetsplug/web/__init__.py | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 49149772d..175aeae89 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -21,7 +21,7 @@ from beets import ui from beets import util import beets.library import flask -from flask import g +from flask import g, make_response, jsonify from werkzeug.routing import BaseConverter, PathConverter import os from unidecode import unidecode @@ -91,6 +91,17 @@ def is_expand(): return flask.request.args.get('expand') is not None +def is_delete(): + """Returns whether the current delete request should remove the selected files.""" + + return flask.request.args.get('delete') is not None + + +def get_method(): + """Returns the HTTP method of the current request.""" + return flask.request.method + + def resource(name): """Decorates a function to handle RESTful HTTP requests for a resource. """ @@ -99,16 +110,30 @@ def resource(name): entities = [retriever(id) for id in ids] entities = [entity for entity in entities if entity] - if len(entities) == 1: - return flask.jsonify(_rep(entities[0], expand=is_expand())) - elif entities: - return app.response_class( - json_generator(entities, root=name), - mimetype='application/json' - ) + if get_method() == "DELETE": + responder.__name__ = 'delete_{0}'.format(name) + + for entity in entities: + entity.remove(delete=is_delete()) + + return flask.make_response(jsonify({'deleted': True}), 200) + + elif get_method() == "GET": + responder.__name__ = 'get_{0}'.format(name) + + if len(entities) == 1: + return flask.jsonify(_rep(entities[0], expand=is_expand())) + elif entities: + return app.response_class( + json_generator(entities, root=name), + mimetype='application/json' + ) + else: + return flask.abort(404) + else: - return flask.abort(404) - responder.__name__ = 'get_{0}'.format(name) + return flask.abort(405) + return responder return make_responder @@ -203,7 +228,7 @@ def before_request(): # Items. -@app.route('/item/') +@app.route('/item/', methods=["GET", "DELETE"]) @resource('items') def get_item(id): return g.lib.get_item(id) @@ -279,12 +304,11 @@ def item_unique_field_values(key): # Albums. -@app.route('/album/') +@app.route('/album/', methods=["GET", "DELETE"]) @resource('albums') def get_album(id): return g.lib.get_album(id) - @app.route('/album/') @app.route('/album/query/') @resource_list('albums') From 29672a434f311d70c5ebd0b434f894e1eaecae81 Mon Sep 17 00:00:00 2001 From: Cadel Watson Date: Sun, 20 Sep 2020 10:23:25 +1000 Subject: [PATCH 27/65] Add DELETE method to resource queries --- beetsplug/web/__init__.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 175aeae89..0168da990 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -143,14 +143,29 @@ def resource_query(name): """ def make_responder(query_func): def responder(queries): - return app.response_class( - json_generator( - query_func(queries), - root='results', expand=is_expand() - ), - mimetype='application/json' - ) - responder.__name__ = 'query_{0}'.format(name) + entities = query_func(queries) + + if get_method() == "DELETE": + responder.__name__ = 'delete_query_{0}'.format(name) + + for entity in entities: + entity.remove(delete=is_delete()) + + return flask.make_response(jsonify({'deleted': True}), 200) + + elif get_method() == "GET": + responder.__name__ = 'query_{0}'.format(name) + + return app.response_class( + json_generator( + entities, + root='results', expand=is_expand() + ), + mimetype='application/json' + ) + else: + return flask.abort(405) + return responder return make_responder @@ -275,7 +290,7 @@ def item_file(item_id): return response -@app.route('/item/query/') +@app.route('/item/query/', methods=["GET", "DELETE"]) @resource_query('items') def item_query(queries): return g.lib.items(queries) @@ -316,7 +331,7 @@ def all_albums(): return g.lib.albums() -@app.route('/album/query/') +@app.route('/album/query/', methods=["GET", "DELETE"]) @resource_query('albums') def album_query(queries): return g.lib.albums(queries) From afcde697e09d19c5f743ccb5ad4a6b46b112a59e Mon Sep 17 00:00:00 2001 From: Cadel Watson Date: Sun, 20 Sep 2020 10:45:12 +1000 Subject: [PATCH 28/65] Add PATCH method to Items --- beetsplug/web/__init__.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 0168da990..7801a647e 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -102,7 +102,7 @@ def get_method(): return flask.request.method -def resource(name): +def resource(name, patchable=False): """Decorates a function to handle RESTful HTTP requests for a resource. """ def make_responder(retriever): @@ -111,15 +111,15 @@ def resource(name): entities = [entity for entity in entities if entity] if get_method() == "DELETE": - responder.__name__ = 'delete_{0}'.format(name) - for entity in entities: entity.remove(delete=is_delete()) return flask.make_response(jsonify({'deleted': True}), 200) - elif get_method() == "GET": - responder.__name__ = 'get_{0}'.format(name) + elif get_method() == "PATCH" and patchable: + for entity in entities: + entity.update(flask.request.get_json()) + entity.try_sync(True, False) # write, don't move if len(entities) == 1: return flask.jsonify(_rep(entities[0], expand=is_expand())) @@ -128,12 +128,23 @@ def resource(name): json_generator(entities, root=name), mimetype='application/json' ) + + elif get_method() == "GET": + if len(entities) == 1: + return flask.jsonify(_rep(entities[0], expand=is_expand())) + elif entities: + return app.response_class( + json_generator(entities, root=name), + mimetype='application/json' + ) else: return flask.abort(404) else: return flask.abort(405) + responder.__name__ = 'get_{0}'.format(name) + return responder return make_responder @@ -146,16 +157,12 @@ def resource_query(name): entities = query_func(queries) if get_method() == "DELETE": - responder.__name__ = 'delete_query_{0}'.format(name) - for entity in entities: entity.remove(delete=is_delete()) return flask.make_response(jsonify({'deleted': True}), 200) elif get_method() == "GET": - responder.__name__ = 'query_{0}'.format(name) - return app.response_class( json_generator( entities, @@ -166,7 +173,10 @@ def resource_query(name): else: return flask.abort(405) + responder.__name__ = 'query_{0}'.format(name) + return responder + return make_responder @@ -243,8 +253,8 @@ def before_request(): # Items. -@app.route('/item/', methods=["GET", "DELETE"]) -@resource('items') +@app.route('/item/', methods=["GET", "DELETE", "PATCH"]) +@resource('items', patchable=True) def get_item(id): return g.lib.get_item(id) From a18b317240d6493443f1a0b94d70b890ad243dbd Mon Sep 17 00:00:00 2001 From: Cadel Watson Date: Sun, 20 Sep 2020 10:46:48 +1000 Subject: [PATCH 29/65] Add PATCH method to item queries --- beetsplug/web/__init__.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 7801a647e..0ce5f2c4c 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -149,7 +149,7 @@ def resource(name, patchable=False): return make_responder -def resource_query(name): +def resource_query(name, patchable=False): """Decorates a function to handle RESTful HTTP queries for resources. """ def make_responder(query_func): @@ -162,6 +162,16 @@ def resource_query(name): return flask.make_response(jsonify({'deleted': True}), 200) + elif get_method() == "PATCH" and patchable: + for entity in entities: + entity.update(flask.request.get_json()) + entity.try_sync(True, False) # write, don't move + + return app.response_class( + json_generator(entities, root=name), + mimetype='application/json' + ) + elif get_method() == "GET": return app.response_class( json_generator( @@ -170,6 +180,7 @@ def resource_query(name): ), mimetype='application/json' ) + else: return flask.abort(405) @@ -300,8 +311,8 @@ def item_file(item_id): return response -@app.route('/item/query/', methods=["GET", "DELETE"]) -@resource_query('items') +@app.route('/item/query/', methods=["GET", "DELETE", "PATCH"]) +@resource_query('items', patchable=True) def item_query(queries): return g.lib.items(queries) From 3723f8a09f66ea013826fbd76c6421decab0e570 Mon Sep 17 00:00:00 2001 From: Cadel Watson Date: Sun, 20 Sep 2020 10:54:06 +1000 Subject: [PATCH 30/65] Update docs and changelog --- docs/changelog.rst | 1 + docs/plugins/web.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3a72132ab..47b0398c0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -152,6 +152,7 @@ New features: all front images instead of blindly selecting the first one. * ``beet remove`` now also allows interactive selection of items from the query similar to ``beet modify`` +* :doc:`/plugins/web`: add DELETE and PATCH methods for modifying items Fixes: diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 85de48dd4..4b069a944 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -183,6 +183,25 @@ representation. :: If there is no item with that id responds with a *404* status code. +``DELETE /item/6`` +++++++++++++++++++ + +Removes the item with id *6* from the beets library. If the *?delete* query string is included, +the matching file will be deleted from disk. + +``PATCH /item/6`` +++++++++++++++++++ + +Updates the item with id *6* and write the changes to the music file. The body should be a JSON object +containing the changes to the object. + +Returns the updated JSON representation. :: + + { + "id": 6, + "title": "A Song", + ... + } ``GET /item/6,12,13`` +++++++++++++++++++++ @@ -192,6 +211,8 @@ the response is the same as for `GET /item/`_. It is *not guaranteed* that the response includes all the items requested. If a track is not found it is silently dropped from the response. +This endpoint also supports *DELETE* and *PATCH* methods as above, to operate on all +items of the list. ``GET /item/path/...`` ++++++++++++++++++++++ @@ -221,6 +242,8 @@ Path elements are joined as parts of a query. For example, To specify literal path separators in a query, use a backslash instead of a slash. +This endpoint also supports *DELETE* and *PATCH* methods as above, to operate on all +items returned by the query. ``GET /item/6/file`` ++++++++++++++++++++ @@ -238,10 +261,16 @@ For albums, the following endpoints are provided: * ``GET /album/5`` +* ``DELETE /album/5`` + * ``GET /album/5,7`` +* ``DELETE /album/5,7`` + * ``GET /album/query/querystring`` +* ``DELETE /album/query/querystring`` + The interface and response format is similar to the item API, except replacing the encapsulation key ``"items"`` with ``"albums"`` when requesting ``/album/`` or ``/album/5,7``. In addition we can request the cover art of an album with From d1f93a26a6ba54efbb85efac74fe74c4b92e576a Mon Sep 17 00:00:00 2001 From: Cadel Watson Date: Sun, 20 Sep 2020 11:30:12 +1000 Subject: [PATCH 31/65] Fix lint errors --- beetsplug/web/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 0ce5f2c4c..a982809c4 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -21,7 +21,7 @@ from beets import ui from beets import util import beets.library import flask -from flask import g, make_response, jsonify +from flask import g, jsonify from werkzeug.routing import BaseConverter, PathConverter import os from unidecode import unidecode @@ -92,7 +92,9 @@ def is_expand(): def is_delete(): - """Returns whether the current delete request should remove the selected files.""" + """Returns whether the current delete request should remove the selected + files. + """ return flask.request.args.get('delete') is not None @@ -345,6 +347,7 @@ def item_unique_field_values(key): def get_album(id): return g.lib.get_album(id) + @app.route('/album/') @app.route('/album/query/') @resource_list('albums') From ad399b3caad7897cff4208c0229cdc6a50597dc6 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 29 Sep 2020 07:43:25 -0500 Subject: [PATCH 32/65] Add beats-ibroadcast to list of external plugins --- docs/plugins/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index aab922fcd..1d5d1b815 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -283,6 +283,8 @@ Here are a few of the plugins written by the beets community: * `beets-follow`_ lets you check for new albums from artists you like. +* `beets-ibroadcast`_ uploads tracks to the `iBroadcast`_ cloud service. + * `beets-setlister`_ generate playlists from the setlists of a given artist. * `beets-noimport`_ adds and removes directories from the incremental import skip list. @@ -336,6 +338,8 @@ Here are a few of the plugins written by the beets community: .. _beet-amazon: https://github.com/jmwatte/beet-amazon .. _beets-alternatives: https://github.com/geigerzaehler/beets-alternatives .. _beets-follow: https://github.com/nolsto/beets-follow +.. _beets-ibroadcast: https://github.com/ctrueden/beets-ibroadcast +.. _iBroadcast: https://ibroadcast.com/ .. _beets-setlister: https://github.com/tomjaspers/beets-setlister .. _beets-noimport: https://gitlab.com/tiago.dias/beets-noimport .. _whatlastgenre: https://github.com/YetAnotherNerd/whatlastgenre/tree/master/plugin/beets From d9582f4bea092836f8d86aab91c60151f824e13b Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 30 Sep 2020 17:36:23 +0200 Subject: [PATCH 33/65] export: Add --format=jsonlines option This adds support for the JSON Lines format as documented at https://jsonlines.org/. In this mode the data is output incrementally, whereas the other modes load every item into memory and don't produce output until the end. --- beetsplug/export.py | 23 +++++++++++++++++++---- docs/changelog.rst | 2 +- docs/plugins/export.rst | 5 +++-- test/test_export.py | 11 +++++++++++ 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/beetsplug/export.py b/beetsplug/export.py index 8d98d0ba2..957180db2 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -54,6 +54,14 @@ class ExportPlugin(BeetsPlugin): 'sort_keys': True } }, + 'jsonlines': { + # JSON Lines formatting options. + 'formatting': { + 'ensure_ascii': False, + 'separators': (',', ': '), + 'sort_keys': True + } + }, 'csv': { # CSV module formatting options. 'formatting': { @@ -95,7 +103,7 @@ class ExportPlugin(BeetsPlugin): ) cmd.parser.add_option( u'-f', u'--format', default='json', - help=u"the output format: json (default), csv, or xml" + help=u"the output format: json (default), jsonlines, csv, or xml" ) return [cmd] @@ -103,6 +111,7 @@ class ExportPlugin(BeetsPlugin): file_path = opts.output file_mode = 'a' if opts.append else 'w' file_format = opts.format or self.config['default_format'].get(str) + file_format_is_line_based = (file_format == 'jsonlines') format_options = self.config[file_format]['formatting'].get(dict) export_format = ExportFormat.factory( @@ -130,9 +139,14 @@ class ExportPlugin(BeetsPlugin): continue data = key_filter(data) - items += [data] - export_format.export(items, **format_options) + if file_format_is_line_based: + export_format.export(data, **format_options) + else: + items += [data] + + if not file_format_is_line_based: + export_format.export(items, **format_options) class ExportFormat(object): @@ -147,7 +161,7 @@ class ExportFormat(object): @classmethod def factory(cls, file_type, **kwargs): - if file_type == "json": + if file_type in ["json", "jsonlines"]: return JsonFormat(**kwargs) elif file_type == "csv": return CSVFormat(**kwargs) @@ -167,6 +181,7 @@ class JsonFormat(ExportFormat): def export(self, data, **kwargs): json.dump(data, self.out_stream, cls=ExportEncoder, **kwargs) + self.out_stream.write('\n') class CSVFormat(ExportFormat): diff --git a/docs/changelog.rst b/docs/changelog.rst index 47b0398c0..e02bebfa2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,7 +25,7 @@ New features: `discogs_artistid` :bug: `3413` * :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; - which allows for the ability to export in json, csv and xml. + which allows for the ability to export in json, jsonlines, csv and xml. Thanks to :user:`austinmm`. :bug:`3402` * :doc:`/plugins/unimported`: lets you find untracked files in your library directory. diff --git a/docs/plugins/export.rst b/docs/plugins/export.rst index f3756718c..284d2b8b6 100644 --- a/docs/plugins/export.rst +++ b/docs/plugins/export.rst @@ -39,14 +39,15 @@ The ``export`` command has these command-line options: * ``--append``: Appends the data to the file instead of writing. -* ``--format`` or ``-f``: Specifies the format the data will be exported as. If not informed, JSON will be used by default. The format options include csv, json and xml. +* ``--format`` or ``-f``: Specifies the format the data will be exported as. If not informed, JSON will be used by default. The format options include csv, json, `jsonlines `_ and xml. Configuration ------------- To configure the plugin, make a ``export:`` section in your configuration file. -For JSON export, these options are available under the ``json`` key: +For JSON export, these options are available under the ``json`` and +``jsonlines`` keys: - **ensure_ascii**: Escape non-ASCII characters with ``\uXXXX`` entities. - **indent**: The number of spaces for indentation. diff --git a/test/test_export.py b/test/test_export.py index 779e74423..f0a8eb0f7 100644 --- a/test/test_export.py +++ b/test/test_export.py @@ -66,6 +66,17 @@ class ExportPluginTest(unittest.TestCase, TestHelper): self.assertTrue(key in json_data) self.assertEqual(val, json_data[key]) + def test_jsonlines_output(self): + item1 = self.create_item() + out = self.execute_command( + format_type='jsonlines', + artist=item1.artist + ) + json_data = json.loads(out) + for key, val in self.test_values.items(): + self.assertTrue(key in json_data) + self.assertEqual(val, json_data[key]) + def test_csv_output(self): item1 = self.create_item() out = self.execute_command( From bfb954877aa2e368a5a6fd1b47070adf7da812b6 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 5 Oct 2020 08:53:51 -0400 Subject: [PATCH 34/65] Avoid mysterious namespace path problem (#3717) I'm still not sure what causes this problem (or more pertinently, what makes this problem *not* happen for me and others), but this should work around it regardless. --- beets/ui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 09f30c109..d882e06fb 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1114,7 +1114,7 @@ def _load_plugins(config): # Extend the `beetsplug` package to include the plugin paths. import beetsplug - beetsplug.__path__ = paths + beetsplug.__path__ + beetsplug.__path__ = paths + list(beetsplug.__path__) # For backwards compatibility, also support plugin paths that # *contain* a `beetsplug` package. From f0ea9da7b26cedfd8554715d7526273b90a76c6b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Sun, 4 Oct 2020 20:47:17 -0500 Subject: [PATCH 35/65] In verbose mode, log when files are ignored --- beets/util/__init__.py | 4 ++++ docs/changelog.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/beets/util/__init__.py b/beets/util/__init__.py index bb84aedc7..384609ee6 100644 --- a/beets/util/__init__.py +++ b/beets/util/__init__.py @@ -197,6 +197,10 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None): skip = False for pat in ignore: if fnmatch.fnmatch(base, pat): + if logger: + logger.debug(u'ignoring {0} due to ignore rule {1}'.format( + base, pat + )) skip = True break if skip: diff --git a/docs/changelog.rst b/docs/changelog.rst index e02bebfa2..4122b2f51 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -204,6 +204,8 @@ Fixes: wiping out their beets database. Thanks to user: `logan-arens`. :bug:`1934` +* ``beet import`` now logs which files are ignored when in debug mode. + :bug:`3764` * :doc:`/plugins/bpd`: Fix the transition to next track when in consume mode. Thanks to :user:`aereaux`. :bug:`3437` From d115f3679b8c884b1a9a7c1bcdeab2b66b2ac908 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 5 Oct 2020 22:08:38 +0100 Subject: [PATCH 36/65] Remove LyricWiki source LyricWiki was shut down on 2020/09/21 and no longer serves lyrics. --- beetsplug/lyrics.py | 28 +--------------------------- docs/changelog.rst | 1 + docs/plugins/lyrics.rst | 5 ++--- test/test_lyrics.py | 3 --- 4 files changed, 4 insertions(+), 33 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 16696d425..e216f33c1 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -55,7 +55,6 @@ except ImportError: from beets import plugins from beets import ui -from beets import util import beets DIV_RE = re.compile(r'<(/?)div>?', re.I) @@ -441,30 +440,6 @@ class Genius(Backend): return lyrics_div.get_text() -class LyricsWiki(SymbolsReplaced): - """Fetch lyrics from LyricsWiki.""" - - if util.SNI_SUPPORTED: - URL_PATTERN = 'https://lyrics.wikia.com/%s:%s' - else: - URL_PATTERN = 'http://lyrics.wikia.com/%s:%s' - - def fetch(self, artist, title): - url = self.build_url(artist, title) - html = self.fetch_url(url) - if not html: - return - - # Get the HTML fragment inside the appropriate HTML element and then - # extract the text from it. - html_frag = extract_text_in(html, u"
") - if html_frag: - lyrics = _scrape_strip_cruft(html_frag, True) - - if lyrics and 'Unfortunately, we are not licensed' not in lyrics: - return lyrics - - def remove_credits(text): """Remove first/last line of text if it contains the word 'lyrics' eg 'Lyrics by songsdatabase.com' @@ -656,10 +631,9 @@ class Google(Backend): class LyricsPlugin(plugins.BeetsPlugin): - SOURCES = ['google', 'lyricwiki', 'musixmatch', 'genius'] + SOURCES = ['google', 'musixmatch', 'genius'] SOURCE_BACKENDS = { 'google': Google, - 'lyricwiki': LyricsWiki, 'musixmatch': MusiXmatch, 'genius': Genius, } diff --git a/docs/changelog.rst b/docs/changelog.rst index 4122b2f51..e33299fab 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -153,6 +153,7 @@ New features: * ``beet remove`` now also allows interactive selection of items from the query similar to ``beet modify`` * :doc:`/plugins/web`: add DELETE and PATCH methods for modifying items +* :doc:`/plugins/lyrics`: Removed LyricWiki source (shut down on 21/09/2020). Fixes: diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 942497a7c..b71764042 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -2,10 +2,9 @@ Lyrics Plugin ============= The ``lyrics`` plugin fetches and stores song lyrics from databases on the Web. -Namely, the current version of the plugin uses `Lyric Wiki`_, -`Musixmatch`_, `Genius.com`_, and, optionally, the Google custom search API. +Namely, the current version of the plugin uses `Musixmatch`_, `Genius.com`_, +and, optionally, the Google custom search API. -.. _Lyric Wiki: https://lyrics.wikia.com/ .. _Musixmatch: https://www.musixmatch.com/ .. _Genius.com: https://genius.com/ diff --git a/test/test_lyrics.py b/test/test_lyrics.py index e0ec1e548..5fce1c476 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -268,7 +268,6 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): DEFAULT_SONG = dict(artist=u'The Beatles', title=u'Lady Madonna') DEFAULT_SOURCES = [ - dict(DEFAULT_SONG, backend=lyrics.LyricsWiki), # dict(artist=u'Santana', title=u'Black magic woman', # backend=lyrics.MusiXmatch), dict(DEFAULT_SONG, backend=lyrics.Genius), @@ -295,8 +294,6 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(DEFAULT_SONG, url='http://www.lyricsmania.com/', path='lady_madonna_lyrics_the_beatles.html'), - dict(DEFAULT_SONG, url=u'http://lyrics.wikia.com/', - path=u'The_Beatles:Lady_Madonna'), dict(DEFAULT_SONG, url=u'http://www.lyricsmode.com', path=u'/lyrics/b/beatles/lady_madonna.html'), From 580495f1d67f7e84a8ed173c1463c98f7465539b Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Mon, 5 Oct 2020 22:51:14 +0100 Subject: [PATCH 37/65] Simplify MusiXmatch backend and remove unused code --- beetsplug/lyrics.py | 49 +++++---------------------------------------- 1 file changed, 5 insertions(+), 44 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index e216f33c1..5591598ae 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -144,39 +144,6 @@ def extract_text_between(html, start_marker, end_marker): return html -def extract_text_in(html, starttag): - """Extract the text from a
tag in the HTML starting with - ``starttag``. Returns None if parsing fails. - """ - # Strip off the leading text before opening tag. - try: - _, html = html.split(starttag, 1) - except ValueError: - return - - # Walk through balanced DIV tags. - level = 0 - parts = [] - pos = 0 - for match in DIV_RE.finditer(html): - if match.group(1): # Closing tag. - level -= 1 - if level == 0: - pos = match.end() - else: # Opening tag. - if level == 0: - parts.append(html[pos:match.start()]) - level += 1 - - if level == -1: - parts.append(html[pos:match.start()]) - break - else: - print(u'no closing tag found!') - return - return u''.join(parts) - - def search_pairs(item): """Yield a pairs of artists and titles to search for. @@ -295,9 +262,9 @@ class Backend(object): raise NotImplementedError() -class SymbolsReplaced(Backend): +class MusiXmatch(Backend): REPLACEMENTS = { - r'\s+': '_', + r'\s+': '-', '<': 'Less_Than', '>': 'Greater_Than', '#': 'Number_', @@ -305,20 +272,14 @@ class SymbolsReplaced(Backend): r'[\]\}]': ')', } + URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s' + @classmethod def _encode(cls, s): for old, new in cls.REPLACEMENTS.items(): s = re.sub(old, new, s) - return super(SymbolsReplaced, cls)._encode(s) - - -class MusiXmatch(SymbolsReplaced): - REPLACEMENTS = dict(SymbolsReplaced.REPLACEMENTS, **{ - r'\s+': '-' - }) - - URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s' + return super(MusiXmatch, cls)._encode(s) def fetch(self, artist, title): url = self.build_url(artist, title) From 713840849198c88da12a11d25430c507bacffc86 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Tue, 6 Oct 2020 00:00:27 +0100 Subject: [PATCH 38/65] Skip AZLyrics on GitHub actions --- test/test_lyrics.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index e0ec1e548..1a47e29c3 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -271,7 +271,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(DEFAULT_SONG, backend=lyrics.LyricsWiki), # dict(artist=u'Santana', title=u'Black magic woman', # backend=lyrics.MusiXmatch), - dict(DEFAULT_SONG, backend=lyrics.Genius), + dict(DEFAULT_SONG, backend=lyrics.Genius, + # GitHub actions is on some form of Cloudflare blacklist. + skip=os.environ.get('GITHUB_ACTIONS') == 'true'), ] GOOGLE_SOURCES = [ @@ -280,7 +282,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): path=u'/lyrics/view/the_beatles/lady_madonna'), dict(DEFAULT_SONG, url=u'http://www.azlyrics.com', - path=u'/lyrics/beatles/ladymadonna.html'), + path=u'/lyrics/beatles/ladymadonna.html', + # AZLyrics returns a 403 on GitHub actions. + skip=os.environ.get('GITHUB_ACTIONS') == 'true'), dict(DEFAULT_SONG, url=u'http://www.chartlyrics.com', path=u'/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx'), @@ -330,11 +334,8 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): """Test default backends with songs known to exist in respective databases. """ errors = [] - # GitHub actions seems to be on a Cloudflare blacklist, so we can't - # contact genius. - sources = [s for s in self.DEFAULT_SOURCES if - s['backend'] != lyrics.Genius or - os.environ.get('GITHUB_ACTIONS') != 'true'] + # Don't test any sources marked as skipped. + sources = [s for s in self.DEFAULT_SOURCES if not s.get("skip", False)] for s in sources: res = s['backend'](self.plugin.config, self.plugin._log).fetch( s['artist'], s['title']) @@ -349,7 +350,9 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): """Test if lyrics present on websites registered in beets google custom search engine are correctly scraped. """ - for s in self.GOOGLE_SOURCES: + # Don't test any sources marked as skipped. + sources = [s for s in self.GOOGLE_SOURCES if not s.get("skip", False)] + for s in sources: url = s['url'] + s['path'] res = lyrics.scrape_lyrics_from_html( raw_backend.fetch_url(url)) From 90cf26389e93960861b012142198b99bbc7340ec Mon Sep 17 00:00:00 2001 From: Nathan Date: Thu, 8 Oct 2020 00:35:03 -0700 Subject: [PATCH 39/65] Swap links --- docs/plugins/chroma.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/chroma.rst b/docs/plugins/chroma.rst index a6b60e6d8..9315a1b8c 100644 --- a/docs/plugins/chroma.rst +++ b/docs/plugins/chroma.rst @@ -80,8 +80,8 @@ You will also need a mechanism for decoding audio files supported by the .. _audioread: https://github.com/beetbox/audioread .. _pyacoustid: https://github.com/beetbox/pyacoustid .. _FFmpeg: https://ffmpeg.org/ -.. _MAD: https://spacepants.org/src/pymad/ -.. _pymad: https://www.underbit.com/products/mad/ +.. _pymad: https://spacepants.org/src/pymad/ +.. _MAD: https://www.underbit.com/products/mad/ .. _Core Audio: https://developer.apple.com/technologies/mac/audio-and-video.html .. _Gstreamer: https://gstreamer.freedesktop.org/ .. _PyGObject: https://wiki.gnome.org/Projects/PyGObject From a63ee0e1a7f27710279941bc8dce23de992c694c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 18 Oct 2020 07:19:21 -0400 Subject: [PATCH 40/65] Try normalizing the dbcore String type Strings were not being normalized, unlike some other types, leading to downstream problems like #3773. --- beets/dbcore/types.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index 5aa2b9812..abda7adcd 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -207,6 +207,12 @@ class String(Type): sql = u'TEXT' query = query.SubstringQuery + def normalize(self, value): + if value is None: + return self.model_type() + else: + return self.model_type(value) + class Boolean(Type): """A boolean type. From e99becb41ed30025aeeebb100e68e478b947986f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 18 Oct 2020 07:32:44 -0400 Subject: [PATCH 41/65] Use Unicode in lyrics tests --- test/test_lyrics.py | 63 +++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 833b86b3a..95b094e98 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -48,71 +48,72 @@ class LyricsPluginTest(unittest.TestCase): lyrics.LyricsPlugin() def test_search_artist(self): - item = Item(artist='Alice ft. Bob', title='song') - self.assertIn(('Alice ft. Bob', ['song']), + item = Item(artist=u'Alice ft. Bob', title=u'song') + self.assertIn((u'Alice ft. Bob', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Alice', ['song']), + self.assertIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice feat Bob', title='song') - self.assertIn(('Alice feat Bob', ['song']), + item = Item(artist=u'Alice feat Bob', title=u'song') + self.assertIn((u'Alice feat Bob', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Alice', ['song']), + self.assertIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice feat. Bob', title='song') - self.assertIn(('Alice feat. Bob', ['song']), + item = Item(artist=u'Alice feat. Bob', title=u'song') + self.assertIn((u'Alice feat. Bob', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Alice', ['song']), + self.assertIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice feats Bob', title='song') - self.assertIn(('Alice feats Bob', ['song']), + item = Item(artist=u'Alice feats Bob', title=u'song') + self.assertIn((u'Alice feats Bob', [u'song']), lyrics.search_pairs(item)) - self.assertNotIn(('Alice', ['song']), + self.assertNotIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice featuring Bob', title='song') - self.assertIn(('Alice featuring Bob', ['song']), + item = Item(artist=u'Alice featuring Bob', title=u'song') + self.assertIn((u'Alice featuring Bob', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Alice', ['song']), + self.assertIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice & Bob', title='song') - self.assertIn(('Alice & Bob', ['song']), + item = Item(artist=u'Alice & Bob', title=u'song') + self.assertIn((u'Alice & Bob', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Alice', ['song']), + self.assertIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice and Bob', title='song') - self.assertIn(('Alice and Bob', ['song']), + item = Item(artist=u'Alice and Bob', title=u'song') + self.assertIn((u'Alice and Bob', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Alice', ['song']), + self.assertIn((u'Alice', [u'song']), lyrics.search_pairs(item)) - item = Item(artist='Alice and Bob', title='song') - self.assertEqual(('Alice and Bob', ['song']), + item = Item(artist=u'Alice and Bob', title=u'song') + self.assertEqual((u'Alice and Bob', [u'song']), list(lyrics.search_pairs(item))[0]) def test_search_artist_sort(self): - item = Item(artist='CHVRCHΞS', title='song', artist_sort='CHVRCHES') - self.assertIn(('CHVRCHΞS', ['song']), + item = Item(artist=u'CHVRCHΞS', title=u'song', artist_sort=u'CHVRCHES') + self.assertIn((u'CHVRCHΞS', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('CHVRCHES', ['song']), + self.assertIn((u'CHVRCHES', [u'song']), lyrics.search_pairs(item)) # Make sure that the original artist name is still the first entry - self.assertEqual(('CHVRCHΞS', ['song']), + self.assertEqual((u'CHVRCHΞS', [u'song']), list(lyrics.search_pairs(item))[0]) - item = Item(artist='横山克', title='song', artist_sort='Masaru Yokoyama') - self.assertIn(('横山克', ['song']), + item = Item(artist=u'横山克', title=u'song', + artist_sort=u'Masaru Yokoyama') + self.assertIn((u'横山克', [u'song']), lyrics.search_pairs(item)) - self.assertIn(('Masaru Yokoyama', ['song']), + self.assertIn((u'Masaru Yokoyama', [u'song']), lyrics.search_pairs(item)) # Make sure that the original artist name is still the first entry - self.assertEqual(('横山克', ['song']), + self.assertEqual((u'横山克', [u'song']), list(lyrics.search_pairs(item))[0]) def test_search_pairs_multi_titles(self): From a9ba25439f15f502b5b4947ee3c896ab9fa6594a Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Fri, 18 Sep 2020 15:48:14 +0100 Subject: [PATCH 42/65] Add --plugins flag --- .github/ISSUE_TEMPLATE/bug-report.md | 6 ++++++ beets/ui/__init__.py | 17 +++++++++++++---- docs/changelog.rst | 1 + docs/reference/cli.rst | 4 ++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 646243812..6fae156b1 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -35,6 +35,12 @@ Here's a link to the music files that trigger the bug (if relevant): * beets version: * Turning off plugins made problem go away (yes/no): + + My configuration (output of `beet config`) is: ```yaml diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index d882e06fb..28879a731 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1102,8 +1102,8 @@ optparse.Option.ALWAYS_TYPED_ACTIONS += ('callback',) # The main entry point and bootstrapping. -def _load_plugins(config): - """Load the plugins specified in the configuration. +def _load_plugins(options, config): + """Load the plugins specified on the command line or in the configuration. """ paths = config['pluginpath'].as_str_seq(split=False) paths = [util.normpath(p) for p in paths] @@ -1120,7 +1120,14 @@ def _load_plugins(config): # *contain* a `beetsplug` package. sys.path += paths - plugins.load_plugins(config['plugins'].as_str_seq()) + # If we were given any plugins on the command line, use those. + if options.plugins is not None: + plugin_list = (options.plugins.split(',') + if len(options.plugins) > 0 else []) + else: + plugin_list = config['plugins'].as_str_seq() + + plugins.load_plugins(plugin_list) plugins.send("pluginload") return plugins @@ -1135,7 +1142,7 @@ def _setup(options, lib=None): config = _configure(options) - plugins = _load_plugins(config) + plugins = _load_plugins(options, config) # Get the default subcommands. from beets.ui.commands import default_commands @@ -1233,6 +1240,8 @@ def _raw_main(args, lib=None): help=u'log more details (use twice for even more)') parser.add_option('-c', '--config', dest='config', help=u'path to configuration file') + parser.add_option('-p', '--plugins', dest='plugins', + help=u'a comma-separated list of plugins to load') parser.add_option('-h', '--help', dest='help', action='store_true', help=u'show this help message and exit') parser.add_option('--version', dest='version', action='store_true', diff --git a/docs/changelog.rst b/docs/changelog.rst index e33299fab..1a1c30ce5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -154,6 +154,7 @@ New features: similar to ``beet modify`` * :doc:`/plugins/web`: add DELETE and PATCH methods for modifying items * :doc:`/plugins/lyrics`: Removed LyricWiki source (shut down on 21/09/2020). +* Added a ``--plugins`` (or ``-p``) flag to specify a list of plugins at startup. Fixes: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 2062193ab..5d2b834b7 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -440,6 +440,10 @@ import ...``. configuration options entirely, the two are merged. Any individual options set in this config file will override the corresponding settings in your base configuration. +* ``-p plugins``: specify a comma-separated list of plugins to enable. If + specified, the plugin list in your configuration is ignored. The long form + of this argument also allows specifying no plugins, effectively disabling + all plugins: ``--plugins=``. Beets also uses the ``BEETSDIR`` environment variable to look for configuration and data. From 7621cdfeb9f085f6634ea00592d111fef9988139 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Thu, 22 Oct 2020 00:02:55 +0100 Subject: [PATCH 43/65] Fix GitHub user page regex --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 018ef5397..0066c3f82 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ extlinks = { linkcheck_ignore = [ r'https://github.com/beetbox/beets/issues/', - r'https://github.com/\w+$', # ignore user pages + r'https://github.com/[^/]+$', # ignore user pages r'.*localhost.*', r'https://www.musixmatch.com/', # blocks requests ] From 14a56303815573666b69f3e60fdbeb940a0bf11e Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Thu, 22 Oct 2020 00:03:18 +0100 Subject: [PATCH 44/65] Add genius to linkcheck ignore list --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 0066c3f82..f77838e81 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,6 +33,7 @@ linkcheck_ignore = [ r'https://github.com/[^/]+$', # ignore user pages r'.*localhost.*', r'https://www.musixmatch.com/', # blocks requests + r'https://genius.com/', # blocks requests ] # Options for HTML output From 950dbe3b9f5a3a4c5ff6689c6312e1862d93cada Mon Sep 17 00:00:00 2001 From: "Kyle R. Conway" Date: Fri, 23 Oct 2020 00:41:58 -0500 Subject: [PATCH 45/65] Update faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A couple of suggested modifications primarily to clarify that this unordered list is of **options** to install the latest version―not a series of commands to complete a single install. Also suggested a layout change to explain the series of commands in option b. --- docs/faq.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index eeab6c1ef..8af404fce 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -137,13 +137,16 @@ it's helpful to run on the "bleeding edge". To run the latest source: ``pip uninstall beets``. 2. Install from source. There are a few easy ways to do this: - - Use ``pip`` to install the latest snapshot tarball: just type - ``pip install https://github.com/beetbox/beets/tarball/master``. - - Grab the source using Git: - ``git clone https://github.com/beetbox/beets.git``. Then - ``cd beets`` and type ``python setup.py install``. - - Use ``pip`` to install an "editable" version of beets based on an - automatic source checkout. For example, run + - **Option 1** - Use ``pip`` to install the latest snapshot tarball: + just type: ``pip install https://github.com/beetbox/beets/tarball/master``. + - **Option 2** - Grab the source using Git: + + 1. First clone the repository ``git clone https://github.com/beetbox/beets.git``. + 2. Then enter the beets directory ``cd beets`` + 3. Finally install ``python setup.py install``. + + - **Option 3** - Use ``pip`` to install an "editable" version of beets based on an + automatic source checkout. For example, run: ``pip install -e git+https://github.com/beetbox/beets#egg=beets`` to clone beets and install it, allowing you to modify the source in-place to try out changes. From 29bab249c68d5112b90cb3d4932aeccc53d62bbf Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 23 Oct 2020 10:10:21 -0400 Subject: [PATCH 46/65] Use `null` instead of `model_type()` --- beets/dbcore/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/dbcore/types.py b/beets/dbcore/types.py index abda7adcd..c85eb1a50 100644 --- a/beets/dbcore/types.py +++ b/beets/dbcore/types.py @@ -209,7 +209,7 @@ class String(Type): def normalize(self, value): if value is None: - return self.model_type() + return self.null else: return self.model_type(value) From d0cbf098c60ca27f3e75cf899026979f19dcca00 Mon Sep 17 00:00:00 2001 From: "Kyle R. Conway" Date: Fri, 23 Oct 2020 11:06:16 -0500 Subject: [PATCH 47/65] Update faq.rst Thanks for the feedback! That's a lot clearer. I made the headers bold to ensure they still stood out as clear options if that's okay. Let me know if there are any other recommendations here. I hit this issue -- https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=964245 -- while testing for another user and found the helpful guide but misread it a couple of times. Thanks! --- docs/faq.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 8af404fce..deecfabcf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -135,18 +135,20 @@ it's helpful to run on the "bleeding edge". To run the latest source: 1. Uninstall beets. If you installed using ``pip``, you can just run ``pip uninstall beets``. -2. Install from source. There are a few easy ways to do this: +2. Install from source. Choose **one** of the following methods: - - **Option 1** - Use ``pip`` to install the latest snapshot tarball: + - **Use ``pip`` to install the latest snapshot tarball:** just type: ``pip install https://github.com/beetbox/beets/tarball/master``. - - **Option 2** - Grab the source using Git: + - **Grab the source using Git:** 1. First clone the repository ``git clone https://github.com/beetbox/beets.git``. 2. Then enter the beets directory ``cd beets`` 3. Finally install ``python setup.py install``. - - **Option 3** - Use ``pip`` to install an "editable" version of beets based on an - automatic source checkout. For example, run: + - **Use ``pip`` to install an "editable" version of beets based on an + automatic source checkout.** + + For example, run: ``pip install -e git+https://github.com/beetbox/beets#egg=beets`` to clone beets and install it, allowing you to modify the source in-place to try out changes. From 013155a9d89df454e4e2ab535489b51513b0f62a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Fri, 23 Oct 2020 20:33:22 -0400 Subject: [PATCH 48/65] Simplifications to #3778 --- docs/faq.rst | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index deecfabcf..f47233430 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -135,20 +135,15 @@ it's helpful to run on the "bleeding edge". To run the latest source: 1. Uninstall beets. If you installed using ``pip``, you can just run ``pip uninstall beets``. -2. Install from source. Choose **one** of the following methods: +2. Install from source. Choose one of these methods: - - **Use ``pip`` to install the latest snapshot tarball:** - just type: ``pip install https://github.com/beetbox/beets/tarball/master``. - - **Grab the source using Git:** - - 1. First clone the repository ``git clone https://github.com/beetbox/beets.git``. - 2. Then enter the beets directory ``cd beets`` - 3. Finally install ``python setup.py install``. - - - **Use ``pip`` to install an "editable" version of beets based on an - automatic source checkout.** - - For example, run: + - Use ``pip`` to install the latest snapshot tarball. Type: + ``pip install https://github.com/beetbox/beets/tarball/master`` + - Grab the source using git. First, clone the repository: + ``git clone https://github.com/beetbox/beets.git``. + Then, ``cd beets`` and ``python setup.py install``. + - Use ``pip`` to install an "editable" version of beets based on an + automatic source checkout. For example, run ``pip install -e git+https://github.com/beetbox/beets#egg=beets`` to clone beets and install it, allowing you to modify the source in-place to try out changes. From 8e17d445ff3b814d6d3c4fba66efd020539ba5f1 Mon Sep 17 00:00:00 2001 From: Georg Schwitalla Date: Sat, 24 Oct 2020 12:47:59 +0200 Subject: [PATCH 49/65] Added a check if config_out was empty --- beets/ui/commands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 49c4b4dc6..2962362a1 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1691,8 +1691,10 @@ def config_func(lib, opts, args): # Dump configuration. else: config_out = config.dump(full=opts.defaults, redact=opts.redact) - print_(util.text_string(config_out)) - + if config_out != '{}\n': + print_(util.text_string(config_out)) + else: + print("Empty Configuration") def config_edit(): """Open a program to edit the user configuration. From 9c1b39a96e96faffd030f04f43539f39dc1a8e00 Mon Sep 17 00:00:00 2001 From: Georg Schwitalla Date: Sat, 24 Oct 2020 12:56:37 +0200 Subject: [PATCH 50/65] had a slight typo --- beets/ui/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 2962362a1..5df0b0c4b 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1694,7 +1694,7 @@ def config_func(lib, opts, args): if config_out != '{}\n': print_(util.text_string(config_out)) else: - print("Empty Configuration") + print("Empty configuration") def config_edit(): """Open a program to edit the user configuration. From 3c8afd9a0e3ea684e2b99c4ac03358ced4711f18 Mon Sep 17 00:00:00 2001 From: Georg Schwitalla Date: Sat, 24 Oct 2020 13:07:27 +0200 Subject: [PATCH 51/65] Added the changelog entry --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e33299fab..c752f204b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* When config is printed with no available configuration a new message is printed. * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. From e898f4396e1dd762602c659dddce305e6347e08e Mon Sep 17 00:00:00 2001 From: Georg Schwitalla Date: Sat, 24 Oct 2020 13:19:29 +0200 Subject: [PATCH 52/65] forgot the second blank line --- beets/ui/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 5df0b0c4b..efa6e2e16 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1696,6 +1696,7 @@ def config_func(lib, opts, args): else: print("Empty configuration") + def config_edit(): """Open a program to edit the user configuration. An empty config file is created if no existing config file exists. From d51e44b9e65dd802b607e8a2b40530dab1316549 Mon Sep 17 00:00:00 2001 From: Georg Schwitalla Date: Sat, 24 Oct 2020 13:29:44 +0200 Subject: [PATCH 53/65] Issue #3569 changed the the text from Keep both to keep all --- beets/ui/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 49c4b4dc6..f8c2decab 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -807,7 +807,7 @@ class TerminalImportSession(importer.ImportSession): )) sel = ui.input_options( - (u'Skip new', u'Keep both', u'Remove old', u'Merge all') + (u'Skip new', u'Keep all', u'Remove old', u'Merge all') ) if sel == u's': From e7013f9f6ff1cccbf752c33489686254173ec70d Mon Sep 17 00:00:00 2001 From: Georg Schwitalla Date: Sat, 24 Oct 2020 13:34:23 +0200 Subject: [PATCH 54/65] Added the changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e33299fab..0de5fda95 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog New features: +* When importing a duplicate album it ask if it should "Keep all" instead of "Keep both". * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. From e49ee7c6867227c809ba2cea5f5e05c3b629bc3c Mon Sep 17 00:00:00 2001 From: Sudo-kun Date: Sun, 25 Oct 2020 11:02:12 +0100 Subject: [PATCH 55/65] Removed white-space sensitivity in the if-clause --- beets/ui/commands.py | 2 +- docs/changelog.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index efa6e2e16..f86984350 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -1691,7 +1691,7 @@ def config_func(lib, opts, args): # Dump configuration. else: config_out = config.dump(full=opts.defaults, redact=opts.redact) - if config_out != '{}\n': + if config_out.strip() != '{}': print_(util.text_string(config_out)) else: print("Empty configuration") diff --git a/docs/changelog.rst b/docs/changelog.rst index c752f204b..cc86b0292 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changelog New features: * When config is printed with no available configuration a new message is printed. + :bug: `3779` * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. From 272463aa4a26f0680f721eb2d13be32f3bc223db Mon Sep 17 00:00:00 2001 From: Sudo-kun Date: Sun, 25 Oct 2020 11:06:00 +0100 Subject: [PATCH 56/65] Edited the changelog entry to contain the Issue number --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0de5fda95..ac7435abc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ Changelog New features: * When importing a duplicate album it ask if it should "Keep all" instead of "Keep both". + :bug:`3569` * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. From 829d08d9b8e800701ac4d9dd26aad6aae128c07c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 25 Oct 2020 06:55:30 -0400 Subject: [PATCH 57/65] Fix some ReST syntax --- docs/changelog.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cc86b0292..ea5a19fae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ Changelog New features: * When config is printed with no available configuration a new message is printed. - :bug: `3779` + :bug:`3779` * :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. @@ -22,10 +22,10 @@ New features: * :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to allow downloading of higher resolution iTunes artwork (at the expense of file size). - :bug: `3391` + :bug:`3391` * :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and `discogs_artistid` - :bug: `3413` + :bug:`3413` * :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; which allows for the ability to export in json, jsonlines, csv and xml. Thanks to :user:`austinmm`. From 6e4bb3d4ed25b312a3e7164a8a1684d908442f0c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 25 Oct 2020 06:57:46 -0400 Subject: [PATCH 58/65] Update docs for #3783 --- docs/guides/tagger.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/tagger.rst b/docs/guides/tagger.rst index 467d605a4..d890f5c08 100644 --- a/docs/guides/tagger.rst +++ b/docs/guides/tagger.rst @@ -234,7 +234,7 @@ If beets finds an album or item in your library that seems to be the same as the one you're importing, you may see a prompt like this:: This album is already in the library! - [S]kip new, Keep both, Remove old, Merge all? + [S]kip new, Keep all, Remove old, Merge all? Beets wants to keep you safe from duplicates, which can be a real pain, so you have four choices in this situation. You can skip importing the new music, From 2cbec2d838bd594c8a3be5ba55c889af499b4139 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Mon, 26 Oct 2020 20:25:25 -0400 Subject: [PATCH 59/65] Changelog entry --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e65e6b1e9..49cf86330 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -266,6 +266,10 @@ Fixes: the current track in the queue. Thanks to :user:`aereaux`. :bug:`3722` +* String-typed fields are now normalized to string values, avoiding an + occasional crash when using both the :doc:`/plugins/fetchart` and the + :doc:`/plugins/discogs` together. + :bug:`3773` :bug:`3774` For plugin developers: From d70287df009fd366429dcf1cecc8f0b8a3f8bef1 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Fri, 23 Oct 2020 15:29:29 -0700 Subject: [PATCH 60/65] Add genre support using musicbrainz tags. This requires this PR: https://github.com/alastair/python-musicbrainzngs/pull/266 --- beets/autotag/mb.py | 5 +++++ docs/changelog.rst | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index ea8ef24da..06b088fd4 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -74,6 +74,8 @@ RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups', TRACK_INCLUDES = ['artists', 'aliases'] if 'work-level-rels' in musicbrainzngs.VALID_INCLUDES['recording']: TRACK_INCLUDES += ['work-level-rels', 'artist-rels'] +if 'genres' in musicbrainzngs.VALID_INCLUDES['recording']: + RELEASE_INCLUDES += ['genres'] def track_url(trackid): @@ -415,6 +417,9 @@ def album_info(release): first_medium = release['medium-list'][0] info.media = first_medium.get('format') + if release.get('genre-list'): + info.genre = ';'.join(g['name'] for g in release['genre-list']) + info.decode() return info diff --git a/docs/changelog.rst b/docs/changelog.rst index e65e6b1e9..25b1d7b82 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -159,6 +159,11 @@ New features: * :doc:`/plugins/web`: add DELETE and PATCH methods for modifying items * :doc:`/plugins/lyrics`: Removed LyricWiki source (shut down on 21/09/2020). * Added a ``--plugins`` (or ``-p``) flag to specify a list of plugins at startup. +* Use musicbrainz genre tag api to get genre information. This currently + depends on functionality that is currently unreleased in musicbrainzngs. + See https://github.com/alastair/python-musicbrainzngs/pull/247 and + https://github.com/alastair/python-musicbrainzngs/pull/266 . + Thanks to :user:`aereaux`. Fixes: From f2dabfef530b2fa26d0d8b95abe337a362420e30 Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Sun, 25 Oct 2020 16:21:28 +0000 Subject: [PATCH 61/65] Fix PIL image quality --- beets/util/artresizer.py | 5 +++++ docs/changelog.rst | 2 ++ 2 files changed, 7 insertions(+) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 8f14c8baf..c57918f16 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -77,6 +77,11 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0): im = Image.open(util.syspath(path_in)) size = maxwidth, maxwidth im.thumbnail(size, Image.ANTIALIAS) + + if quality == 0: + # Use PIL's default quality. + quality = -1 + im.save(util.py3_path(path_out), quality=quality) return path_out except IOError: diff --git a/docs/changelog.rst b/docs/changelog.rst index 65119d37d..340bdc9ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -265,6 +265,8 @@ Fixes: the current track in the queue. Thanks to :user:`aereaux`. :bug:`3722` +* Fix a bug causing PIL to generate poor quality JPEGs when resizing artwork. + :bug:`3743` For plugin developers: From d011a4e5778fcf1422d88b8ae9a3cbd89cc97670 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Wed, 28 Oct 2020 19:31:31 +0000 Subject: [PATCH 62/65] Update beets/autotag/mb.py Co-authored-by: Adrian Sampson --- beets/autotag/mb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 06b088fd4..211f4c42f 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -417,8 +417,9 @@ def album_info(release): first_medium = release['medium-list'][0] info.media = first_medium.get('format') - if release.get('genre-list'): - info.genre = ';'.join(g['name'] for g in release['genre-list']) + genres = release.get('genre-list') + if genres: + info.genre = ';'.join(g['name'] for g in genres) info.decode() return info From a79b239d4ffd4e4d005d8ad99c0624adecebdaee Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Thu, 29 Oct 2020 07:47:44 -0700 Subject: [PATCH 63/65] Add musicbrainz genre config option. --- beets/autotag/mb.py | 2 +- beets/config_default.yaml | 1 + docs/changelog.rst | 6 ++++-- docs/reference/config.rst | 8 ++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 211f4c42f..7952c5566 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -418,7 +418,7 @@ def album_info(release): info.media = first_medium.get('format') genres = release.get('genre-list') - if genres: + if config['musicbrainz']['genres'] and genres: info.genre = ';'.join(g['name'] for g in genres) info.decode() diff --git a/beets/config_default.yaml b/beets/config_default.yaml index c75778b80..f3e9acad1 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -105,6 +105,7 @@ musicbrainz: ratelimit_interval: 1.0 searchlimit: 5 extra_tags: [] + genres: no match: strong_rec_thresh: 0.04 diff --git a/docs/changelog.rst b/docs/changelog.rst index a31c6869b..2f12015a8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -159,9 +159,11 @@ New features: * :doc:`/plugins/web`: add DELETE and PATCH methods for modifying items * :doc:`/plugins/lyrics`: Removed LyricWiki source (shut down on 21/09/2020). * Added a ``--plugins`` (or ``-p``) flag to specify a list of plugins at startup. -* Use musicbrainz genre tag api to get genre information. This currently +* Use the musicbrainz genre tag api to get genre information. This currently depends on functionality that is currently unreleased in musicbrainzngs. - See https://github.com/alastair/python-musicbrainzngs/pull/247 and + Once the functionality has been released, you can enable it with the + ``use_mb_genres`` option. See + https://github.com/alastair/python-musicbrainzngs/pull/247 and https://github.com/alastair/python-musicbrainzngs/pull/266 . Thanks to :user:`aereaux`. diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 2f8cee3c9..021c3d2b7 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -356,6 +356,14 @@ Sets the albumartist for various-artist compilations. Defaults to ``'Various Artists'`` (the MusicBrainz standard). Affects other sources, such as :doc:`/plugins/discogs`, too. +.. _use_mb_genres: + +use_mb_genres +~~~~~~~~~~~~~ + +Use Musicbrainz genre tags to populate the ``genre`` tag. This will make it a +semicolon-separated list of all the genres tagged for the release on +musicbrainz. UI Options ---------- From 0a8c755a6bce46df234ff2bb1c7b26c8d11a75f1 Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Fri, 30 Oct 2020 14:36:28 +0000 Subject: [PATCH 64/65] Apply suggestions from code review Co-authored-by: Adrian Sampson --- docs/reference/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 021c3d2b7..57c03cf8e 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -361,9 +361,9 @@ Artists'`` (the MusicBrainz standard). Affects other sources, such as use_mb_genres ~~~~~~~~~~~~~ -Use Musicbrainz genre tags to populate the ``genre`` tag. This will make it a +Use MusicBrainz genre tags to populate the ``genre`` tag. This will make it a semicolon-separated list of all the genres tagged for the release on -musicbrainz. +MusicBrainz. UI Options ---------- From ceb046d60cc53ce7d11dd72b7f6cd7310de0daff Mon Sep 17 00:00:00 2001 From: Aidan Epstein Date: Thu, 29 Oct 2020 07:47:44 -0700 Subject: [PATCH 65/65] Add musicbrainz genre config option. --- docs/changelog.rst | 2 +- docs/reference/config.rst | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f12015a8..458e4b8d2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -162,7 +162,7 @@ New features: * Use the musicbrainz genre tag api to get genre information. This currently depends on functionality that is currently unreleased in musicbrainzngs. Once the functionality has been released, you can enable it with the - ``use_mb_genres`` option. See + ``genres`` option inside the ``musicbrainz`` config. See https://github.com/alastair/python-musicbrainzngs/pull/247 and https://github.com/alastair/python-musicbrainzngs/pull/266 . Thanks to :user:`aereaux`. diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 57c03cf8e..6aa9f5f53 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -356,15 +356,6 @@ Sets the albumartist for various-artist compilations. Defaults to ``'Various Artists'`` (the MusicBrainz standard). Affects other sources, such as :doc:`/plugins/discogs`, too. -.. _use_mb_genres: - -use_mb_genres -~~~~~~~~~~~~~ - -Use MusicBrainz genre tags to populate the ``genre`` tag. This will make it a -semicolon-separated list of all the genres tagged for the release on -MusicBrainz. - UI Options ---------- @@ -729,6 +720,17 @@ above example. Default: ``[]`` +.. _genres: + +genres +~~~~~~ + +Use MusicBrainz genre tags to populate the ``genre`` tag. This will make it a +semicolon-separated list of all the genres tagged for the release on +MusicBrainz. + +Default: ``no`` + .. _match-config: Autotagger Matching Options