From 72515448ad505ef0b73c0d0aa4aa424419b55755 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 02:38:46 +0200 Subject: [PATCH 001/130] Add fallback for item access to album's attributes Allows queries (especially for pathspecs) based on an album's flexattrs while operating on items. Fixes #2797. --- beets/dbcore/db.py | 29 +++++++++++++++-------------- beets/library.py | 37 ++++++++++++++++++++++++++++++++++++- test/test_ipfs.py | 5 +++-- test/test_library.py | 33 +++++++++++++++++++++++++++++++++ test/test_query.py | 13 ++++++++++++- 5 files changed, 99 insertions(+), 18 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 3195b52c9..5aa418632 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -345,9 +345,9 @@ class Model(object): """ return cls._fields.get(key) or cls._types.get(key) or types.DEFAULT - def __getitem__(self, key): - """Get the value for a field. Raise a KeyError if the field is - not available. + def _get(self, key, default=None, raise_=False): + """Get the value for a field, or `default`. Alternatively, + raise a KeyError if the field is not available. """ getters = self._getters() if key in getters: # Computed. @@ -359,8 +359,18 @@ class Model(object): return self._type(key).null elif key in self._values_flex: # Flexible. return self._values_flex[key] - else: + elif raise_: raise KeyError(key) + else: + return default + + get = _get + + def __getitem__(self, key): + """Get the value for a field. Raise a KeyError if the field is + not available. + """ + return self._get(key, raise_=True) def _setitem(self, key, value): """Assign the value for a field, return whether new and old value @@ -435,19 +445,10 @@ class Model(object): for key in self: yield key, self[key] - def get(self, key, default=None): - """Get the value for a given key or `default` if it does not - exist. - """ - if key in self: - return self[key] - else: - return default - def __contains__(self, key): """Determine whether `key` is an attribute on this object. """ - return key in self.keys(True) + return key in self.keys(computed=True) def __iter__(self): """Iterate over the available field names (excluding computed diff --git a/beets/library.py b/beets/library.py index bb49d0e99..5740f20ee 100644 --- a/beets/library.py +++ b/beets/library.py @@ -386,7 +386,7 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def album_keys(self): album_keys = [] if self.album: - for key in self.album.keys(True): + for key in self.album.keys(computed=True): if key in Album.item_keys \ or key not in self.item._fields.keys(): album_keys.append(key) @@ -571,6 +571,41 @@ class Item(LibModel): if changed and key in MediaFile.fields(): self.mtime = 0 # Reset mtime on dirty. + def __getitem__(self, key): + """Get the value for a field, falling back to the album if + necessary. Raise a KeyError if the field is not available. + """ + try: + return super(Item, self).__getitem__(key) + except KeyError: + album = self.get_album() + if album: + return album[key] + raise + + def keys(self, computed=False, with_album=True): + """Get a list of available field names. `with_album` + controls whether the album's fields are included. + """ + keys = super(Item, self).keys(computed=computed) + if with_album: + album = self.get_album() + if album: + keys += album.keys(computed=computed) + return keys + + def get(self, key, default=None, with_album=True): + """Get the value for a given key or `default` if it does not + exist. Set `with_album` to false to skip album fallback. + """ + try: + return self._get(key, default, raise_=with_album) + except KeyError: + album = self.get_album() + if album: + return album.get(key, default) + return default + def update(self, values): """Set all key/value pairs in the mapping. If mtime is specified, it is not reset (as it might otherwise be). diff --git a/test/test_ipfs.py b/test/test_ipfs.py index d670bfc25..2fe89e7e5 100644 --- a/test/test_ipfs.py +++ b/test/test_ipfs.py @@ -49,7 +49,7 @@ class IPFSPluginTest(unittest.TestCase, TestHelper): want_item = test_album.items()[2] for check_item in added_album.items(): try: - if check_item.ipfs: + if check_item.get('ipfs', with_album=False): ipfs_item = os.path.basename(want_item.path).decode( _fsencoding(), ) @@ -57,7 +57,8 @@ class IPFSPluginTest(unittest.TestCase, TestHelper): ipfs_item) want_path = bytestring_path(want_path) self.assertEqual(check_item.path, want_path) - self.assertEqual(check_item.ipfs, want_item.ipfs) + self.assertEqual(check_item.get('ipfs', with_album=False), + want_item.ipfs) self.assertEqual(check_item.title, want_item.title) found = True except AttributeError: diff --git a/test/test_library.py b/test/test_library.py index 4e3be878c..51171b1f8 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -132,6 +132,21 @@ class GetSetTest(_common.TestCase): def test_invalid_field_raises_attributeerror(self): self.assertRaises(AttributeError, getattr, self.i, u'xyzzy') + def test_album_fallback(self): + # integration test of item-album fallback + lib = beets.library.Library(':memory:') + i = item(lib) + album = lib.add_album([i]) + album['flex'] = u'foo' + album.store() + + self.assertTrue('flex' in i) + self.assertFalse('flex' in i.keys(with_album=False)) + self.assertEqual(i['flex'], u'foo') + self.assertEqual(i.get('flex'), u'foo') + self.assertEqual(i.get('flex', with_album=False), None) + self.assertEqual(i.get('flexx'), None) + class DestinationTest(_common.TestCase): def setUp(self): @@ -491,6 +506,24 @@ class DestinationTest(_common.TestCase): dest = self.i.destination() self.assertEqual(dest[-2:], b'XX') + def test_album_field_query(self): + self.lib.directory = b'one' + self.lib.path_formats = [(u'default', u'two'), + (u'flex:foo', u'three')] + album = self.lib.add_album([self.i]) + self.assertEqual(self.i.destination(), np('one/two')) + album['flex'] = u'foo' + album.store() + self.assertEqual(self.i.destination(), np('one/three')) + + def test_album_field_in_template(self): + self.lib.directory = b'one' + self.lib.path_formats = [(u'default', u'$flex/two')] + album = self.lib.add_album([self.i]) + album['flex'] = u'foo' + album.store() + self.assertEqual(self.i.destination(), np('one/foo/two')) + class ItemFormattedMappingTest(_common.LibTestCase): def test_formatted_item_value(self): diff --git a/test/test_query.py b/test/test_query.py index c0ab2a171..56134da7b 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -109,7 +109,7 @@ class DummyDataTestCase(_common.TestCase, AssertsMixin): items[2].comp = False for item in items: self.lib.add(item) - self.lib.add_album(items[:2]) + self.album = self.lib.add_album(items[:2]) def assert_items_matched_all(self, results): self.assert_items_matched(results, [ @@ -300,6 +300,17 @@ class GetTest(DummyDataTestCase): results = self.lib.items(q) self.assertFalse(results) + def test_album_field_fallback(self): + self.album['albumflex'] = u'foo' + self.album.store() + + q = u'albumflex:foo' + results = self.lib.items(q) + self.assert_items_matched(results, [ + u'foo bar', + u'baz qux', + ]) + def test_invalid_query(self): with self.assertRaises(InvalidQueryArgumentValueError) as raised: dbcore.query.NumericQuery('year', u'199a') From a112358f766260298e4b9d8be9661e85df3a7576 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Sun, 22 Jul 2018 20:26:23 +0200 Subject: [PATCH 002/130] Override FormattedItemMapping.model_keys model_keys was inferred from `self.keys(True)`, which would include the fallback album keys. Since FormattedItemMapping has its own algorithm for album attributes, we only care about the item's *actual* keys. --- beets/library.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beets/library.py b/beets/library.py index 5740f20ee..ba822c513 100644 --- a/beets/library.py +++ b/beets/library.py @@ -376,6 +376,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def __init__(self, item, for_path=False): super(FormattedItemMapping, self).__init__(item, for_path) + # We treat album and item keys specially here, + # so exclude transitive album keys from the model's keys. + self.model_keys = item.keys(computed=True, with_album=False) self.item = item @lazy_property From e17c478f74f3331a465af851d5f995c74db62ca6 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 02:39:22 +0200 Subject: [PATCH 003/130] Cache an item's album with a property Use Album.load() whenever the album is requested, which causes it to be reloaded from the database. Drawback: This adds a slowdown of 100% (6.2s to 12.6s) to `beet list` on my setup. --- beets/importer.py | 2 +- beets/library.py | 45 ++++++++++++++++++++++++++++++++------------ beetsplug/convert.py | 2 +- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index d2943b511..4c4316a8e 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -772,7 +772,7 @@ class ImportTask(BaseImportTask): if (not dup_item.album_id or dup_item.album_id in replaced_album_ids): continue - replaced_album = dup_item.get_album() + replaced_album = dup_item._cached_album if replaced_album: replaced_album_ids.add(dup_item.album_id) self.replaced_albums[replaced_album.path] = replaced_album diff --git a/beets/library.py b/beets/library.py index ba822c513..87bbcd134 100644 --- a/beets/library.py +++ b/beets/library.py @@ -395,9 +395,9 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): album_keys.append(key) return album_keys - @lazy_property + @property def album(self): - return self.item.get_album() + return self.item._cached_album def _get(self, key): """Get the value for a key, either from the album or the item. @@ -542,6 +542,29 @@ class Item(LibModel): _format_config_key = 'format_item' + __album = None + """Cached album object. Read-only.""" + + @property + def _cached_album(self): + """The Album object that this item belongs to, if any, or + None if the item is a singleton or is not associated with a + library. + The instance is cached and refreshed on access. + + DO NOT MODIFY! + If you want a copy to modify, use :meth:`get_album`. + """ + if not self.__album and self._db: + self.__album = self._db.get_album(self) + elif self.__album: + self.__album.load() + return self.__album + + @_cached_album.setter + def _cached_album(self, album): + self.__album = album + @classmethod def _getters(cls): getters = plugins.item_field_getters() @@ -568,6 +591,8 @@ class Item(LibModel): value = bytestring_path(value) elif isinstance(value, BLOB_TYPE): value = bytes(value) + elif key == 'album_id': + self._cached_album = None changed = super(Item, self)._setitem(key, value) @@ -581,9 +606,8 @@ class Item(LibModel): try: return super(Item, self).__getitem__(key) except KeyError: - album = self.get_album() - if album: - return album[key] + if self._cached_album: + return self._cached_album[key] raise def keys(self, computed=False, with_album=True): @@ -591,10 +615,8 @@ class Item(LibModel): controls whether the album's fields are included. """ keys = super(Item, self).keys(computed=computed) - if with_album: - album = self.get_album() - if album: - keys += album.keys(computed=computed) + if with_album and self._cached_album: + keys += self._cached_album.keys(computed=computed) return keys def get(self, key, default=None, with_album=True): @@ -604,9 +626,8 @@ class Item(LibModel): try: return self._get(key, default, raise_=with_album) except KeyError: - album = self.get_album() - if album: - return album.get(key, default) + if self._cached_album: + return self._cached_album.get(key, default) return default def update(self, values): diff --git a/beetsplug/convert.py b/beetsplug/convert.py index 6ed139da0..c873ec7c1 100644 --- a/beetsplug/convert.py +++ b/beetsplug/convert.py @@ -332,7 +332,7 @@ class ConvertPlugin(BeetsPlugin): item.store() # Store new path and audio data. if self.config['embed']: - album = item.get_album() + album = item._cached_album if album and album.artpath: self._log.debug(u'embedding album art from {}', util.displayable_path(album.artpath)) From dfc23e8efe23d4e9b4092c0cdad0c6eba639bf0f Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Fri, 14 Sep 2018 02:43:38 +0200 Subject: [PATCH 004/130] Implement database and model revision checking Prevents reloading a model from the database when it hasn't changed. Now we're back to almost the same speed as before the addition of album field fallbacks. --- beets/dbcore/db.py | 35 ++++++++++++++++++++++++++++++-- test/test_dbcore.py | 49 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 5aa418632..3786bbbd2 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -251,6 +251,11 @@ class Model(object): value is the same as the old value (e.g., `o.f = o.f`). """ + _revision = -1 + """A revision number from when the model was loaded from or written + to the database. + """ + @classmethod def _getters(cls): """Return a mapping from field names to getter functions. @@ -303,9 +308,11 @@ class Model(object): def clear_dirty(self): """Mark all fields as *clean* (i.e., not needing to be stored to - the database). + the database). Also update the revision. """ self._dirty = set() + if self._db: + self._revision = self._db.revision def _check_db(self, need_id=True): """Ensure that this object is associated with a database row: it @@ -533,8 +540,14 @@ class Model(object): def load(self): """Refresh the object's metadata from the library database. + + If check_revision is true, the database is only queried loaded when a + transaction has been committed since the item was last loaded. """ self._check_db() + if not self._dirty and self._db.revision == self._revision: + # Exit early + return stored_obj = self._db._get(type(self), self.id) assert stored_obj is not None, u"object {0} not in DB".format(self.id) self._values_fixed = LazyConvertDict(self) @@ -789,6 +802,12 @@ class Transaction(object): """A context manager for safe, concurrent access to the database. All SQL commands should be executed through a transaction. """ + + _mutated = False + """A flag storing whether a mutation has been executed in the + current transaction. + """ + def __init__(self, db): self.db = db @@ -810,12 +829,15 @@ class Transaction(object): entered but not yet exited transaction. If it is the last active transaction, the database updates are committed. """ + # Beware of races; currently secured by db._db_lock + self.db.revision += self._mutated with self.db._tx_stack() as stack: assert stack.pop() is self empty = not stack if empty: # Ending a "root" transaction. End the SQLite transaction. self.db._connection().commit() + self._mutated = False self.db._db_lock.release() def query(self, statement, subvals=()): @@ -831,7 +853,6 @@ class Transaction(object): """ try: cursor = self.db._connection().execute(statement, subvals) - return cursor.lastrowid except sqlite3.OperationalError as e: # In two specific cases, SQLite reports an error while accessing # the underlying database file. We surface these exceptions as @@ -841,9 +862,14 @@ class Transaction(object): raise DBAccessError(e.args[0]) else: raise + else: + self._mutated = True + return cursor.lastrowid def script(self, statements): """Execute a string containing multiple SQL statements.""" + # We don't know whether this mutates, but quite likely it does. + self._mutated = True self.db._connection().executescript(statements) @@ -859,6 +885,11 @@ class Database(object): supports_extensions = hasattr(sqlite3.Connection, 'enable_load_extension') """Whether or not the current version of SQLite supports extensions""" + revision = 0 + """The current revision of the database. To be increased whenever + data is written in a transaction. + """ + def __init__(self, path, timeout=5.0): self.path = path self.timeout = timeout diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 9bf78de67..c3a8a60ee 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -224,6 +224,31 @@ class MigrationTest(unittest.TestCase): self.fail("select failed") +class TransactionTest(unittest.TestCase): + def setUp(self): + self.db = DatabaseFixture1(':memory:') + + def tearDown(self): + self.db._connection().close() + + def test_mutate_increase_revision(self): + old_rev = self.db.revision + with self.db.transaction() as tx: + tx.mutate( + 'INSERT INTO {0} ' + '(field_one) ' + 'VALUES (?);'.format(ModelFixture1._table), + (111,), + ) + self.assertGreater(self.db.revision, old_rev) + + def test_query_no_increase_revision(self): + old_rev = self.db.revision + with self.db.transaction() as tx: + tx.query('PRAGMA table_info(%s)' % ModelFixture1._table) + self.assertEqual(self.db.revision, old_rev) + + class ModelTest(unittest.TestCase): def setUp(self): self.db = DatabaseFixture1(':memory:') @@ -245,6 +270,30 @@ class ModelTest(unittest.TestCase): row = self.db._connection().execute('select * from test').fetchone() self.assertEqual(row['field_one'], 123) + def test_revision(self): + old_rev = self.db.revision + model = ModelFixture1() + model.add(self.db) + model.store() + self.assertEqual(model._revision, self.db.revision) + self.assertGreater(self.db.revision, old_rev) + + mid_rev = self.db.revision + model2 = ModelFixture1() + model2.add(self.db) + model2.store() + self.assertGreater(model2._revision, mid_rev) + self.assertGreater(self.db.revision, model._revision) + + # revision changed, so the model should be re-loaded + model.load() + self.assertEqual(model._revision, self.db.revision) + + # revision did not change, so no reload + mod2_old_rev = model2._revision + model2.load() + self.assertEqual(model2._revision, mod2_old_rev) + def test_retrieve_by_id(self): model = ModelFixture1() model.add(self.db) From 26b1aa990e9b9dc5810097437eece16b81fccb0e Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Fri, 14 Sep 2018 02:52:53 +0200 Subject: [PATCH 005/130] Add changelog for #2797 --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e58325482..862074124 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,11 @@ New features: to MPD commands we don't support yet. Let us know if you find an MPD client that doesn't get along with BPD! :bug:`3214` :bug:`800` +* Fields in queries now fall back to an item's album and check its fields too. + Notably, this allows querying items by an album flex attribute (also in path + configuration). Plugins: Also applies to normal item access. + Thanks to :user:`FichteFoll`. + :bug:`2797` :bug:`2988` Fixes: From ed76da57e53dcc8be96eac8cd5e11f1103f48a9a Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Mon, 17 Sep 2018 23:51:38 +0200 Subject: [PATCH 006/130] Add changelog section for developers --- docs/changelog.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 862074124..dd1e265c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ New features: that doesn't get along with BPD! :bug:`3214` :bug:`800` * Fields in queries now fall back to an item's album and check its fields too. - Notably, this allows querying items by an album flex attribute (also in path - configuration). Plugins: Also applies to normal item access. + Notably, this allows querying items by an album flex attribute, also in path + configuration. Thanks to :user:`FichteFoll`. :bug:`2797` :bug:`2988` @@ -49,6 +49,12 @@ For plugin developers: is almost identical apart from the name change. Again, we'll re-export at the old location (with a deprecation warning) for backwards compatibility, but might stop doing this in a future release. +* Item (and attribute) access on an item now falls back to the album's + attributes as well. If you specifically want to access an item's attributes, + use ``Item.get(key, with_album=False)``. :bug:`2988` +* ``Item.keys`` also has a ``with_album`` argument now, defaulting to ``True``. +* A ``revision`` attribute has been added to ``Database``. It is increased on + every transaction that mutates it. :bug:`2988` For packagers: From 9c35da69bab1c18f3828aa562e97e2a299272629 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Wed, 5 Jun 2019 02:13:25 +0200 Subject: [PATCH 007/130] Update the items' type information from plugins Plugins can provide item and album attributes. We need to carry over the type information of album attributes so that our item-to-album fallback has these and allows for e.g. range queries. Courtesy of @arcresu (via https://github.com/beetbox/beets/pull/2988#issuecomment-492444925). --- beets/ui/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index a88ed9aef..d798f5134 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -1146,8 +1146,13 @@ def _setup(options, lib=None): plugins.send("library_opened", lib=lib) # Add types and queries defined by plugins. - library.Item._types.update(plugins.types(library.Item)) - library.Album._types.update(plugins.types(library.Album)) + plugin_types_album = plugins.types(library.Album) + library.Album._types.update(plugin_types_album) + item_types = plugin_types_album.copy() + item_types.update(library.Item._types) + item_types.update(plugins.types(library.Item)) + library.Item._types = item_types + library.Item._queries.update(plugins.named_queries(library.Item)) library.Album._queries.update(plugins.named_queries(library.Album)) From 306571883ec2f20b71a2e658ccf27d124eebc512 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Wed, 18 Mar 2020 22:59:55 -0600 Subject: [PATCH 008/130] Implement tekstowo lyrics provider --- beetsplug/lyrics.py | 54 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0e797d5a3..0762cd3d1 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -436,6 +436,50 @@ class LyricsWiki(SymbolsReplaced): return lyrics +class Tekstowo(Backend): + # Fetch lyrics from Tekstowo.pl. + + BASE_URL = 'http://www.tekstowo.pl' + URL_PATTERN = BASE_URL + '/wyszukaj.html?search-title=%s&search-artist=%s' + + def fetch(self, artist, title): + url = self.build_url(title, artist) + search_results = self.fetch_url(url) + song_page_url = self.parse_search_results(search_results) + + if not song_page_url: + return None + + song_page_html = self.fetch_url(song_page_url) + return self.extract_lyrics(song_page_html) + + def parse_search_results(self, html): + if not HAS_BEAUTIFUL_SOUP: + return None + + html = _scrape_strip_cruft(html) + html = _scrape_merge_paragraphs(html) + + try: + html = BeautifulSoup(html, "html.parser") + except HTMLParseError: + return None + + href = html.find("div", class_="content").find_all("div", class_="box-przeboje")[0].find('a').get('href') + return self.BASE_URL + href + + def extract_lyrics(self, html): + html = _scrape_strip_cruft(html) + html = _scrape_merge_paragraphs(html) + + try: + html = BeautifulSoup(html, "html.parser") + except HTMLParseError: + return None + + return html.find("div", class_="song-text").get_text() + + def remove_credits(text): """Remove first/last line of text if it contains the word 'lyrics' eg 'Lyrics by songsdatabase.com' @@ -627,12 +671,13 @@ class Google(Backend): class LyricsPlugin(plugins.BeetsPlugin): - SOURCES = ['google', 'lyricwiki', 'musixmatch', 'genius'] + SOURCES = ['google', 'lyricwiki', 'musixmatch', 'genius', 'tekstowo'] SOURCE_BACKENDS = { 'google': Google, 'lyricwiki': LyricsWiki, 'musixmatch': MusiXmatch, 'genius': Genius, + 'tekstowo': Tekstowo, } def __init__(self): @@ -693,6 +738,13 @@ class LyricsPlugin(plugins.BeetsPlugin): ) sources.remove('genius') + if 'tekstowo' in sources and not HAS_BEAUTIFUL_SOUP: + self._log.debug( + u'The Tekstowo.pl backend requires BeautifulSoup, which is not ' + u'installed, so the source is disabled.' + ) + sources.remove('tekstowo') + self.config['bing_lang_from'] = [ x.lower() for x in self.config['bing_lang_from'].as_str_seq()] self.bing_auth_token = None From 1266a04998b1273402655067aecc8fe9d35531dd Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Thu, 19 Mar 2020 21:10:52 -0600 Subject: [PATCH 009/130] Docs, lint, and cleanup beautifulsoup source check --- beetsplug/lyrics.py | 42 +++++++++++++++++++++-------------------- docs/changelog.rst | 6 ++++-- docs/plugins/lyrics.rst | 18 ++++++++++++++---- test/test_lyrics.py | 2 ++ 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0762cd3d1..91fc157a5 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -465,7 +465,13 @@ class Tekstowo(Backend): except HTMLParseError: return None - href = html.find("div", class_="content").find_all("div", class_="box-przeboje")[0].find('a').get('href') + song_row = html.find("div", class_="content"). \ + find_all("div", class_="box-przeboje")[0] + + if not song_row: + return None + + href = song_row.find('a').get('href') return self.BASE_URL + href def extract_lyrics(self, html): @@ -672,6 +678,7 @@ class Google(Backend): class LyricsPlugin(plugins.BeetsPlugin): SOURCES = ['google', 'lyricwiki', 'musixmatch', 'genius', 'tekstowo'] + SOURCES_USING_BEAUTIFUL_SOUP = ['google', 'genius', 'tekstowo'] SOURCE_BACKENDS = { 'google': Google, 'lyricwiki': LyricsWiki, @@ -716,6 +723,9 @@ class LyricsPlugin(plugins.BeetsPlugin): sources = plugins.sanitize_choices( self.config['sources'].as_str_seq(), available_sources) + if not HAS_BEAUTIFUL_SOUP: + sources = self.sanitize_beautiful_soup_sources(sources) + if 'google' in sources: if not self.config['google_API_key'].get(): # We log a *debug* message here because the default @@ -725,25 +735,6 @@ class LyricsPlugin(plugins.BeetsPlugin): self._log.debug(u'Disabling google source: ' u'no API key configured.') sources.remove('google') - elif not HAS_BEAUTIFUL_SOUP: - self._log.warning(u'To use the google lyrics source, you must ' - u'install the beautifulsoup4 module. See ' - u'the documentation for further details.') - sources.remove('google') - - if 'genius' in sources and not HAS_BEAUTIFUL_SOUP: - self._log.debug( - u'The Genius backend requires BeautifulSoup, which is not ' - u'installed, so the source is disabled.' - ) - sources.remove('genius') - - if 'tekstowo' in sources and not HAS_BEAUTIFUL_SOUP: - self._log.debug( - u'The Tekstowo.pl backend requires BeautifulSoup, which is not ' - u'installed, so the source is disabled.' - ) - sources.remove('tekstowo') self.config['bing_lang_from'] = [ x.lower() for x in self.config['bing_lang_from'].as_str_seq()] @@ -757,6 +748,17 @@ class LyricsPlugin(plugins.BeetsPlugin): self.backends = [self.SOURCE_BACKENDS[source](self.config, self._log) for source in sources] + def sanitize_beautiful_soup_sources(self, sources): + for source in self.SOURCES_USING_BEAUTIFUL_SOUP: + if source in sources: + self._log.warning(u'To use the %s lyrics source, you must ' + u'install the beautifulsoup4 module. See ' + u'the documentation for further details.' + % source) + sources.remove(source) + + return sources + def get_bing_access_token(self): params = { 'client_id': 'beets', diff --git a/docs/changelog.rst b/docs/changelog.rst index a51c85cad..981b2f89a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,7 +19,7 @@ New features: * :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and `discogs_artistid` :bug: `3413` -* :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; +* :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; which allows for the ability to export in json, csv and xml. Thanks to :user:`austinmm`. :bug:`3402` @@ -114,7 +114,7 @@ New features: Fixes: -* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take +* :doc:`/plugins/fetchart`: Fixed a bug that caused fetchart to not take environment variables such as proxy servers into account when making requests :bug:`3450` * :doc:`/plugins/inline`: In function-style field definitions that refer to @@ -174,6 +174,8 @@ Fixes: * Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method failing with AttributeError when called. It is now an instance method. :bug:`3516` :bug:`3517` +* :doc:`/plugins/lyrics`: Added Tekstowo.pl lyrics provider + :bug:`3344` For plugin developers: diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index fac07ad87..72a664319 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -3,11 +3,12 @@ Lyrics Plugin The ``lyrics`` plugin fetches and stores song lyrics from databases on the Web. Namely, the current version of the plugin uses `Lyric Wiki`_, -`Musixmatch`_, `Genius.com`_, and, optionally, the Google custom search API. +`Musixmatch`_, `Genius.com`_, `Tekstowo.pl`_, and, optionally, the Google custom search API. .. _Lyric Wiki: https://lyrics.wikia.com/ .. _Musixmatch: https://www.musixmatch.com/ .. _Genius.com: https://genius.com/ +.. _Tekstowo.pl: https://www.tekstowo.pl/ Fetch Lyrics During Import @@ -59,11 +60,10 @@ configuration file. The available options are: sources known to be scrapeable. - **sources**: List of sources to search for lyrics. An asterisk ``*`` expands to all available sources. - Default: ``google lyricwiki musixmatch genius``, i.e., all the + Default: ``google lyricwiki musixmatch genius tekstowo``, i.e., all the available sources. The ``google`` source will be automatically deactivated if no ``google_API_key`` is setup. - Both it and the ``genius`` source will only be enabled if BeautifulSoup is - installed. + The following sources will only be enabled if BeatifulSoup is installed: ``[google, genius, tekstowo]`` Here's an example of ``config.yaml``:: @@ -166,6 +166,16 @@ library. Install it by typing:: The backend is enabled by default. +Activate Tekstowo.pl Lyrics +---------------------- + +Like the Google backend, the Tekstowo.pl backend requires the `BeautifulSoup`_ +library. Install it by typing:: + + pip install beautifulsoup4 + +The backend is enabled by default. + .. _lyrics-translation: Activate On-the-Fly Translation diff --git a/test/test_lyrics.py b/test/test_lyrics.py index f7ea538e2..9a5e9e893 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -251,6 +251,8 @@ class LyricsPluginSourcesTest(LyricsGoogleBaseTest): dict(artist=u'Santana', title=u'Black magic woman', backend=lyrics.MusiXmatch), dict(DEFAULT_SONG, backend=lyrics.Genius), + dict(artist=u'Boy In Space', title=u'u n eye', + backend=lyrics.Tekstowo), ] GOOGLE_SOURCES = [ From 58bfe4567e7cf4f026f2c20a7b61b07699d12756 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sat, 28 Mar 2020 11:34:03 -0600 Subject: [PATCH 010/130] Code review --- beetsplug/lyrics.py | 8 ++++---- docs/plugins/lyrics.rst | 25 +++++-------------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 91fc157a5..b22525978 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -678,7 +678,7 @@ class Google(Backend): class LyricsPlugin(plugins.BeetsPlugin): SOURCES = ['google', 'lyricwiki', 'musixmatch', 'genius', 'tekstowo'] - SOURCES_USING_BEAUTIFUL_SOUP = ['google', 'genius', 'tekstowo'] + BS_SOURCES = ['google', 'genius', 'tekstowo'] SOURCE_BACKENDS = { 'google': Google, 'lyricwiki': LyricsWiki, @@ -724,7 +724,7 @@ class LyricsPlugin(plugins.BeetsPlugin): self.config['sources'].as_str_seq(), available_sources) if not HAS_BEAUTIFUL_SOUP: - sources = self.sanitize_beautiful_soup_sources(sources) + sources = self.sanitize_bs_sources(sources) if 'google' in sources: if not self.config['google_API_key'].get(): @@ -748,8 +748,8 @@ class LyricsPlugin(plugins.BeetsPlugin): self.backends = [self.SOURCE_BACKENDS[source](self.config, self._log) for source in sources] - def sanitize_beautiful_soup_sources(self, sources): - for source in self.SOURCES_USING_BEAUTIFUL_SOUP: + def sanitize_bs_sources(self, sources): + for source in self.BS_SOURCES: if source in sources: self._log.warning(u'To use the %s lyrics source, you must ' u'install the beautifulsoup4 module. See ' diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 72a664319..820af89cd 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -129,11 +129,6 @@ few suggestions. Activate Google Custom Search ------------------------------ -Using the Google backend requires `BeautifulSoup`_, which you can install -using `pip`_ by typing:: - - pip install beautifulsoup4 - You also need to `register for a Google API key`_. Set the ``google_API_key`` configuration option to your key. Then add ``google`` to the list of sources in your configuration (or use @@ -156,25 +151,15 @@ After that, the lyrics plugin will fall back on other declared data sources. .. _pip: https://pip.pypa.io .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ -Activate Genius Lyrics ----------------------- +Installing BeautifulSoup for Google, Genius, or Tekstowo.pl Backends +-------------------------------------------------------------------- -Like the Google backend, the Genius backend requires the `BeautifulSoup`_ -library. Install it by typing:: +Using the Google, Genius, or Tekstowo.pl backends requires `BeautifulSoup`_, +which you can install using `pip`_ by typing:: pip install beautifulsoup4 -The backend is enabled by default. - -Activate Tekstowo.pl Lyrics ----------------------- - -Like the Google backend, the Tekstowo.pl backend requires the `BeautifulSoup`_ -library. Install it by typing:: - - pip install beautifulsoup4 - -The backend is enabled by default. +These backends are enabled by default. .. _lyrics-translation: From 38ea35aa5e59011179a10bf11dff0f1d44f8f84f Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sat, 28 Mar 2020 11:44:56 -0600 Subject: [PATCH 011/130] Lint --- beetsplug/lyrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index b22525978..1d92b24e5 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -469,7 +469,7 @@ class Tekstowo(Backend): find_all("div", class_="box-przeboje")[0] if not song_row: - return None + return None href = song_row.find('a').get('href') return self.BASE_URL + href @@ -752,9 +752,9 @@ class LyricsPlugin(plugins.BeetsPlugin): for source in self.BS_SOURCES: if source in sources: self._log.warning(u'To use the %s lyrics source, you must ' - u'install the beautifulsoup4 module. See ' - u'the documentation for further details.' - % source) + u'install the beautifulsoup4 module. See ' + u'the documentation for further details.' + % source) sources.remove(source) return sources From 5c875c50de462b5c92af84efbe1db6f848bfa4bb Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Mon, 21 Sep 2020 20:24:41 +0100 Subject: [PATCH 012/130] AURA: Add aura plugin and docs --- beetsplug/aura.py | 847 +++++++++++++++++++++++++++++++++++++++++ docs/plugins/aura.rst | 195 ++++++++++ docs/plugins/index.rst | 3 + 3 files changed, 1045 insertions(+) create mode 100644 beetsplug/aura.py create mode 100644 docs/plugins/aura.rst diff --git a/beetsplug/aura.py b/beetsplug/aura.py new file mode 100644 index 000000000..4269a4df8 --- /dev/null +++ b/beetsplug/aura.py @@ -0,0 +1,847 @@ +# -*- coding: utf-8 -*- + +# This file is part of beets. +# Copyright 2020, Callum Brown. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""An AURA server using Flask. +""" + +from __future__ import division, absolute_import, print_function + +from mimetypes import guess_type +import re +from os.path import isfile, getsize + +from beets.plugins import BeetsPlugin +from beets.ui import Subcommand, _open_library +from beets import config +from beets.util import displayable_path +from beets.library import Item, Album +from beets.dbcore.query import ( + MatchQuery, + NotQuery, + RegexpQuery, + AndQuery, + FixedFieldSort, + SlowFieldSort, + MultipleSort, +) + +from flask import ( + Blueprint, + Flask, + current_app, + send_file, + make_response, + request, +) + + +# Constants + +# AURA server information +# TODO: Add version information +SERVER_INFO = { + "aura-version": "0", + "server": "beets-aura", + "server-version": "0", + "auth-required": False, + "features": ["albums", "artists", "images"], +} + +# Maps AURA Track attribute to beets Item attribute +TRACK_ATTR_MAP = { + # Required + "title": "title", + "artist": "artist", + # Optional + "album": "album", + "track": "track", # Track number on album + "tracktotal": "tracktotal", + "disc": "disc", + "disctotal": "disctotal", + "year": "year", + "month": "month", + "day": "day", + "bpm": "bpm", + "genre": "genre", + "recording-mbid": "mb_trackid", # beets trackid is MB recording + "track-mbid": "mb_releasetrackid", + "composer": "composer", + "albumartist": "albumartist", + "comments": "comments", + # Optional for Audio Metadata + # TODO: Support the mimetype attribute, format != mime type + # "mimetype": track.format, + "duration": "length", + "framerate": "samplerate", + # I don't think beets has a framecount field + # "framecount": ???, + "channels": "channels", + "bitrate": "bitrate", + "bitdepth": "bitdepth", + "size": "filesize", +} + +# Maps AURA Album attribute to beets Album attribute +ALBUM_ATTR_MAP = { + # Required + "title": "album", + "artist": "albumartist", + # Optional + "tracktotal": "albumtotal", + "disctotal": "disctotal", + "year": "year", + "month": "month", + "day": "day", + "genre": "genre", + "release-mbid": "mb_albumid", + "release-group-mbid": "mb_releasegroupid", +} + +# Maps AURA Artist attribute to beets Item field +# Artists are not first-class in beets, so information is extracted from +# beets Items. +ARTIST_ATTR_MAP = { + # Required + "name": "artist", + # Optional + "artist_mbid": "mb_artistid", +} + + +class AURADocument: + """Base class for building AURA documents.""" + + @staticmethod + def error(status, title, detail): + """Make a response for an error following the JSON:API spec.""" + document = { + "errors": [{"status": status, "title": title, "detail": detail}] + } + return make_response(document, status) + + def translate_attribute(self, aura_attr): + """Translate AURA attribute name to beets attribute name.""" + try: + return self.attribute_map[aura_attr] + except KeyError: + # Assume native beets attribute + return aura_attr + + def translate_filters(self): + """Translate filters from request arguments to a beets Query.""" + # The format of each filter key in the request parameter is: + # filter[]. This regex extracts . + pattern = re.compile(r"filter\[(?P\w+)\]") + queries = [] + for key, value in request.args.items(): + match = pattern.match(key) + if match is not None: + # Extract attribute name from key + aura_attr = match.group("attribute") + # Get the beets version of the attribute name + beets_attr = self.translate_attribute(aura_attr) + converter = self.get_attribute_converter(beets_attr) + value = converter(value) + # Add exact match query to list + # Use a slow query so it works with all fields + queries.append(MatchQuery(beets_attr, value, fast=False)) + # NOTE: AURA doesn't officially support multiple queries + return AndQuery(queries) + + def translate_sorts(self, sort_arg): + """Translate an AURA sort parameter into a beets Sort.""" + # Change HTTP query parameter to a list + aura_sorts = sort_arg.strip(",").split(",") + sorts = [] + for aura_attr in aura_sorts: + if aura_attr[0] == "-": + ascending = False + # Remove leading "-" + aura_attr = aura_attr[1:] + else: + # JSON:API default + ascending = True + # Get the beets version of the attribute name + beets_attr = self.translate_attribute(aura_attr) + # Use slow sort so it works with all fields (inc. computed) + sorts.append(SlowFieldSort(beets_attr, ascending=ascending)) + return MultipleSort(sorts) + + def paginate(self, collection): + """Get a page of the collection and the URL to the next page.""" + # Pages start from zero + page = request.args.get("page", 0, int) + # Use page limit defined in config by default. + default_limit = config["aura"]["page_limit"].get(int) + limit = request.args.get("limit", default_limit, int) + # start = offset of first item to return + start = page * limit + # end = offset of last item + 1 + end = start + limit + if end > len(collection): + end = len(collection) + next_url = None + else: + # Not the last page so work out links.next url + if len(request.args) == 0: + # No existing arguments, so current page is 0 + next_url = request.url + "?page=1" + elif request.args.get("page", None) is None: + # No existing page argument, so add one to the end + next_url = request.url + "&page=1" + else: + # Increment page token by 1 + next_url = request.url.replace( + "page={}".format(page), "page={}".format(page + 1) + ) + # Get only the items in the page range + data = [self.resource_object(collection[i]) for i in range(start, end)] + return data, next_url + + def get_included(self, data, include_str): + """Build a list of resource objects for inclusion. + + :param data: Array of dicts in the form of resource objects + :param include_str: Comma separated list of resources to include + """ + # Change HTTP query parameter to a list + to_include = include_str.strip(",").split(",") + # Build a list of unique type and id combinations + # For each resource object in the primary data, iterate over it's + # relationships. If a relationship matches one of the types + # requested for inclusion (e.g. "albums") then add each type-id pair + # under the "data" key to unique_identifiers, checking first that + # it has not already been added. This ensures that no resources are + # included more than once. + unique_identifiers = [] + for res_obj in data: + for rel_name, rel_obj in res_obj["relationships"].items(): + if rel_name in to_include: + # NOTE: Assumes relationship is to-many + for identifier in rel_obj["data"]: + if identifier not in unique_identifiers: + unique_identifiers.append(identifier) + # TODO: I think this could be improved + included = [] + for identifier in unique_identifiers: + res_type = identifier["type"] + if res_type == "track": + track_id = int(identifier["id"]) + track = current_app.config["lib"].get_item(track_id) + included.append(TrackDocument.resource_object(track)) + elif res_type == "album": + album_id = int(identifier["id"]) + album = current_app.config["lib"].get_album(album_id) + included.append(AlbumDocument.resource_object(album)) + elif res_type == "artist": + artist_id = identifier["id"] + included.append(ArtistDocument.resource_object(artist_id)) + elif res_type == "image": + image_id = identifier["id"] + included.append(ImageDocument.resource_object(image_id)) + else: + raise ValueError("Invalid resource type: {}".format(res_type)) + return included + + def all_resources(self): + """Build document for /tracks, /albums or /artists.""" + query = self.translate_filters() + sort_arg = request.args.get("sort", None) + if sort_arg is not None: + sort = self.translate_sorts(sort_arg) + # For each sort field add a query which ensures all results + # have a non-empty, non-zero value for that field. + for s in sort.sorts: + query.subqueries.append( + NotQuery( + # Match empty fields (^$) or zero fields, (^0$) + RegexpQuery(s.field, "(^$|^0$)", fast=False) + ) + ) + else: + sort = None + # Get information from the library + collection = self.get_collection(query=query, sort=sort) + # Convert info to AURA form and paginate it + data, next_url = self.paginate(collection) + document = {"data": data} + # If there are more pages then provide a way to access them + if next_url is not None: + document["links"] = {"next": next_url} + # Include related resources for each element in "data" + include_str = request.args.get("include", None) + if include_str is not None: + document["included"] = self.get_included(data, include_str) + return document + + def single_resource_document(self, resource_object): + """Build document for a specific requested resource.""" + document = {"data": resource_object} + include_str = request.args.get("include", None) + if include_str is not None: + # [document["data"]] is because arg needs to be list + document["included"] = self.get_included( + [document["data"]], include_str + ) + return document + + +class TrackDocument(AURADocument): + """Class for building documents for /tracks endpoints.""" + + attribute_map = TRACK_ATTR_MAP + + def get_collection(self, query=None, sort=None): + """Get Item objects from the library.""" + return current_app.config["lib"].items(query, sort) + + def get_attribute_converter(self, beets_attr): + """Work out what data type an attribute should be for beets.""" + # filesize is a special field (read from disk not db?) + if beets_attr == "filesize": + converter = int + else: + try: + # Look for field in list of Item fields + # and get python type of database type. + # See beets.library.Item and beets.dbcore.types + converter = Item._fields[beets_attr].model_type + except KeyError: + # Fall back to string (NOTE: probably not good) + converter = str + return converter + + @staticmethod + def resource_object(track): + """Construct a JSON:API resource object from a beets Item.""" + attributes = {} + # Use aura => beets attribute map, e.g. size => filesize + for aura_attr, beets_attr in TRACK_ATTR_MAP.items(): + a = getattr(track, beets_attr) + # Only set attribute if it's not None, 0, "", etc. + # NOTE: This could result in required attributes not being set + if a: + attributes[aura_attr] = a + + # JSON:API one-to-many relationship to parent album + relationships = { + "artists": {"data": [{"type": "artist", "id": track.artist}]} + } + # Only add album relationship if not singleton + if not track.singleton: + relationships["albums"] = { + "data": [{"type": "album", "id": str(track.album_id)}] + } + + return { + "type": "track", + "id": str(track.id), + "attributes": attributes, + "relationships": relationships, + } + + def single_resource(self, track_id): + """Get track from the library and build a document.""" + track = current_app.config["lib"].get_item(track_id) + if track is None: + return self.error( + "404 Not Found", + "No track with the requested id.", + ("There is no track with an id of {} in the library.").format( + track_id + ), + ) + return self.single_resource_document(self.resource_object(track)) + + +class AlbumDocument(AURADocument): + """Class for building documents for /albums endpoints.""" + + attribute_map = ALBUM_ATTR_MAP + + def get_collection(self, query=None, sort=None): + """Get Album objects from the library.""" + return current_app.config["lib"].albums(query, sort) + + def get_attribute_converter(self, beets_attr): + """Work out what data type an attribute should be for beets.""" + # filesize is a special field (read from disk not db?) + try: + # Look for field in list of Album fields + # and get python type of database type. + # See beets.library.Album and beets.dbcore.types + converter = Album._fields[beets_attr].model_type + except KeyError: + # Fall back to string (NOTE: probably not good) + converter = str + return converter + + @staticmethod + def resource_object(album): + """Construct a JSON:API resource object from a beets Album.""" + attributes = {} + # Use aura => beets attribute name map + for aura_attr, beets_attr in ALBUM_ATTR_MAP.items(): + a = getattr(album, beets_attr) + # Only set attribute if it's not None, 0, "", etc. + # NOTE: This could mean required attributes are not set + if a: + attributes[aura_attr] = a + + # Get beets Item objects for all tracks in the album sorted by + # track number. Sorting is not required but it's nice. + query = MatchQuery("album_id", album.id) + sort = FixedFieldSort("track", ascending=True) + tracks = current_app.config["lib"].items(query, sort) + # JSON:API one-to-many relationship to tracks on the album + relationships = { + "tracks": { + "data": [{"type": "track", "id": str(t.id)} for t in tracks] + } + } + # Add images relationship if album has associated images + if album.artpath is not None: + path = displayable_path(album.artpath) + filename = path.split("/")[-1] + image_id = "album-{}-{}".format(album.id, filename) + relationships["images"] = { + "data": [{"type": "image", "id": image_id}] + } + # Add artist relationship if artist name is same on tracks + # Tracks are used to define artists so don't albumartist + # Check for all tracks in case some have featured artists + if album.albumartist in [t.artist for t in tracks]: + relationships["artists"] = { + "data": [{"type": "artist", "id": album.albumartist}] + } + + return { + "type": "album", + "id": str(album.id), + "attributes": attributes, + "relationships": relationships, + } + + def single_resource(self, album_id): + """Get album from the library and build a document.""" + album = current_app.config["lib"].get_album(album_id) + if album is None: + return self.error( + "404 Not Found", + "No album with the requested id.", + ("There is no album with an id of {} in the library.").format( + album_id + ), + ) + return self.single_resource_document(self.resource_object(album)) + + +class ArtistDocument(AURADocument): + """Class for building documents for /artists endpoints.""" + + attribute_map = ARTIST_ATTR_MAP + + def get_collection(self, query=None, sort=None): + """Get a list of artist names from the library.""" + # Gets only tracks with matching artist information + tracks = current_app.config["lib"].items(query, sort) + collection = [] + for track in tracks: + # Do not add duplicates + if track.artist not in collection: + collection.append(track.artist) + return collection + + def get_attribute_converter(self, beets_attr): + """Work out what data type an attribute should be for beets.""" + try: + # Look for field in list of Item fields + # and get python type of database type. + # See beets.library.Item and beets.dbcore.types + converter = Item._fields[beets_attr].model_type + except KeyError: + # Fall back to string (NOTE: probably not good) + converter = str + return converter + + @staticmethod + def resource_object(artist_id): + """Construct a JSON:API resource object for the given artist.""" + # Get tracks where artist field exactly matches artist_id + query = MatchQuery("artist", artist_id) + tracks = current_app.config["lib"].items(query) + if len(tracks) == 0: + return None + + # Get artist information from the first track + # NOTE: It could be that the first track doesn't have a + # MusicBrainz id but later tracks do, which isn't ideal. + attributes = {} + # Use aura => beets attribute map, e.g. artist => name + for aura_attr, beets_attr in ARTIST_ATTR_MAP.items(): + a = getattr(tracks[0], beets_attr) + # Only set attribute if it's not None, 0, "", etc. + # NOTE: This could mean required attributes are not set + if a: + attributes[aura_attr] = a + + relationships = { + "tracks": { + "data": [{"type": "track", "id": str(t.id)} for t in tracks] + } + } + album_query = MatchQuery("albumartist", artist_id) + albums = current_app.config["lib"].albums(query=album_query) + if len(albums) != 0: + relationships["albums"] = { + "data": [{"type": "album", "id": str(a.id)} for a in albums] + } + + return { + "type": "artist", + "id": artist_id, + "attributes": attributes, + "relationships": relationships, + } + + def single_resource(self, artist_id): + """Get info for the requested artist and build a document.""" + artist_resource = self.resource_object(artist_id) + if artist_resource is None: + return self.error( + "404 Not Found", + "No artist with the requested id.", + ("There is no artist with an id of {} in the library.").format( + artist_id + ), + ) + return self.single_resource_document(artist_resource) + + +class ImageDocument(AURADocument): + """Class for building documents for /images/(id) endpoints.""" + + @staticmethod + def get_image_path(image_id): + """Works out the full path to the image with the given id. + + Returns None if there is no such image. + image_id is in the form -- + """ + # Split image_id into its constituent parts + id_split = image_id.split("-") + if len(id_split) < 3: + # image_id is not in the required format + return None + parent_type = id_split[0] + parent_id = id_split[1] + img_filename = "-".join(id_split[2:]) + + # Get the path to the directory parent's images are in + if parent_type == "album": + album = current_app.config["lib"].get_album(int(parent_id)) + if album is None or album.artpath is None: + return None + # Cut the filename off of artpath + # This is in preparation for supporting images in the same + # directory that are not tracked by beets. + artpath = displayable_path(album.artpath) + dir_path = "/".join(artpath.split("/")[:-1]) + else: + # Images for other resource types are not supported + return None + + img_path = dir_path + "/" + img_filename + # Check the image actually exists + if isfile(img_path): + return img_path + else: + return None + + @staticmethod + def resource_object(image_id): + """Construct a JSON:API resource object for the given image.""" + # Could be called as a static method, so can't use + # self.get_image_path() + image_path = ImageDocument.get_image_path(image_id) + if image_path is None: + return None + + attributes = { + "role": "cover", + "mimetype": guess_type(image_path)[0], + "size": getsize(image_path), + } + try: + from PIL import Image + except ImportError: + pass + else: + im = Image.open(image_path) + attributes["width"] = im.width + attributes["height"] = im.height + + relationships = {} + # Split id into [parent_type, parent_id, filename] + id_split = image_id.split("-") + relationships[id_split[0] + "s"] = { + "data": [{"type": id_split[0], "id": id_split[1]}] + } + + return { + "id": image_id, + "type": "image", + # Remove attributes that are None, 0, "", etc. + "attributes": {k: v for k, v in attributes.items() if v}, + "relationships": relationships, + } + + def single_resource(self, image_id): + """Get info for the requested image and build a document.""" + image_resource = self.resource_object(image_id) + if image_resource is None: + return self.error( + "404 Not Found", + "No image with the requested id.", + ("There is no image with an id of {} in the library.").format( + image_id + ), + ) + return self.single_resource_document(image_resource) + + +# Initialise flask blueprint +aura_bp = Blueprint("aura_bp", __name__) + + +@aura_bp.route("/server") +def server_info(): + """Respond with info about the server.""" + return {"data": {"type": "server", "id": "0", "attributes": SERVER_INFO}} + + +# Track endpoints + + +@aura_bp.route("/tracks") +def all_tracks(): + """Respond with a list of all tracks and related information.""" + doc = TrackDocument() + return doc.all_resources() + + +@aura_bp.route("/tracks/") +def single_track(track_id): + """Respond with info about the specified track.""" + doc = TrackDocument() + return doc.single_resource(track_id) + + +@aura_bp.route("/tracks//audio") +def audio_file(track_id): + """Supply an audio file for the specified track.""" + track = current_app.config["lib"].get_item(track_id) + if track is None: + return AURADocument.error( + "404 Not Found", + "No track with the requested id.", + ("There is no track with an id of {} in the library.").format( + track_id + ), + ) + + path = displayable_path(track.destination()) + if not isfile(path): + return AURADocument.error( + "404 Not Found", + "No audio file for the requested track.", + ( + "There is no audio file for track {} at the expected location" + ).format(track_id), + ) + + file_mimetype = guess_type(path)[0] + if file_mimetype is None: + return AURADocument.error( + "500 Internal Server Error", + "Requested audio file has an unknown mimetype.", + ( + "The audio file for track {} has an unknown mimetype. " + "It's file extension is {}." + ).format(track_id, path.split(".")[-1]), + ) + + # Check that the Accept header contains the file's mimetype + # Takes into account */* and audio/* + # Adding support for the bitrate parameter would require some effort so I + # left it out. This means the client could be sent an error even if the + # audio doesn't need transcoding. + if request.accept_mimetypes.best_match([file_mimetype]) is None: + return AURADocument.error( + "406 Not Acceptable", + "Unsupported MIME type or bitrate parameter in Accept header.", + ( + "The audio file for track {} is only available as {} and " + "bitrate parameters are not supported." + ).format(track_id, file_mimetype), + ) + + return send_file( + path, + mimetype=file_mimetype, + # Handles filename in Content-Disposition header + as_attachment=True, + # Tries to upgrade the stream to support range requests + conditional=True, + ) + + +# Album endpoints + + +@aura_bp.route("/albums") +def all_albums(): + """Respond with a list of all albums and related information.""" + doc = AlbumDocument() + return doc.all_resources() + + +@aura_bp.route("/albums/") +def single_album(album_id): + """Respond with info about the specified album.""" + doc = AlbumDocument() + return doc.single_resource(album_id) + + +# Artist endpoints +# Artist ids are their names + + +@aura_bp.route("/artists") +def all_artists(): + """Respond with a list of all artists and related information.""" + doc = ArtistDocument() + return doc.all_resources() + + +# Using the path converter allows slashes in artist_id +@aura_bp.route("/artists/") +def single_artist(artist_id): + """Respond with info about the specified artist.""" + doc = ArtistDocument() + return doc.single_resource(artist_id) + + +# Image endpoints +# Image ids are in the form -- +# For example: album-13-cover.jpg + + +@aura_bp.route("/images/") +def single_image(image_id): + """Respond with info about the specified image.""" + doc = ImageDocument() + return doc.single_resource(image_id) + + +@aura_bp.route("/images//file") +def image_file(image_id): + """Supply an image file for the specified image.""" + img_path = ImageDocument.get_image_path(image_id) + if img_path is None: + return AURADocument.error( + "404 Not Found", + "No image with the requested id.", + ("There is no image with an id of {} in the library").format( + image_id + ), + ) + return send_file(img_path) + + +# WSGI app + + +def create_app(): + """An application factory for use by a WSGI server.""" + app = Flask(__name__) + # Register AURA blueprint view functions under a URL prefix + app.register_blueprint(aura_bp, url_prefix="/aura") + # AURA specifies mimetype MUST be this + app.config["JSONIFY_MIMETYPE"] = "application/vnd.api+json" + # Disable auto-sorting of JSON keys + app.config["JSON_SORT_KEYS"] = False + # Provide a way to access the beets library + # The normal method of using the Library and config provided in the + # command function is not used because create_app() could be called + # by an external WSGI server. + # NOTE: this uses a 'private' function from beets.ui.__init__ + app.config["lib"] = _open_library(config) + + # Enable CORS if required + cors = config["aura"]["cors"].get(list) + if cors: + from flask_cors import CORS + + # "Accept" is the only header clients use + app.config["CORS_ALLOW_HEADERS"] = "Accept" + app.config["CORS_RESOURCES"] = {r"/aura/*": {"origins": cors}} + app.config["CORS_SUPPORTS_CREDENTIALS"] = config["aura"][ + "cors_supports_credentials" + ].get(bool) + CORS(app) + + return app + + +# Beets Plugin Hook + + +class AURAPlugin(BeetsPlugin): + def __init__(self): + super(AURAPlugin, self).__init__() + self.config.add( + { + "host": u"127.0.0.1", + "port": 8337, + "cors": [], + "cors_supports_credentials": False, + "page_limit": 500, + } + ) + + def commands(self): + def run_aura(lib, opts, args): + """Run the application using Flask's built in-server.""" + app = create_app() + # Start the built-in server (not intended for production) + app.run( + host=self.config["host"].get(str), + port=self.config["port"].get(int), + debug=opts.debug, + threaded=True, + ) + + run_aura_cmd = Subcommand("aura", help=u"run an AURA server") + run_aura_cmd.parser.add_option( + u"-d", + u"--debug", + action="store_true", + default=False, + help=u"use Flask debug mode", + ) + run_aura_cmd.func = run_aura + return [run_aura_cmd] diff --git a/docs/plugins/aura.rst b/docs/plugins/aura.rst new file mode 100644 index 000000000..2891d25e1 --- /dev/null +++ b/docs/plugins/aura.rst @@ -0,0 +1,195 @@ +AURA Plugin +=========== + +This plugin is a server implementation of the `AURA`_ specification using the +`Flask`_ framework. AURA is still a work in progress and doesn't yet have a +stable version, but this server should be kept up to date. You are advised to +read the :ref:`aura-issues` section. + +.. _AURA: https://auraspec.readthedocs.io +.. _Flask: https://palletsprojects.com/p/flask/ + +Install +------- + +The ``aura`` plugin depends on `Flask`_, which can be installed using +``python -m pip install flask``. Then you can enable the ``aura`` plugin in +your configuration (see :ref:`using-plugins`). + +It is likely that you will need to enable :ref:`aura-cors`, which introduces +an additional dependency: `flask-cors`_. This can be installed with +``python -m pip install flask-cors``. + +If `Pillow`_ is installed (``python -m pip install Pillow``) then the optional +``width`` and ``height`` attributes are included in image resource objects. + +.. _flask-cors: https://flask-cors.readthedocs.io +.. _Pillow: https://pillow.readthedocs.io + + +Usage +----- + +Use ``beet aura`` to start the AURA server. +By default Flask's built-in server is used, which will give a warning about +using it in a production environment. It is safe to ignore this warning if the +server will have only a few users. + +Alternatively, you can use ``beet aura -d`` to start the server in +`development mode`_, which will reload the server every time the AURA plugin +file is changed. + +You can specify the hostname and port number used by the server in your +:doc:`configuration file `. For more detail see the +:ref:`configuration` section below. + +If you would prefer to use a different WSGI server, such as gunicorn or uWSGI, +then see :ref:`aura-external-server`. + +AURA is designed to separate the client and server functionality. This plugin +provides the server but not the client, so unless you like looking at JSON you +will need a separate client. Unfortunately there are no AURA clients yet +(discounting the broken and outdated reference implementation). + +By default the API is served under http://127.0.0.1:8337/aura/. For example +information about the track with an id of 3 can be obtained at +http://127.0.0.1:8337/aura/tracks/3. + +**Note the absence of a trailing slash**: +http://127.0.0.1:8337/aura/tracks/3/ returns a ``404 Not Found`` error. + +.. _development mode: https://flask.palletsprojects.com/en/1.1.x/server + + +.. _configuration: + +Configuration +------------- + +To configure the plugin, make an ``aura:`` section in your +configuration file. The available options are: + +- **host**: The server hostname. Set this to 0.0.0.0 to bind to all interfaces. + Default: ``127.0.0.1``. +- **port**: The server port. + Default: ``8337``. +- **cors**: A YAML list of origins to allow CORS requests from (see + :ref:`aura-cors`, below). + Default: disabled. +- **cors_supports_credentials**: Allow authenticated requests when using CORS. + Default: disabled. +- **page_limit**: The number of items responses should be truncated to if the + client does not specify. Default ``500``. + + +.. _aura-cors: + +Cross-Origin Resource Sharing (CORS) +------------------------------------ + +`CORS`_ allows browser clients to make requests to the AURA server. You should +set the ``cors`` configuration option to a YAML list of allowed origins. + +For example:: + + aura: + cors: + - http://www.example.com + - https://aura.example.org + +Alternatively you can set it to ``'*'`` to enable access from all origins. +Note that there are security implications if you set the origin to ``'*'``, +so please research this before using it. Note the use of quote marks when +allowing all origins. Quote marks are also required when the origin is +``null``, for example when using ``file:///``. + +If the server is behind a proxy that uses credentials, you might want to set +the ``cors_supports_credentials`` configuration option to true to let +in-browser clients log in. Note that this option has not been tested, so it +may not work. + +.. _CORS: https://en.wikipedia.org/wiki/Cross-origin_resource_sharing + + +.. _aura-external-server: + +Using an External WSGI Server +----------------------------- + +If you would like to use a different WSGI server (not Flask's built-in one), +then you can! The ``beetsplug.aura`` module provides a WSGI callable called +``create_app()`` which can be used by many WSGI servers. + +For example to run the AURA server using `gunicorn`_ use +``gunicorn 'beetsplug.aura:create_app()'``, or for `uWSGI`_ use +``uwsgi --http :9090 --module 'beetsplug.aura:create_app()'``. +Note that these commands just show how to use the AURA app and you would +probably use something a bit different in a production environment. Read the +relevant server's documentation to figure out what you need. + +.. _gunicorn: https://gunicorn.org +.. _uWSGI: https://uwsgi-docs.readthedocs.io + + +Reverse Proxy Support +--------------------- + +The plugin should work behind a reverse proxy without further configuration, +however this has not been tested extensively. For details of what headers must +be rewritten and a sample NGINX configuration see `Flask proxy setups`_. + +It may be possibly to run the application under a URL prefix (for example so +you could have ``/foo/aura/server`` rather than ``/aura/server``), but it is +likely this would require changes in the code. + +Do not add a trailing slash (``/``) to the URL where the application is +running, otherwise you will get a 404. For example with NGINX you should use +``proxy_pass http://127.0.0.1:8000;`` rather than +``proxy_pass http://127.0.0.1:8000/;``. + +.. _Flask proxy setups: https://flask.palletsprojects.com/en/1.1.x/deploying/wsgi-standalone/#proxy-setups + + +.. _aura-issues: + +Issues +------ + +As of writing there are some differences between the specification and this +implementation: + +- Compound filters are not specified in AURA, but this server interprets + multiple ``filter`` parameters as AND. See `issue #19`_ for discussion. +- The ``bitrate`` parameter used for content negotiation is not supported. + Adding support for this is doable, but the way Flask handles acceptable MIME + types means it's a lot easier not to bother with it. This means an error + could be returned even if no transcoding was required. + +It is possible that some attributes required by AURA could be absent from the +server's response if beets does not have a saved value for them. However, this +has not happened so far. + +The ``mimetype`` and ``framecount`` attributes for track resources are not +supported. The first is due to beets storing the file type (e.g. ``MP3``), so +it is hard to filter by MIME type. The second is because there is no +corresponding beets field. + +Artists are defined by the ``artist`` field on beets Items, which means some +albums have no ``artists`` relationship. Albums only have related artists +when their beets ``albumartist`` field is the same as the ``artist`` field on +at least one of it's constituent tracks. + +The only art tracked by beets is a single cover image, so only albums have +related images at the moment. This could be expanded to looking in the same +directory for other images, and relating tracks to their album's image. + +There are likely to be some performance issues, especially with larger +libraries. Pagination and inclusion (most notably of images) are probably two +of the main offenders. On a related note, the program attempts to import Pillow +every time it constructs an image resource object, which is not very good. + +The beets library is accessed using a so called private function (with a single +leading underscore) ``beets.ui.__init__._open_library()``. This shouldn't cause +any issues but it is probably not best practice. + +.. _issue #19: https://github.com/beetbox/aura/issues/19 diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index aab922fcd..e28e2a2cf 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -61,6 +61,7 @@ following to your configuration:: absubmit acousticbrainz + aura badfiles beatport bpd @@ -184,6 +185,7 @@ Path Formats Interoperability ---------------- +* :doc:`aura`: A server implementation of the `AURA`_ specification. * :doc:`badfiles`: Check audio file integrity. * :doc:`embyupdate`: Automatically notifies `Emby`_ whenever the beets library changes. * :doc:`fish`: Adds `Fish shell`_ tab autocompletion to ``beet`` commands. @@ -205,6 +207,7 @@ Interoperability library changes. +.. _AURA: https://auraspec.readthedocs.io .. _Emby: https://emby.media .. _Fish shell: https://fishshell.com/ .. _Plex: https://plex.tv From 51c3f310e91afb275de818d8f63c0563aef9a174 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sun, 27 Sep 2020 18:09:16 +0100 Subject: [PATCH 013/130] AURA: Fix docstrings for tox -e lint --- beetsplug/aura.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 4269a4df8..8050cc621 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -14,8 +14,7 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""An AURA server using Flask. -""" +"""An AURA server using Flask.""" from __future__ import division, absolute_import, print_function @@ -811,7 +810,10 @@ def create_app(): class AURAPlugin(BeetsPlugin): + """The BeetsPlugin subclass for the AURA server plugin.""" + def __init__(self): + """Add configuration options for the AURA plugin.""" super(AURAPlugin, self).__init__() self.config.add( { @@ -824,6 +826,8 @@ class AURAPlugin(BeetsPlugin): ) def commands(self): + """Add subcommand used to run the AURA server.""" + def run_aura(lib, opts, args): """Run the application using Flask's built in-server.""" app = create_app() From e8aa96ef72edd8e9a8ff3f0967fd0d0c1caabb8e Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sun, 27 Sep 2020 18:58:39 +0100 Subject: [PATCH 014/130] AURA: Add argument info to docstrings Follows the google docstring style: https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings --- beetsplug/aura.py | 188 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 157 insertions(+), 31 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 8050cc621..f62dc2987 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -125,14 +125,25 @@ class AURADocument: @staticmethod def error(status, title, detail): - """Make a response for an error following the JSON:API spec.""" + """Make a response for an error following the JSON:API spec. + + Args: + status: An HTTP status code string, e.g. "404 Not Found". + title: A short, human-readable summary of the problem. + detail: A human-readable explanation specific to this + occurrence of the problem. + """ document = { "errors": [{"status": status, "title": title, "detail": detail}] } return make_response(document, status) def translate_attribute(self, aura_attr): - """Translate AURA attribute name to beets attribute name.""" + """Translate AURA attribute name to beets attribute name. + + Args: + aura_attr: The attribute name to convert, e.g. "title". + """ try: return self.attribute_map[aura_attr] except KeyError: @@ -161,7 +172,13 @@ class AURADocument: return AndQuery(queries) def translate_sorts(self, sort_arg): - """Translate an AURA sort parameter into a beets Sort.""" + """Translate an AURA sort parameter into a beets Sort. + + Args: + sort_arg: The value of the 'sort' query parameter; a comma + separated list of fields to sort by, in order. + E.g. "-year,title". + """ # Change HTTP query parameter to a list aura_sorts = sort_arg.strip(",").split(",") sorts = [] @@ -180,7 +197,13 @@ class AURADocument: return MultipleSort(sorts) def paginate(self, collection): - """Get a page of the collection and the URL to the next page.""" + """Get a page of the collection and the URL to the next page. + + Args: + collection: The raw data from which resource objects can be + built. Could be an sqlite3.Cursor object (tracks and + albums) or a list of strings (artists). + """ # Pages start from zero page = request.args.get("page", 0, int) # Use page limit defined in config by default. @@ -213,8 +236,10 @@ class AURADocument: def get_included(self, data, include_str): """Build a list of resource objects for inclusion. - :param data: Array of dicts in the form of resource objects - :param include_str: Comma separated list of resources to include + Args: + data: An array of dicts in the form of resource objects. + include_str: A comma separated list of resource types to + include. E.g. "tracks,images". """ # Change HTTP query parameter to a list to_include = include_str.strip(",").split(",") @@ -287,7 +312,12 @@ class AURADocument: return document def single_resource_document(self, resource_object): - """Build document for a specific requested resource.""" + """Build document for a specific requested resource. + + Args: + resource_object: A dictionary in the form of a JSON:API + resource object. + """ document = {"data": resource_object} include_str = request.args.get("include", None) if include_str is not None: @@ -304,11 +334,20 @@ class TrackDocument(AURADocument): attribute_map = TRACK_ATTR_MAP def get_collection(self, query=None, sort=None): - """Get Item objects from the library.""" + """Get Item objects from the library. + + Args: + query: A beets Query object or a beets query string. + sort: A beets Sort object. + """ return current_app.config["lib"].items(query, sort) def get_attribute_converter(self, beets_attr): - """Work out what data type an attribute should be for beets.""" + """Work out what data type an attribute should be for beets. + + Args: + beets_attr: The name of the beets attribute, e.g. "title". + """ # filesize is a special field (read from disk not db?) if beets_attr == "filesize": converter = int @@ -325,7 +364,11 @@ class TrackDocument(AURADocument): @staticmethod def resource_object(track): - """Construct a JSON:API resource object from a beets Item.""" + """Construct a JSON:API resource object from a beets Item. + + Args: + track: A beets Item object. + """ attributes = {} # Use aura => beets attribute map, e.g. size => filesize for aura_attr, beets_attr in TRACK_ATTR_MAP.items(): @@ -353,7 +396,11 @@ class TrackDocument(AURADocument): } def single_resource(self, track_id): - """Get track from the library and build a document.""" + """Get track from the library and build a document. + + Args: + track_id: The beets id of the track (integer). + """ track = current_app.config["lib"].get_item(track_id) if track is None: return self.error( @@ -372,12 +419,20 @@ class AlbumDocument(AURADocument): attribute_map = ALBUM_ATTR_MAP def get_collection(self, query=None, sort=None): - """Get Album objects from the library.""" + """Get Album objects from the library. + + Args: + query: A beets Query object or a beets query string. + sort: A beets Sort object. + """ return current_app.config["lib"].albums(query, sort) def get_attribute_converter(self, beets_attr): - """Work out what data type an attribute should be for beets.""" - # filesize is a special field (read from disk not db?) + """Work out what data type an attribute should be for beets. + + Args: + beets_attr: The name of the beets attribute, e.g. "title". + """ try: # Look for field in list of Album fields # and get python type of database type. @@ -390,7 +445,11 @@ class AlbumDocument(AURADocument): @staticmethod def resource_object(album): - """Construct a JSON:API resource object from a beets Album.""" + """Construct a JSON:API resource object from a beets Album. + + Args: + album: A beets Album object. + """ attributes = {} # Use aura => beets attribute name map for aura_attr, beets_attr in ALBUM_ATTR_MAP.items(): @@ -435,7 +494,11 @@ class AlbumDocument(AURADocument): } def single_resource(self, album_id): - """Get album from the library and build a document.""" + """Get album from the library and build a document. + + Args: + album_id: The beets id of the album (integer). + """ album = current_app.config["lib"].get_album(album_id) if album is None: return self.error( @@ -454,7 +517,12 @@ class ArtistDocument(AURADocument): attribute_map = ARTIST_ATTR_MAP def get_collection(self, query=None, sort=None): - """Get a list of artist names from the library.""" + """Get a list of artist names from the library. + + Args: + query: A beets Query object or a beets query string. + sort: A beets Sort object. + """ # Gets only tracks with matching artist information tracks = current_app.config["lib"].items(query, sort) collection = [] @@ -465,7 +533,11 @@ class ArtistDocument(AURADocument): return collection def get_attribute_converter(self, beets_attr): - """Work out what data type an attribute should be for beets.""" + """Work out what data type an attribute should be for beets. + + Args: + beets_attr: The name of the beets attribute, e.g. "artist". + """ try: # Look for field in list of Item fields # and get python type of database type. @@ -478,7 +550,11 @@ class ArtistDocument(AURADocument): @staticmethod def resource_object(artist_id): - """Construct a JSON:API resource object for the given artist.""" + """Construct a JSON:API resource object for the given artist. + + Args: + artist_id: A string which is the artist's name. + """ # Get tracks where artist field exactly matches artist_id query = MatchQuery("artist", artist_id) tracks = current_app.config["lib"].items(query) @@ -517,7 +593,11 @@ class ArtistDocument(AURADocument): } def single_resource(self, artist_id): - """Get info for the requested artist and build a document.""" + """Get info for the requested artist and build a document. + + Args: + artist_id: A string which is the artist's name. + """ artist_resource = self.resource_object(artist_id) if artist_resource is None: return self.error( @@ -538,7 +618,10 @@ class ImageDocument(AURADocument): """Works out the full path to the image with the given id. Returns None if there is no such image. - image_id is in the form -- + + Args: + image_id: A string in the form + "--". """ # Split image_id into its constituent parts id_split = image_id.split("-") @@ -572,7 +655,12 @@ class ImageDocument(AURADocument): @staticmethod def resource_object(image_id): - """Construct a JSON:API resource object for the given image.""" + """Construct a JSON:API resource object for the given image. + + Args: + image_id: A string in the form + "--". + """ # Could be called as a static method, so can't use # self.get_image_path() image_path = ImageDocument.get_image_path(image_id) @@ -609,7 +697,12 @@ class ImageDocument(AURADocument): } def single_resource(self, image_id): - """Get info for the requested image and build a document.""" + """Get info for the requested image and build a document. + + Args: + image_id: A string in the form + "--". + """ image_resource = self.resource_object(image_id) if image_resource is None: return self.error( @@ -644,14 +737,22 @@ def all_tracks(): @aura_bp.route("/tracks/") def single_track(track_id): - """Respond with info about the specified track.""" + """Respond with info about the specified track. + + Args: + track_id: The id of the track provided in the URL (integer). + """ doc = TrackDocument() return doc.single_resource(track_id) -@aura_bp.route("/tracks//audio") +@aura_bp.route("/tracks//audio") def audio_file(track_id): - """Supply an audio file for the specified track.""" + """Supply an audio file for the specified track. + + Args: + track_id: The id of the track provided in the URL (integer). + """ track = current_app.config["lib"].get_item(track_id) if track is None: return AURADocument.error( @@ -720,7 +821,11 @@ def all_albums(): @aura_bp.route("/albums/") def single_album(album_id): - """Respond with info about the specified album.""" + """Respond with info about the specified album. + + Args: + album_id: The id of the album provided in the URL (integer). + """ doc = AlbumDocument() return doc.single_resource(album_id) @@ -739,7 +844,12 @@ def all_artists(): # Using the path converter allows slashes in artist_id @aura_bp.route("/artists/") def single_artist(artist_id): - """Respond with info about the specified artist.""" + """Respond with info about the specified artist. + + Args: + artist_id: The id of the artist provided in the URL. A string + which is the artist's name. + """ doc = ArtistDocument() return doc.single_resource(artist_id) @@ -751,14 +861,24 @@ def single_artist(artist_id): @aura_bp.route("/images/") def single_image(image_id): - """Respond with info about the specified image.""" + """Respond with info about the specified image. + + Args: + image_id: The id of the image provided in the URL. A string in + the form "--". + """ doc = ImageDocument() return doc.single_resource(image_id) @aura_bp.route("/images//file") def image_file(image_id): - """Supply an image file for the specified image.""" + """Supply an image file for the specified image. + + Args: + image_id: The id of the image provided in the URL. A string in + the form "--". + """ img_path = ImageDocument.get_image_path(image_id) if img_path is None: return AURADocument.error( @@ -829,7 +949,13 @@ class AURAPlugin(BeetsPlugin): """Add subcommand used to run the AURA server.""" def run_aura(lib, opts, args): - """Run the application using Flask's built in-server.""" + """Run the application using Flask's built in-server. + + Args: + lib: A beets Library object (not used). + opts: Command line options. An optparse.Values object. + args: The list of arguments to process (not used). + """ app = create_app() # Start the built-in server (not intended for production) app.run( From e067298224fa983b0bf957bbb8595c9746f7f7ae Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Wed, 28 Oct 2020 18:53:39 +0000 Subject: [PATCH 015/130] Add default config values in create_app() So if not run through beet aura then default values will be available --- beetsplug/aura.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index f62dc2987..6cc965b21 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -896,6 +896,16 @@ def image_file(image_id): def create_app(): """An application factory for use by a WSGI server.""" + config["aura"].add( + { + "host": u"127.0.0.1", + "port": 8337, + "cors": [], + "cors_supports_credentials": False, + "page_limit": 500, + } + ) + app = Flask(__name__) # Register AURA blueprint view functions under a URL prefix app.register_blueprint(aura_bp, url_prefix="/aura") @@ -935,15 +945,6 @@ class AURAPlugin(BeetsPlugin): def __init__(self): """Add configuration options for the AURA plugin.""" super(AURAPlugin, self).__init__() - self.config.add( - { - "host": u"127.0.0.1", - "port": 8337, - "cors": [], - "cors_supports_credentials": False, - "page_limit": 500, - } - ) def commands(self): """Add subcommand used to run the AURA server.""" From 72c041255244b8fc767f30079ae643a881e4e618 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Fri, 30 Oct 2020 12:27:22 +0000 Subject: [PATCH 016/130] AURA: Small updates to documentation --- docs/plugins/aura.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/plugins/aura.rst b/docs/plugins/aura.rst index 2891d25e1..1cadb45db 100644 --- a/docs/plugins/aura.rst +++ b/docs/plugins/aura.rst @@ -69,8 +69,8 @@ Configuration To configure the plugin, make an ``aura:`` section in your configuration file. The available options are: -- **host**: The server hostname. Set this to 0.0.0.0 to bind to all interfaces. - Default: ``127.0.0.1``. +- **host**: The server hostname. Set this to ``0.0.0.0`` to bind to all + interfaces. Default: ``127.0.0.1``. - **port**: The server port. Default: ``8337``. - **cors**: A YAML list of origins to allow CORS requests from (see @@ -122,7 +122,7 @@ then you can! The ``beetsplug.aura`` module provides a WSGI callable called For example to run the AURA server using `gunicorn`_ use ``gunicorn 'beetsplug.aura:create_app()'``, or for `uWSGI`_ use -``uwsgi --http :9090 --module 'beetsplug.aura:create_app()'``. +``uwsgi --http :8337 --module 'beetsplug.aura:create_app()'``. Note that these commands just show how to use the AURA app and you would probably use something a bit different in a production environment. Read the relevant server's documentation to figure out what you need. @@ -144,8 +144,8 @@ likely this would require changes in the code. Do not add a trailing slash (``/``) to the URL where the application is running, otherwise you will get a 404. For example with NGINX you should use -``proxy_pass http://127.0.0.1:8000;`` rather than -``proxy_pass http://127.0.0.1:8000/;``. +``proxy_pass http://127.0.0.1:8337;`` rather than +``proxy_pass http://127.0.0.1:8337/;``. .. _Flask proxy setups: https://flask.palletsprojects.com/en/1.1.x/deploying/wsgi-standalone/#proxy-setups @@ -184,9 +184,9 @@ related images at the moment. This could be expanded to looking in the same directory for other images, and relating tracks to their album's image. There are likely to be some performance issues, especially with larger -libraries. Pagination and inclusion (most notably of images) are probably two -of the main offenders. On a related note, the program attempts to import Pillow -every time it constructs an image resource object, which is not very good. +libraries. Sorting, pagination and inclusion (most notably of images) are +probably the main offenders. On a related note, the program attempts to import +Pillow every time it constructs an image resource object, which is not good. The beets library is accessed using a so called private function (with a single leading underscore) ``beets.ui.__init__._open_library()``. This shouldn't cause From 2d024d2f38fcc8ac7a134ec290cb6a6b9f3b5ca4 Mon Sep 17 00:00:00 2001 From: FichteFoll Date: Mon, 2 Nov 2020 01:06:05 +0100 Subject: [PATCH 017/130] Avoid overeager inclusion of album attributes Co-Authored-By: Curtis Rueden --- beets/dbcore/db.py | 5 +++-- beets/library.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 524ee0b00..409ecc9af 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -56,10 +56,11 @@ class FormattedMapping(Mapping): are replaced. """ - def __init__(self, model, for_path=False): + def __init__(self, model, for_path=False, compute_keys=True): self.for_path = for_path self.model = model - self.model_keys = model.keys(True) + if compute_keys: + self.model_keys = model.keys(True) def __getitem__(self, key): if key in self.model_keys: diff --git a/beets/library.py b/beets/library.py index 5e6a0ec8a..f9b5a2ab4 100644 --- a/beets/library.py +++ b/beets/library.py @@ -375,9 +375,10 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): """ def __init__(self, item, for_path=False): - super(FormattedItemMapping, self).__init__(item, for_path) # We treat album and item keys specially here, # so exclude transitive album keys from the model's keys. + super(FormattedItemMapping, self).__init__(item, for_path, + compute_keys=False) self.model_keys = item.keys(computed=True, with_album=False) self.item = item From c2a92fdbf8849fdd558afb8c5df9c5f13d94333c Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sat, 21 Nov 2020 18:40:44 +0000 Subject: [PATCH 018/130] AURA: Update reverse proxy docs --- docs/plugins/aura.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/plugins/aura.rst b/docs/plugins/aura.rst index 1cadb45db..7af8b19e6 100644 --- a/docs/plugins/aura.rst +++ b/docs/plugins/aura.rst @@ -138,14 +138,13 @@ The plugin should work behind a reverse proxy without further configuration, however this has not been tested extensively. For details of what headers must be rewritten and a sample NGINX configuration see `Flask proxy setups`_. -It may be possibly to run the application under a URL prefix (for example so -you could have ``/foo/aura/server`` rather than ``/aura/server``), but it is -likely this would require changes in the code. +It is (reportedly) possible to run the application under a URL prefix (for +example so you could have ``/foo/aura/server`` rather than ``/aura/server``), +but you'll have to work it out for yourself :-) -Do not add a trailing slash (``/``) to the URL where the application is -running, otherwise you will get a 404. For example with NGINX you should use -``proxy_pass http://127.0.0.1:8337;`` rather than -``proxy_pass http://127.0.0.1:8337/;``. +If using NGINX, do **not** add a trailing slash (``/``) to the URL where the +application is running, otherwise you will get a 404. However if you are using +Apache then you **should** add a trailing slash. .. _Flask proxy setups: https://flask.palletsprojects.com/en/1.1.x/deploying/wsgi-standalone/#proxy-setups From 414b6821230c39ddd6e070a88764ec1d13129e31 Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Mon, 22 Feb 2021 11:19:23 +0100 Subject: [PATCH 019/130] Add strip_path to mpdstats --- beetsplug/mpdstats.py | 10 +++++++++- docs/plugins/mpdstats.rst | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 599aa7631..3daa08fe2 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -53,6 +53,9 @@ class MPDClientWrapper(object): self.music_directory = ( mpd_config['music_directory'].as_str()) + self.strip_path = ( + mpd_config['strip_path'].as_str()) + if sys.version_info < (3, 0): # On Python 2, use_unicode will enable the utf-8 mode for # python-mpd2 @@ -118,12 +121,16 @@ class MPDClientWrapper(object): """Return the path to the currently playing song, along with its songid. Prefixes paths with the music_directory, to get the absolute path. + In some cases, we need to remove the local path from MPD server, + we replace 'strip_path' with ''. + `strip_path` defaults to ''. """ result = None entry = self.get('currentsong') if 'file' in entry: if not is_url(entry['file']): - result = os.path.join(self.music_directory, entry['file']) + file = entry['file'].replace(self.strip_path, '') + result = os.path.join(self.music_directory, file) else: result = entry['file'] return result, entry.get('id') @@ -334,6 +341,7 @@ class MPDStatsPlugin(plugins.BeetsPlugin): super(MPDStatsPlugin, self).__init__() mpd_config.add({ 'music_directory': config['directory'].as_filename(), + 'strip_path': u'', 'rating': True, 'rating_mix': 0.75, 'host': os.environ.get('MPD_HOST', u'localhost'), diff --git a/docs/plugins/mpdstats.rst b/docs/plugins/mpdstats.rst index de9b2ca59..33b3aa43a 100644 --- a/docs/plugins/mpdstats.rst +++ b/docs/plugins/mpdstats.rst @@ -53,6 +53,9 @@ configuration file. The available options are: - **music_directory**: If your MPD library is at a different location from the beets library (e.g., because one is mounted on a NFS share), specify the path here. +- **strip_path**: If your MPD library contains local path, specify the part to remove + here. Combining this with **music_directory** you can mangle MPD path to match the + **beets library** one. Default: The beets library directory. - **rating**: Enable rating updates. Default: ``yes``. From af2229c1b7a149444fa8276cf069f4eaefc15ffd Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Mon, 22 Feb 2021 14:28:14 +0100 Subject: [PATCH 020/130] Fix docs and changelog --- docs/changelog.rst | 2 ++ docs/plugins/mpdstats.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2a62e6d8..cabebdd5b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog New features: +* :doc:`/plugins/mpdstats`: Add strip_path option to help build the right local path + from MPD information * Submitting acoustID information on tracks which already have a fingerprint :bug:`3834` * conversion uses par_map to parallelize conversion jobs in python3 diff --git a/docs/plugins/mpdstats.rst b/docs/plugins/mpdstats.rst index 33b3aa43a..c5adbc64b 100644 --- a/docs/plugins/mpdstats.rst +++ b/docs/plugins/mpdstats.rst @@ -55,7 +55,7 @@ configuration file. The available options are: here. - **strip_path**: If your MPD library contains local path, specify the part to remove here. Combining this with **music_directory** you can mangle MPD path to match the - **beets library** one. + beets library one. Default: The beets library directory. - **rating**: Enable rating updates. Default: ``yes``. From ce974d4fc7c3974c0c7c60a02b26f0586bf2a184 Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Mon, 22 Feb 2021 14:30:50 +0100 Subject: [PATCH 021/130] One line :) --- beetsplug/mpdstats.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 3daa08fe2..8c32d5867 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -50,11 +50,9 @@ class MPDClientWrapper(object): def __init__(self, log): self._log = log - self.music_directory = ( - mpd_config['music_directory'].as_str()) + self.music_directory = (mpd_config['music_directory'].as_str()) - self.strip_path = ( - mpd_config['strip_path'].as_str()) + self.strip_path = (mpd_config['strip_path'].as_str()) if sys.version_info < (3, 0): # On Python 2, use_unicode will enable the utf-8 mode for From 99de85b5c295d673dda1e43bbd7220b8aa4cbddc Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Mon, 22 Feb 2021 14:33:01 +0100 Subject: [PATCH 022/130] Only leftmost subpath is replaced --- beetsplug/mpdstats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 8c32d5867..37093a97a 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -127,7 +127,7 @@ class MPDClientWrapper(object): entry = self.get('currentsong') if 'file' in entry: if not is_url(entry['file']): - file = entry['file'].replace(self.strip_path, '') + file = entry['file'].replace(self.strip_path, '', count=1) result = os.path.join(self.music_directory, file) else: result = entry['file'] From 921bc424cd3774d6dea3e74e0f0f77b009f3bcaa Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Mon, 22 Feb 2021 14:47:58 +0100 Subject: [PATCH 023/130] Better handling of strip_path --- beetsplug/mpdstats.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 37093a97a..be543229d 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -127,8 +127,9 @@ class MPDClientWrapper(object): entry = self.get('currentsong') if 'file' in entry: if not is_url(entry['file']): - file = entry['file'].replace(self.strip_path, '', count=1) - result = os.path.join(self.music_directory, file) + if entry.startswith(self.strip_path): + entry = entry[len(self.strip_path):] + result = os.path.join(self.music_directory, entry) else: result = entry['file'] return result, entry.get('id') From 0e67b6801c2d4fa1071df7661b62cadb060ac5c4 Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Mon, 22 Feb 2021 14:50:30 +0100 Subject: [PATCH 024/130] ARGH! Wrong variable --- beetsplug/mpdstats.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index be543229d..8047a5c94 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -127,9 +127,10 @@ class MPDClientWrapper(object): entry = self.get('currentsong') if 'file' in entry: if not is_url(entry['file']): - if entry.startswith(self.strip_path): - entry = entry[len(self.strip_path):] - result = os.path.join(self.music_directory, entry) + file = entry['file'] + if file.startswith(self.strip_path): + file = file[len(self.strip_path):] + result = os.path.join(self.music_directory, file) else: result = entry['file'] return result, entry.get('id') From 48e7d2964e2fd48985ad1a4eb6c00d025ba0d723 Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Tue, 23 Feb 2021 05:45:53 +0100 Subject: [PATCH 025/130] Style changes --- beetsplug/mpdstats.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index 8047a5c94..b4e04a1f4 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -50,9 +50,8 @@ class MPDClientWrapper(object): def __init__(self, log): self._log = log - self.music_directory = (mpd_config['music_directory'].as_str()) - - self.strip_path = (mpd_config['strip_path'].as_str()) + self.music_directory = mpd_config['music_directory'].as_str() + self.strip_path = mpd_config['strip_path'].as_str() if sys.version_info < (3, 0): # On Python 2, use_unicode will enable the utf-8 mode for From 1a65501cee664f32199467ffa950760d4082c141 Mon Sep 17 00:00:00 2001 From: Andrea Mistrali Date: Tue, 23 Feb 2021 10:40:59 +0100 Subject: [PATCH 026/130] Clean up strip_path and logging --- beetsplug/mpdstats.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beetsplug/mpdstats.py b/beetsplug/mpdstats.py index b4e04a1f4..5c8402329 100644 --- a/beetsplug/mpdstats.py +++ b/beetsplug/mpdstats.py @@ -53,6 +53,13 @@ class MPDClientWrapper(object): self.music_directory = mpd_config['music_directory'].as_str() self.strip_path = mpd_config['strip_path'].as_str() + # Ensure strip_path end with '/' + if not self.strip_path.endswith('/'): + self.strip_path += '/' + + self._log.debug('music_directory: {0}', self.music_directory) + self._log.debug('strip_path: {0}', self.strip_path) + if sys.version_info < (3, 0): # On Python 2, use_unicode will enable the utf-8 mode for # python-mpd2 @@ -132,6 +139,7 @@ class MPDClientWrapper(object): result = os.path.join(self.music_directory, file) else: result = entry['file'] + self._log.debug('returning: {0}', result) return result, entry.get('id') def status(self): From 00252ab28ffc7d8e25cd5e420b6c086bfc178023 Mon Sep 17 00:00:00 2001 From: George Rawlinson Date: Thu, 25 Feb 2021 20:00:34 +1300 Subject: [PATCH 027/130] Fix #3608: Replace discogs-client with python3-discogs-client discogs-client has been deprecated since June 2020, the replacement is actively developed by the community and does not have any breaking API changes. Signed-off-by: George Rawlinson --- beetsplug/discogs.py | 2 +- docs/changelog.rst | 2 ++ docs/plugins/discogs.rst | 8 ++++---- setup.py | 9 +++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index b1b1593cd..05abb80c2 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -14,7 +14,7 @@ # included in all copies or substantial portions of the Software. """Adds Discogs album search support to the autotagger. Requires the -discogs-client library. +python3-discogs-client library. """ from __future__ import division, absolute_import, print_function diff --git a/docs/changelog.rst b/docs/changelog.rst index cabebdd5b..f8debb3da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -302,6 +302,8 @@ Fixes: :bug:`3798` * Fix :bug:`3308` by using browsing for big releases to retrieve additional information. Thanks to :user:`dosoe`. +* :doc:`/plugins/discogs`: Replace deprecated discogs-client library with community + supported python3-discogs-client library. :bug:`3608` For plugin developers: diff --git a/docs/plugins/discogs.rst b/docs/plugins/discogs.rst index c199ccf49..40875b022 100644 --- a/docs/plugins/discogs.rst +++ b/docs/plugins/discogs.rst @@ -10,9 +10,9 @@ Installation ------------ To use the ``discogs`` plugin, first enable it in your configuration (see -:ref:`using-plugins`). Then, install the `discogs-client`_ library by typing:: +:ref:`using-plugins`). Then, install the `python3-discogs-client`_ library by typing:: - pip install discogs-client + pip install python3-discogs-client You will also need to register for a `Discogs`_ account, and provide authentication credentials via a personal access token or an OAuth2 @@ -36,7 +36,7 @@ Authentication via Personal Access Token As an alternative to OAuth, you can get a token from Discogs and add it to your configuration. -To get a personal access token (called a "user token" in the `discogs-client`_ +To get a personal access token (called a "user token" in the `python3-discogs-client`_ documentation), login to `Discogs`_, and visit the `Developer settings page `_. Press the ``Generate new @@ -89,4 +89,4 @@ Here are two things you can try: * Make sure that your system clock is accurate. The Discogs servers can reject your request if your clock is too out of sync. -.. _discogs-client: https://github.com/discogs/discogs_client +.. _python3-discogs-client: https://github.com/joalla/discogs_client diff --git a/setup.py b/setup.py index a6cfc9310..ec1d2c819 100755 --- a/setup.py +++ b/setup.py @@ -113,7 +113,6 @@ setup( 'test': [ 'beautifulsoup4', 'coverage', - 'discogs-client', 'flask', 'mock', 'pylast', @@ -128,6 +127,9 @@ setup( ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] ) + [ 'rarfile<4' if sys.version_info < (3, 6, 0) else 'rarfile', + ] + [ + 'discogs-client' if (sys.version_info < (3, 0, 0)) + else 'python3-discogs-client' ], 'lint': [ 'flake8', @@ -144,7 +146,10 @@ setup( 'embyupdate': ['requests'], 'chroma': ['pyacoustid'], 'gmusic': ['gmusicapi'], - 'discogs': ['discogs-client>=2.2.1'], + 'discogs': ( + ['discogs-client' if (sys.version_info < (3, 0, 0)) + else 'python3-discogs-client'] + ), 'beatport': ['requests-oauthlib>=0.6.1'], 'kodiupdate': ['requests'], 'lastgenre': ['pylast'], From 1b1209a6c31e3c801c11ec5f2028be52e8183d5a Mon Sep 17 00:00:00 2001 From: Adam Tygart Date: Sat, 27 Feb 2021 14:02:19 -0600 Subject: [PATCH 028/130] Add trackdisambig from musicbrainz. fixes beetbox/beets#1904 --- beets/autotag/hooks.py | 3 ++- beets/autotag/mb.py | 3 +++ beets/library.py | 1 + docs/changelog.rst | 3 +++ test/test_mb.py | 17 ++++++++++++++++- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 065d88170..c86579322 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -163,7 +163,7 @@ class TrackInfo(AttrDict): composer=None, composer_sort=None, arranger=None, track_alt=None, work=None, mb_workid=None, work_disambig=None, bpm=None, initial_key=None, genre=None, - **kwargs): + trackdisambig=None, **kwargs): self.title = title self.track_id = track_id self.release_track_id = release_track_id @@ -191,6 +191,7 @@ class TrackInfo(AttrDict): self.bpm = bpm self.initial_key = initial_key self.genre = genre + self.trackdisambig = trackdisambig self.update(kwargs) # As above, work around a bug in python-musicbrainz-ngs. diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 03ea5b382..bdfc0d0c9 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -223,6 +223,9 @@ def track_info(recording, index=None, medium=None, medium_index=None, if recording.get('length'): info.length = int(recording['length']) / (1000.0) + if recording.get('disambiguation'): + info.trackdisambig = recording.get('disambiguation') + lyricist = [] composer = [] composer_sort = [] diff --git a/beets/library.py b/beets/library.py index a060e93d6..78552bb61 100644 --- a/beets/library.py +++ b/beets/library.py @@ -477,6 +477,7 @@ class Item(LibModel): 'mb_artistid': types.STRING, 'mb_albumartistid': types.STRING, 'mb_releasetrackid': types.STRING, + 'trackdisambig': types.STRING, 'albumtype': types.STRING, 'label': types.STRING, 'acoustid_fingerprint': types.STRING, diff --git a/docs/changelog.rst b/docs/changelog.rst index e2a62e6d8..ce1cc6c01 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,6 +178,9 @@ New features: :bug:`3478` * Removes usage of the bs1770gain replaygain backend. Thanks to :user:`SamuelCook`. +* Added ``trackdisambig`` which stores the recording disambiguation from + musicbrainz for each track. + :bug:`1904` Fixes: diff --git a/test/test_mb.py b/test/test_mb.py index de1ffd9a7..9eca57c80 100644 --- a/test/test_mb.py +++ b/test/test_mb.py @@ -111,7 +111,8 @@ class MBAlbumInfoTest(_common.TestCase): }) return release - def _make_track(self, title, tr_id, duration, artist=False, video=False): + def _make_track(self, title, tr_id, duration, artist=False, video=False, + disambiguation=None): track = { 'title': title, 'id': tr_id, @@ -131,6 +132,8 @@ class MBAlbumInfoTest(_common.TestCase): ] if video: track['video'] = 'true' + if disambiguation: + track['disambiguation'] = disambiguation return track def test_parse_release_with_year(self): @@ -445,6 +448,18 @@ class MBAlbumInfoTest(_common.TestCase): self.assertEqual(d.tracks[1].title, 'TITLE TWO') self.assertEqual(d.tracks[2].title, 'TITLE VIDEO') + def test_track_disambiguation(self): + tracks = [self._make_track('TITLE ONE', 'ID ONE', 100.0 * 1000.0), + self._make_track('TITLE TWO', 'ID TWO', 200.0 * 1000.0, + disambiguation="SECOND TRACK")] + release = self._make_release(tracks=tracks) + + d = mb.album_info(release) + t = d.tracks + self.assertEqual(len(t), 2) + self.assertEqual(t[0].trackdisambig, None) + self.assertEqual(t[1].trackdisambig, "SECOND TRACK") + class ParseIDTest(_common.TestCase): def test_parse_id_correct(self): From f3e7edb21ed0c710768908ab8f514b6067aaf5f2 Mon Sep 17 00:00:00 2001 From: Adam Tygart Date: Sat, 27 Feb 2021 15:23:35 -0600 Subject: [PATCH 029/130] Attempt to address the comments from sampsyo Undoes the changes to autotag/hooks.py If trackdisambig is not set to None at some point in time, the test case for a non-existent trackdisambig fails. as such, try to get the disambiguation with a default of None when setting trackdisambig. Not sure if this is reasonable. --- beets/autotag/hooks.py | 3 +-- beets/autotag/mb.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index c86579322..065d88170 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -163,7 +163,7 @@ class TrackInfo(AttrDict): composer=None, composer_sort=None, arranger=None, track_alt=None, work=None, mb_workid=None, work_disambig=None, bpm=None, initial_key=None, genre=None, - trackdisambig=None, **kwargs): + **kwargs): self.title = title self.track_id = track_id self.release_track_id = release_track_id @@ -191,7 +191,6 @@ class TrackInfo(AttrDict): self.bpm = bpm self.initial_key = initial_key self.genre = genre - self.trackdisambig = trackdisambig self.update(kwargs) # As above, work around a bug in python-musicbrainz-ngs. diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index bdfc0d0c9..740f0fcae 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -223,8 +223,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, if recording.get('length'): info.length = int(recording['length']) / (1000.0) - if recording.get('disambiguation'): - info.trackdisambig = recording.get('disambiguation') + info.trackdisambig = recording.get('disambiguation', None) lyricist = [] composer = [] From ce78f75ce9d2461b145b0190c32b8281a5bd250c Mon Sep 17 00:00:00 2001 From: Adam Tygart Date: Sat, 27 Feb 2021 18:31:16 -0600 Subject: [PATCH 030/130] Further attempt to address the comments from sampsyo --- beets/autotag/mb.py | 2 +- docs/changelog.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 740f0fcae..3ca5463c2 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -223,7 +223,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, if recording.get('length'): info.length = int(recording['length']) / (1000.0) - info.trackdisambig = recording.get('disambiguation', None) + info.trackdisambig = recording.get('disambiguation') lyricist = [] composer = [] diff --git a/docs/changelog.rst b/docs/changelog.rst index ce1cc6c01..69f64456c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -179,7 +179,7 @@ New features: * Removes usage of the bs1770gain replaygain backend. Thanks to :user:`SamuelCook`. * Added ``trackdisambig`` which stores the recording disambiguation from - musicbrainz for each track. + MusicBrainz for each track. :bug:`1904` Fixes: From dc55480f51bf93d11042acc03ce797d3f05b8c97 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 15:27:00 +0000 Subject: [PATCH 031/130] Document "id" handling in test setup and add test to confirm self.lib.add ignores the "id" field from the entry being added and overwrites it with the next free number. So remove the misleading id fields. Add a test to confirm that the album query response contains the expected album id number. Signed-off-by: Graham R. Cobb --- test/test_web.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index e7d1f334f..e0022e60f 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -22,10 +22,15 @@ class WebPluginTest(_common.LibTestCase): # Add fixtures for track in self.lib.items(): track.remove() - self.lib.add(Item(title=u'title', path='/path_1', id=1)) - self.lib.add(Item(title=u'another title', path='/path_2', id=2)) - self.lib.add(Album(album=u'album', id=3)) - self.lib.add(Album(album=u'another album', id=4)) + + # Add library elements. Note that self.lib.add overrides any "id=" and assigns + # the next free id number. + # The following adds will create items #1 and #2 + self.lib.add(Item(title=u'title', path='/path_1')) + self.lib.add(Item(title=u'another title', path='/path_2', album_id=2)) + # The following adds will create albums #1 and #2 + self.lib.add(Album(album=u'album')) + self.lib.add(Album(album=u'another album')) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -140,6 +145,15 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) self.assertEqual(len(res_json['albums']), 2) + def test_get_simple_album_query(self): + response = self.client.get('/album/query/another') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['album'], + u'another album') + self.assertEqual(res_json['results'][0]['id'], 2) def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 79b41f6f3852ec498794b199fefec5f4a627f18a Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 15:56:46 +0000 Subject: [PATCH 032/130] Add test for getting album track listing using ?expand=1 Added test_get_album_details to test fetching /album/2?expand=1. Also changed second album name to make tests more robust. Signed-off-by: Graham R. Cobb --- test/test_web.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index e0022e60f..6cf0d2dbb 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -26,11 +26,11 @@ class WebPluginTest(_common.LibTestCase): # Add library elements. Note that self.lib.add overrides any "id=" and assigns # the next free id number. # The following adds will create items #1 and #2 - self.lib.add(Item(title=u'title', path='/path_1')) - self.lib.add(Item(title=u'another title', path='/path_2', album_id=2)) + self.lib.add(Item(title=u'title', path='/path_1', album_id=2)) + self.lib.add(Item(title=u'another title', path='/path_2')) # The following adds will create albums #1 and #2 self.lib.add(Album(album=u'album')) - self.lib.add(Album(album=u'another album')) + self.lib.add(Album(album=u'other album')) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -120,7 +120,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) response_albums = [album['album'] for album in res_json['albums']] - assertCountEqual(self, response_albums, [u'album', u'another album']) + assertCountEqual(self, response_albums, [u'album', u'other album']) def test_get_single_album_by_id(self): response = self.client.get('/album/2') @@ -128,7 +128,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) self.assertEqual(res_json['id'], 2) - self.assertEqual(res_json['album'], u'another album') + self.assertEqual(res_json['album'], u'other album') def test_get_multiple_albums_by_id(self): response = self.client.get('/album/1,2') @@ -136,7 +136,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) response_albums = [album['album'] for album in res_json['albums']] - assertCountEqual(self, response_albums, [u'album', u'another album']) + assertCountEqual(self, response_albums, [u'album', u'other album']) def test_get_album_empty_query(self): response = self.client.get('/album/query/') @@ -146,15 +146,25 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(len(res_json['albums']), 2) def test_get_simple_album_query(self): - response = self.client.get('/album/query/another') + response = self.client.get('/album/query/other') res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) self.assertEqual(len(res_json['results']), 1) self.assertEqual(res_json['results'][0]['album'], - u'another album') + u'other album') self.assertEqual(res_json['results'][0]['id'], 2) + def test_get_album_details(self): + response = self.client.get('/album/2?expand=1') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['items']), 1) + self.assertEqual(res_json['items'][0]['album'], + u'other album') + self.assertEqual(res_json['items'][0]['id'], 1) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 15be534e7cc80f6056c0e421ccb51ab3487b5a5a Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 16:17:14 +0000 Subject: [PATCH 033/130] Test ?expand using documented syntax Documentation says that ?expand does not need a value (i.e. ?expand=1 is wrong), so the test is changed to reflect that syntax. Signed-off-by: Graham R. Cobb --- test/test_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_web.py b/test/test_web.py index 6cf0d2dbb..34a22f8d1 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -156,7 +156,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(res_json['results'][0]['id'], 2) def test_get_album_details(self): - response = self.client.get('/album/2?expand=1') + response = self.client.get('/album/2?expand') res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) From de58334ecb45a5c3dc10f23336530073c2ac54c6 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 16:29:26 +0000 Subject: [PATCH 034/130] Test web API /stats Added a test (test_get_stats) for the /stats web API. This involved adding another item so that we can check both items and albums are being counted correctly, which required a few small changes to some item tests. Signed-off-by: Graham R. Cobb --- test/test_web.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index 34a22f8d1..d4f365c73 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -25,9 +25,10 @@ class WebPluginTest(_common.LibTestCase): # Add library elements. Note that self.lib.add overrides any "id=" and assigns # the next free id number. - # The following adds will create items #1 and #2 + # The following adds will create items #1, #2 and #3 self.lib.add(Item(title=u'title', path='/path_1', album_id=2)) self.lib.add(Item(title=u'another title', path='/path_2')) + self.lib.add(Item(title=u'and a third')) # The following adds will create albums #1 and #2 self.lib.add(Album(album=u'album')) self.lib.add(Album(album=u'other album')) @@ -58,7 +59,7 @@ class WebPluginTest(_common.LibTestCase): res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(len(res_json['items']), 2) + self.assertEqual(len(res_json['items']), 3) def test_get_single_item_by_id(self): response = self.client.get('/item/1') @@ -78,7 +79,7 @@ class WebPluginTest(_common.LibTestCase): assertCountEqual(self, response_titles, [u'title', u'another title']) def test_get_single_item_not_found(self): - response = self.client.get('/item/3') + response = self.client.get('/item/4') self.assertEqual(response.status_code, 404) def test_get_single_item_by_path(self): @@ -103,7 +104,7 @@ class WebPluginTest(_common.LibTestCase): res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(len(res_json['items']), 2) + self.assertEqual(len(res_json['items']), 3) def test_get_simple_item_query(self): response = self.client.get('/item/query/another') @@ -165,6 +166,14 @@ class WebPluginTest(_common.LibTestCase): u'other album') self.assertEqual(res_json['items'][0]['id'], 1) + def test_get_stats(self): + response = self.client.get('/stats') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['items'], 3) + self.assertEqual(res_json['albums'], 2) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 3846e3519d1ea38f876b3e05a51e074574fee4f3 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 16:38:03 +0000 Subject: [PATCH 035/130] Update changelog to mention small test coverage improvement for web plugin. Signed-off-by: Graham R. Cobb --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f31ecfe3..ecc926a33 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -186,6 +186,7 @@ New features: Fixes: +* Small improvement to ``web`` plugin test coverage (for `expand` and `stats` features) * :bug:`/plugins/discogs`: Fixed a bug with ``index_tracks`` options that sometimes caused the index to be discarded. Also remove the extra semicolon that was added when there is no index track. From 7d3fb0d7ecf42b531b2d4edeb355defba84c3bc0 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 16:52:21 +0000 Subject: [PATCH 036/130] Fix lint errors Signed-off-by: Graham R. Cobb --- test/test_web.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index d4f365c73..e9ca028d9 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -23,8 +23,8 @@ class WebPluginTest(_common.LibTestCase): for track in self.lib.items(): track.remove() - # Add library elements. Note that self.lib.add overrides any "id=" and assigns - # the next free id number. + # Add library elements. Note that self.lib.add overrides any "id=" + # and assigns the next free id number. # The following adds will create items #1, #2 and #3 self.lib.add(Item(title=u'title', path='/path_1', album_id=2)) self.lib.add(Item(title=u'another title', path='/path_2')) @@ -174,6 +174,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(res_json['items'], 3) self.assertEqual(res_json['albums'], 2) + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From d2a125175654fb6390ff68dcf9217a0fd85feed2 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 5 Mar 2021 22:42:27 +0000 Subject: [PATCH 037/130] Drop changelog as requested by @sampsyo. Signed-off-by: Graham R. Cobb --- docs/changelog.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ecc926a33..2f31ecfe3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -186,7 +186,6 @@ New features: Fixes: -* Small improvement to ``web`` plugin test coverage (for `expand` and `stats` features) * :bug:`/plugins/discogs`: Fixed a bug with ``index_tracks`` options that sometimes caused the index to be discarded. Also remove the extra semicolon that was added when there is no index track. From 7e819d2a2eb06c7142622b29109c7593dbcbe90a Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sat, 6 Mar 2021 12:01:02 +0000 Subject: [PATCH 038/130] AURA: Update artist-mbid attribute to use '-' --- beetsplug/aura.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 6cc965b21..60507b78f 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -116,7 +116,7 @@ ARTIST_ATTR_MAP = { # Required "name": "artist", # Optional - "artist_mbid": "mb_artistid", + "artist-mbid": "mb_artistid", } From b1baeb37f1d3e563b5de6f0f8ea068768fa4d797 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sat, 6 Mar 2021 12:08:35 +0000 Subject: [PATCH 039/130] AURA: replace translate_attribute with a dict.get --- beetsplug/aura.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 60507b78f..536448d58 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -138,18 +138,6 @@ class AURADocument: } return make_response(document, status) - def translate_attribute(self, aura_attr): - """Translate AURA attribute name to beets attribute name. - - Args: - aura_attr: The attribute name to convert, e.g. "title". - """ - try: - return self.attribute_map[aura_attr] - except KeyError: - # Assume native beets attribute - return aura_attr - def translate_filters(self): """Translate filters from request arguments to a beets Query.""" # The format of each filter key in the request parameter is: @@ -162,7 +150,7 @@ class AURADocument: # Extract attribute name from key aura_attr = match.group("attribute") # Get the beets version of the attribute name - beets_attr = self.translate_attribute(aura_attr) + beets_attr = self.attribute_map.get(aura_attr, aura_attr) converter = self.get_attribute_converter(beets_attr) value = converter(value) # Add exact match query to list @@ -191,7 +179,7 @@ class AURADocument: # JSON:API default ascending = True # Get the beets version of the attribute name - beets_attr = self.translate_attribute(aura_attr) + beets_attr = self.attribute_map.get(aura_attr, aura_attr) # Use slow sort so it works with all fields (inc. computed) sorts.append(SlowFieldSort(beets_attr, ascending=ascending)) return MultipleSort(sorts) From 07cfaaa3b378afa1f8be43a91ebccad29e1650e2 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sat, 6 Mar 2021 12:25:33 +0000 Subject: [PATCH 040/130] AURA: Simplify if statements Get rid of ` is not None` Change `len(x) == 0` to `not x` Change `x is None` to `not x` --- beetsplug/aura.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 536448d58..5a949dcf6 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -146,7 +146,7 @@ class AURADocument: queries = [] for key, value in request.args.items(): match = pattern.match(key) - if match is not None: + if match: # Extract attribute name from key aura_attr = match.group("attribute") # Get the beets version of the attribute name @@ -206,10 +206,10 @@ class AURADocument: next_url = None else: # Not the last page so work out links.next url - if len(request.args) == 0: + if not request.args: # No existing arguments, so current page is 0 next_url = request.url + "?page=1" - elif request.args.get("page", None) is None: + elif not request.args.get("page", None): # No existing page argument, so add one to the end next_url = request.url + "&page=1" else: @@ -272,7 +272,7 @@ class AURADocument: """Build document for /tracks, /albums or /artists.""" query = self.translate_filters() sort_arg = request.args.get("sort", None) - if sort_arg is not None: + if sort_arg: sort = self.translate_sorts(sort_arg) # For each sort field add a query which ensures all results # have a non-empty, non-zero value for that field. @@ -291,11 +291,11 @@ class AURADocument: data, next_url = self.paginate(collection) document = {"data": data} # If there are more pages then provide a way to access them - if next_url is not None: + if next_url: document["links"] = {"next": next_url} # Include related resources for each element in "data" include_str = request.args.get("include", None) - if include_str is not None: + if include_str: document["included"] = self.get_included(data, include_str) return document @@ -308,7 +308,7 @@ class AURADocument: """ document = {"data": resource_object} include_str = request.args.get("include", None) - if include_str is not None: + if include_str: # [document["data"]] is because arg needs to be list document["included"] = self.get_included( [document["data"]], include_str @@ -390,7 +390,7 @@ class TrackDocument(AURADocument): track_id: The beets id of the track (integer). """ track = current_app.config["lib"].get_item(track_id) - if track is None: + if not track: return self.error( "404 Not Found", "No track with the requested id.", @@ -459,7 +459,7 @@ class AlbumDocument(AURADocument): } } # Add images relationship if album has associated images - if album.artpath is not None: + if album.artpath: path = displayable_path(album.artpath) filename = path.split("/")[-1] image_id = "album-{}-{}".format(album.id, filename) @@ -488,7 +488,7 @@ class AlbumDocument(AURADocument): album_id: The beets id of the album (integer). """ album = current_app.config["lib"].get_album(album_id) - if album is None: + if not album: return self.error( "404 Not Found", "No album with the requested id.", @@ -546,7 +546,7 @@ class ArtistDocument(AURADocument): # Get tracks where artist field exactly matches artist_id query = MatchQuery("artist", artist_id) tracks = current_app.config["lib"].items(query) - if len(tracks) == 0: + if not tracks: return None # Get artist information from the first track @@ -587,7 +587,7 @@ class ArtistDocument(AURADocument): artist_id: A string which is the artist's name. """ artist_resource = self.resource_object(artist_id) - if artist_resource is None: + if not artist_resource: return self.error( "404 Not Found", "No artist with the requested id.", @@ -623,7 +623,7 @@ class ImageDocument(AURADocument): # Get the path to the directory parent's images are in if parent_type == "album": album = current_app.config["lib"].get_album(int(parent_id)) - if album is None or album.artpath is None: + if not album or not album.artpath: return None # Cut the filename off of artpath # This is in preparation for supporting images in the same @@ -652,7 +652,7 @@ class ImageDocument(AURADocument): # Could be called as a static method, so can't use # self.get_image_path() image_path = ImageDocument.get_image_path(image_id) - if image_path is None: + if not image_path: return None attributes = { @@ -692,7 +692,7 @@ class ImageDocument(AURADocument): "--". """ image_resource = self.resource_object(image_id) - if image_resource is None: + if not image_resource: return self.error( "404 Not Found", "No image with the requested id.", @@ -742,7 +742,7 @@ def audio_file(track_id): track_id: The id of the track provided in the URL (integer). """ track = current_app.config["lib"].get_item(track_id) - if track is None: + if not track: return AURADocument.error( "404 Not Found", "No track with the requested id.", @@ -762,7 +762,7 @@ def audio_file(track_id): ) file_mimetype = guess_type(path)[0] - if file_mimetype is None: + if not file_mimetype: return AURADocument.error( "500 Internal Server Error", "Requested audio file has an unknown mimetype.", @@ -777,7 +777,7 @@ def audio_file(track_id): # Adding support for the bitrate parameter would require some effort so I # left it out. This means the client could be sent an error even if the # audio doesn't need transcoding. - if request.accept_mimetypes.best_match([file_mimetype]) is None: + if not request.accept_mimetypes.best_match([file_mimetype]): return AURADocument.error( "406 Not Acceptable", "Unsupported MIME type or bitrate parameter in Accept header.", @@ -868,7 +868,7 @@ def image_file(image_id): the form "--". """ img_path = ImageDocument.get_image_path(image_id) - if img_path is None: + if not img_path: return AURADocument.error( "404 Not Found", "No image with the requested id.", From fbc76887ad11d67c7a0262f8626c094eca62ff4b Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sat, 6 Mar 2021 13:22:28 +0000 Subject: [PATCH 041/130] AURA: Fix styling when formatting error strings --- beetsplug/aura.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 5a949dcf6..8980885fd 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -394,7 +394,7 @@ class TrackDocument(AURADocument): return self.error( "404 Not Found", "No track with the requested id.", - ("There is no track with an id of {} in the library.").format( + "There is no track with an id of {} in the library.".format( track_id ), ) @@ -492,7 +492,7 @@ class AlbumDocument(AURADocument): return self.error( "404 Not Found", "No album with the requested id.", - ("There is no album with an id of {} in the library.").format( + "There is no album with an id of {} in the library.".format( album_id ), ) @@ -591,7 +591,7 @@ class ArtistDocument(AURADocument): return self.error( "404 Not Found", "No artist with the requested id.", - ("There is no artist with an id of {} in the library.").format( + "There is no artist with an id of {} in the library.".format( artist_id ), ) @@ -696,7 +696,7 @@ class ImageDocument(AURADocument): return self.error( "404 Not Found", "No image with the requested id.", - ("There is no image with an id of {} in the library.").format( + "There is no image with an id of {} in the library.".format( image_id ), ) @@ -746,7 +746,7 @@ def audio_file(track_id): return AURADocument.error( "404 Not Found", "No track with the requested id.", - ("There is no track with an id of {} in the library.").format( + "There is no track with an id of {} in the library.".format( track_id ), ) @@ -768,7 +768,7 @@ def audio_file(track_id): "Requested audio file has an unknown mimetype.", ( "The audio file for track {} has an unknown mimetype. " - "It's file extension is {}." + "Its file extension is {}." ).format(track_id, path.split(".")[-1]), ) @@ -872,7 +872,7 @@ def image_file(image_id): return AURADocument.error( "404 Not Found", "No image with the requested id.", - ("There is no image with an id of {} in the library").format( + "There is no image with an id of {} in the library".format( image_id ), ) From 2fe2f4f31eb2ca70128e4739bb849847c1bebea4 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sat, 6 Mar 2021 13:30:31 +0000 Subject: [PATCH 042/130] AURA: Ensure CORS allowed origins are strings --- beetsplug/aura.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 8980885fd..201e099e3 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -909,7 +909,7 @@ def create_app(): app.config["lib"] = _open_library(config) # Enable CORS if required - cors = config["aura"]["cors"].get(list) + cors = config["aura"]["cors"].as_str_seq(list) if cors: from flask_cors import CORS From 49c747d144da581b0377663e3f90bdf90f617844 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Sat, 6 Mar 2021 15:14:41 +0000 Subject: [PATCH 043/130] Small documentation change to make GET /album/.../art clearer Signed-off-by: Graham R. Cobb --- docs/plugins/web.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 4b069a944..16dd43174 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -261,6 +261,8 @@ For albums, the following endpoints are provided: * ``GET /album/5`` +* ``GET /album/5/art`` + * ``DELETE /album/5`` * ``GET /album/5,7`` From 9986b9892c9f144ee5ce2031b5c5b0e3c08f49db Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Sat, 6 Mar 2021 15:25:48 +0000 Subject: [PATCH 044/130] Fix bug where album artpath not returned when INCLUDE_PATHS is set Track item paths and album artpaths should be removed from results unless INCLUDE_PATHS is set. This works for items but for albums the artpath is always removed. This patch makes the artpath removal conditional on INCLUDE_PATHS not being set and includes a regression test. Note: the default value for INCLUDE_PATHS is False so no changes will be seen by users unless they already have INCLUDE_PATHS set. Signed-off-by: Graham R. Cobb --- beetsplug/web/__init__.py | 5 ++++- test/test_web.py | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index a982809c4..e80c8c29e 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -59,7 +59,10 @@ def _rep(obj, expand=False): return out elif isinstance(obj, beets.library.Album): - del out['artpath'] + if app.config.get('INCLUDE_PATHS', False): + out['artpath'] = util.displayable_path(out['artpath']) + else: + del out['artpath'] if expand: out['items'] = [_rep(item) for item in obj.items()] return out diff --git a/test/test_web.py b/test/test_web.py index e9ca028d9..88be31365 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -31,7 +31,7 @@ class WebPluginTest(_common.LibTestCase): self.lib.add(Item(title=u'and a third')) # The following adds will create albums #1 and #2 self.lib.add(Album(album=u'album')) - self.lib.add(Album(album=u'other album')) + self.lib.add(Album(album=u'other album', artpath='/art_path_2')) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -46,6 +46,14 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) self.assertEqual(res_json['path'], u'/path_1') + def test_config_include_artpaths_true(self): + web.app.config['INCLUDE_PATHS'] = True + response = self.client.get('/album/2') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['artpath'], u'/art_path_2') + def test_config_include_paths_false(self): web.app.config['INCLUDE_PATHS'] = False response = self.client.get('/item/1') @@ -54,6 +62,14 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) self.assertNotIn('path', res_json) + def test_config_include_artpaths_false(self): + web.app.config['INCLUDE_PATHS'] = False + response = self.client.get('/album/2') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertNotIn('artpath', res_json) + def test_get_all_items(self): response = self.client.get('/item/') res_json = json.loads(response.data.decode('utf-8')) From c5556ff859a2a5076716cf87930e1f967141d271 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Sat, 6 Mar 2021 16:03:37 +0000 Subject: [PATCH 045/130] Changelog entry for fixing web include_paths artpath bug #3866 Signed-off-by: Graham R. Cobb --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f31ecfe3..b9020621b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -186,6 +186,9 @@ New features: Fixes: +* :bug:`/plugins/web`: Fixed a small bug which caused album artpath to be + redacted even when ``include_paths`` option is set. + :bug:`3866` * :bug:`/plugins/discogs`: Fixed a bug with ``index_tracks`` options that sometimes caused the index to be discarded. Also remove the extra semicolon that was added when there is no index track. From d329c70338d2a5c880495ce478aac352e3b163b2 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 6 Mar 2021 16:57:22 -0500 Subject: [PATCH 046/130] Use Python 3 for docs --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 69308235d..64acf1402 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ commands = lint: python -m flake8 {posargs} {[_lint]files} [testenv:docs] -basepython = python2.7 +basepython = python3.9 deps = sphinx commands = sphinx-build -W -q -b html docs {envtmpdir}/html {posargs} From 748e5318b409955b6846da30ab0165d315765365 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 6 Mar 2021 16:58:23 -0500 Subject: [PATCH 047/130] Change "main" Python version to 3.9 --- .github/workflows/ci.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ecb7e03dd..10441df44 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,7 +6,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9-dev] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] env: PY_COLORS: 1 @@ -25,17 +25,17 @@ jobs: python -m pip install tox sphinx - name: Test with tox - if: matrix.python-version != '3.8' + if: matrix.python-version != '3.9' run: | tox -e py-test - name: Test with tox and get coverage - if: matrix.python-version == '3.8' + if: matrix.python-version == '3.9' run: | tox -vv -e py-cov - + - name: Upload code coverage - if: matrix.python-version == '3.8' + if: matrix.python-version == '3.9' run: | pip install codecov || true codecov || true @@ -71,10 +71,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install base dependencies run: | From defe96b132484331403ea3cfd27dd5b006397bba Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 6 Mar 2021 16:59:49 -0500 Subject: [PATCH 048/130] *Actually* use 3.9 for docs --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10441df44..2a8b3a784 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,10 +49,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 2.7 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 2.7 + python-version: 3.9 - name: Install base dependencies run: | From 477eed3b25c8983b77e59f259891dd536289a378 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sun, 7 Mar 2021 10:25:20 +0000 Subject: [PATCH 049/130] AURA: Use py3_path rather than displayable_path displayable_path may remove 'bad' characters, yielding a wrong path. Also use track.path rather than track.destination() as that is where the file is actually located rather than where it should be located according to the beets path system. --- beetsplug/aura.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index 201e099e3..f3f8980ae 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -25,7 +25,7 @@ from os.path import isfile, getsize from beets.plugins import BeetsPlugin from beets.ui import Subcommand, _open_library from beets import config -from beets.util import displayable_path +from beets.util import py3_path from beets.library import Item, Album from beets.dbcore.query import ( MatchQuery, @@ -460,7 +460,7 @@ class AlbumDocument(AURADocument): } # Add images relationship if album has associated images if album.artpath: - path = displayable_path(album.artpath) + path = py3_path(album.artpath) filename = path.split("/")[-1] image_id = "album-{}-{}".format(album.id, filename) relationships["images"] = { @@ -628,7 +628,7 @@ class ImageDocument(AURADocument): # Cut the filename off of artpath # This is in preparation for supporting images in the same # directory that are not tracked by beets. - artpath = displayable_path(album.artpath) + artpath = py3_path(album.artpath) dir_path = "/".join(artpath.split("/")[:-1]) else: # Images for other resource types are not supported @@ -751,7 +751,7 @@ def audio_file(track_id): ), ) - path = displayable_path(track.destination()) + path = py3_path(track.path) if not isfile(path): return AURADocument.error( "404 Not Found", From a54ee43d57a2b3b6913bb8827b3fda8b63449d61 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sun, 7 Mar 2021 14:23:17 +0000 Subject: [PATCH 050/130] AURA: Allow '-' character in filter attribute --- beetsplug/aura.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index f3f8980ae..f0cda3688 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -142,7 +142,7 @@ class AURADocument: """Translate filters from request arguments to a beets Query.""" # The format of each filter key in the request parameter is: # filter[]. This regex extracts . - pattern = re.compile(r"filter\[(?P\w+)\]") + pattern = re.compile(r"filter\[(?P[a-zA-Z0-9_-]+)\]") queries = [] for key, value in request.args.items(): match = pattern.match(key) From a24a094562960b952edff464406e26e50d436421 Mon Sep 17 00:00:00 2001 From: Callum Brown Date: Sun, 7 Mar 2021 18:24:57 +0000 Subject: [PATCH 051/130] AURA: Small updates to docs and set server version --- beetsplug/aura.py | 2 +- docs/changelog.rst | 1 + docs/plugins/aura.rst | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/beetsplug/aura.py b/beetsplug/aura.py index f0cda3688..df76ca856 100644 --- a/beetsplug/aura.py +++ b/beetsplug/aura.py @@ -54,7 +54,7 @@ from flask import ( SERVER_INFO = { "aura-version": "0", "server": "beets-aura", - "server-version": "0", + "server-version": "0.1", "auth-required": False, "features": ["albums", "artists", "images"], } diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f31ecfe3..090c5b6f7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -183,6 +183,7 @@ New features: * Added ``trackdisambig`` which stores the recording disambiguation from MusicBrainz for each track. :bug:`1904` +* The :doc:`/plugins/aura` has arrived! Fixes: diff --git a/docs/plugins/aura.rst b/docs/plugins/aura.rst index 7af8b19e6..83c186249 100644 --- a/docs/plugins/aura.rst +++ b/docs/plugins/aura.rst @@ -48,8 +48,7 @@ then see :ref:`aura-external-server`. AURA is designed to separate the client and server functionality. This plugin provides the server but not the client, so unless you like looking at JSON you -will need a separate client. Unfortunately there are no AURA clients yet -(discounting the broken and outdated reference implementation). +will need a separate client. Currently the only client is `AURA Web Client`_. By default the API is served under http://127.0.0.1:8337/aura/. For example information about the track with an id of 3 can be obtained at @@ -59,6 +58,7 @@ http://127.0.0.1:8337/aura/tracks/3. http://127.0.0.1:8337/aura/tracks/3/ returns a ``404 Not Found`` error. .. _development mode: https://flask.palletsprojects.com/en/1.1.x/server +.. _AURA Web Client: https://sr.ht/~callum/aura-web-client/ .. _configuration: @@ -168,6 +168,10 @@ It is possible that some attributes required by AURA could be absent from the server's response if beets does not have a saved value for them. However, this has not happened so far. +Beets fields (including flexible fields) that do not have an AURA equivalent +are not provided in any resource's attributes section, however these fields may +be used for filtering. + The ``mimetype`` and ``framecount`` attributes for track resources are not supported. The first is due to beets storing the file type (e.g. ``MP3``), so it is hard to filter by MIME type. The second is because there is no From 3d1b421b97d6357648e8f8147bfe7372141a0e0b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 7 Mar 2021 19:50:31 -0500 Subject: [PATCH 052/130] Cursory changelog summary --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index df286a10a..251328ea6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog 1.5.0 (in development) ---------------------- +This long overdue release of beets includes far too many exciting and useful +features than could ever be satisfactorily enumerated. +As a technical detail, it also introduces two new external libraries: +`MediaFile`_ and `Confuse`_ used to be part of beets but are now reusable +dependencies---packagers, please take note. +Finally, this is the last version of beets where we intend to support Python +2.x; future releases will soon require Python 3.5. + New features: * :doc:`/plugins/mpdstats`: Add strip_path option to help build the right local path From 39e597240728e83af43e1aecdc686debd232f8e5 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 7 Mar 2021 19:54:42 -0500 Subject: [PATCH 053/130] A little changelog editing --- docs/changelog.rst | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 251328ea6..7e6509e05 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,18 +14,22 @@ Finally, this is the last version of beets where we intend to support Python New features: -* :doc:`/plugins/mpdstats`: Add strip_path option to help build the right local path - from MPD information -* Submitting acoustID information on tracks which already have a fingerprint - :bug:`3834` -* conversion uses par_map to parallelize conversion jobs in python3 -* Add ``title_case`` config option to lastgenre to make TitleCasing optional. -* When config is printed with no available configuration a new message is printed. +* :doc:`/plugins/mpdstats`: Add a new `strip_path` option to help build the + right local path from MPD information. +* :doc:`/plugins/convert`: Conversion can now parallelize conversion jobs on + Python 3. +* :doc:`/plugins/lastgenre`: Add a new `title_case` config option to make + title-case formatting optional. +* There's a new message when running ``beet config`` when there's no available + configuration file. :bug:`3779` -* When importing a duplicate album it ask if it should "Keep all" instead of "Keep both". +* When importing a duplicate album, the prompt now says "keep all" instead of + "keep both" to reflect that there may be more than two albums involved. :bug:`3569` -* :doc:`/plugins/chroma`: Update file metadata after generating fingerprints through the `submit` command. -* :doc:`/plugins/lastgenre`: Added more heavy metal genres: https://en.wikipedia.org/wiki/Heavy_metal_genres to genres.txt and genres-tree.yaml +* :doc:`/plugins/chroma`: The plugin now updates file metadata after + generating fingerprints through the `submit` command. +* :doc:`/plugins/lastgenre`: Added more heavy metal genres to the built-in + genre filter lists. * :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. * :doc:`/plugins/subsonicupdate`: Automatically choose between token and password-based authentication based on server version @@ -323,6 +327,9 @@ Fixes: information. Thanks to :user:`dosoe`. * :doc:`/plugins/discogs`: Replace deprecated discogs-client library with community supported python3-discogs-client library. :bug:`3608` +* :doc:`/plugins/chroma`: Fixed submitting AcoustID information for tracks + that already have a fingerprint. + :bug:`3834` For plugin developers: From 8cf46bce6d6448331365d04e73c583ad1f063e75 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 7 Mar 2021 21:14:53 -0500 Subject: [PATCH 054/130] A little more changelog stuff --- docs/changelog.rst | 55 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7e6509e05..48179bd72 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,7 +12,20 @@ dependencies---packagers, please take note. Finally, this is the last version of beets where we intend to support Python 2.x; future releases will soon require Python 3.5. -New features: +Major new features: + +* A new :ref:`reflink` config option instructs the importer to create fast, + copy-on-write file clones on filesystems that support them. Thanks to + :user:`rubdos`. +* A new :doc:`/plugins/unimported` lets you find untracked files in your + library directory. +* We now fetch information about `works`_ from MusicBrainz. + MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` + (the MBID), and ``work_disambig`` (the disambiguation string). + Thanks to :user:`dosoe`. + :bug:`2580` :bug:`3272` + +Other new things: * :doc:`/plugins/mpdstats`: Add a new `strip_path` option to help build the right local path from MPD information. @@ -30,15 +43,13 @@ New features: generating fingerprints through the `submit` command. * :doc:`/plugins/lastgenre`: Added more heavy metal genres to the built-in genre filter lists. -* :doc:`/plugins/subsonicplaylist`: import playlist from a subsonic server. -* :doc:`/plugins/subsonicupdate`: Automatically choose between token and - password-based authentication based on server version -* A new :ref:`reflink` config option instructs the importer to create fast, - copy-on-write file clones on filesystems that support them. Thanks to - :user:`rubdos`. +* A new :doc:`/plugins/subsonicplaylist` can import playlists from a Subsonic + server. +* :doc:`/plugins/subsonicupdate`: The plugin now automatically chooses between + token and password-based authentication based on server version * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. -* A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets +* A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets. * :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is resized. @@ -48,34 +59,28 @@ New features: allow downloading of higher resolution iTunes artwork (at the expense of file size). :bug:`3391` -* :doc:`plugins/discogs` now adds two extra fields: `discogs_labelid` and - `discogs_artistid` +* :doc:`plugins/discogs`: The plugin applies two new fields: `discogs_labelid` + and `discogs_artistid`. :bug:`3413` -* :doc:`/plugins/export`: Added new ``-f`` (``--format``) flag; - which allows for the ability to export in json, jsonlines, csv and xml. +* :doc:`/plugins/export`: Added a new ``-f`` (``--format``) flag, + which can export your data as JSON, JSON lines, CSV, or XML. Thanks to :user:`austinmm`. :bug:`3402` -* :doc:`/plugins/unimported`: lets you find untracked files in your library directory. -* We now fetch information about `works`_ from MusicBrainz. - MusicBrainz matches provide the fields ``work`` (the title), ``mb_workid`` - (the MBID), and ``work_disambig`` (the disambiguation string). - Thanks to :user:`dosoe`. - :bug:`2580` :bug:`3272` -* :doc:`/plugins/convert`: Added new ``-l`` (``--link``) flag and ``link`` +* :doc:`/plugins/convert`: Added a new ``-l`` (``--link``) flag and ``link`` option as well as the ``-H`` (``--hardlink``) flag and ``hardlink`` - option which symlinks or hardlinks files that do not need to - be converted instead of copying them. + option, which symlink or hardlink files that do not need to + be converted (instead of copying them). :bug:`2324` * :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 of the MPD protocol. This is enough to get it talking to more complicated clients like ncmpcpp, but there are still some incompatibilities, largely due - to MPD commands we don't support yet. Let us know if you find an MPD client - that doesn't get along with BPD! + to MPD commands we don't support yet. (Let us know if you find an MPD client + that doesn't get along with BPD!) :bug:`3214` :bug:`800` * :doc:`/plugins/replaygain`: The plugin now supports a ``per_disc`` option - which enables calculation of album ReplayGain on disc level instead of album + that enables calculation of album ReplayGain on disc level instead of album level. - Thanks to :user:`samuelnilsson` + Thanks to :user:`samuelnilsson`. :bug:`293` * :doc:`/plugins/replaygain`: The new ``ffmpeg`` ReplayGain backend supports ``R128_`` tags. From 1a2724c32768048254172416279bd27215515dfb Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 7 Mar 2021 21:30:32 -0500 Subject: [PATCH 055/130] More changelog editing --- docs/changelog.rst | 75 +++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48179bd72..5d9aadfb1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,20 @@ Major new features: (the MBID), and ``work_disambig`` (the disambiguation string). Thanks to :user:`dosoe`. :bug:`2580` :bug:`3272` +* A new :doc:`/plugins/parentwork` gets information about the original work, + which is useful for classical music. + Thanks to :user:`dosoe`. + :bug:`2580` :bug:`3279` +* :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 + of the MPD protocol. This is enough to get it talking to more complicated + clients like ncmpcpp, but there are still some incompatibilities, largely due + to MPD commands we don't support yet. (Let us know if you find an MPD client + that doesn't get along with BPD!) + :bug:`3214` :bug:`800` +* A new :doc:`/plugins/deezer` can autotag tracks and albums using the + `Deezer`_ database. + Thanks to :user:`rhlahuja`. + :bug:`3355` Other new things: @@ -46,14 +60,14 @@ Other new things: * A new :doc:`/plugins/subsonicplaylist` can import playlists from a Subsonic server. * :doc:`/plugins/subsonicupdate`: The plugin now automatically chooses between - token and password-based authentication based on server version -* A new :ref:`extra_tags` configuration option allows more tagged metadata - to be included in MusicBrainz queries. + token- and password-based authentication based on server version +* A new :ref:`extra_tags` configuration option lets you use more metadata in + MusicBrainz queries to further narrow the search. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets. * :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` option that controls the quality of the image output when the image is resized. -* :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ +* :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_. Thanks to :user:`BrainDamage`. * :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to allow downloading of higher resolution iTunes artwork (at the expense of @@ -71,12 +85,6 @@ Other new things: option, which symlink or hardlink files that do not need to be converted (instead of copying them). :bug:`2324` -* :doc:`/plugins/bpd`: BPD now supports most of the features of version 0.16 - of the MPD protocol. This is enough to get it talking to more complicated - clients like ncmpcpp, but there are still some incompatibilities, largely due - to MPD commands we don't support yet. (Let us know if you find an MPD client - that doesn't get along with BPD!) - :bug:`3214` :bug:`800` * :doc:`/plugins/replaygain`: The plugin now supports a ``per_disc`` option that enables calculation of album ReplayGain on disc level instead of album level. @@ -85,20 +93,15 @@ Other new things: * :doc:`/plugins/replaygain`: The new ``ffmpeg`` ReplayGain backend supports ``R128_`` tags. :bug:`3056` -* :doc:`plugins/replaygain`: ``r128_targetlevel`` is a new configuration option - for the ReplayGain plugin: It defines the reference volume for files using - ``R128_`` tags. ``targetlevel`` only configures the reference volume for - ``REPLAYGAIN_`` files. +* :doc:`plugins/replaygain`: A new ``r128_targetlevel`` configuration option + defines the reference volume for files using ``R128_`` tags. ``targetlevel`` + only configures the reference volume for ``REPLAYGAIN_`` files. :bug:`3065` -* A new :doc:`/plugins/parentwork` gets information about the original work, - which is useful for classical music. - Thanks to :user:`dosoe`. - :bug:`2580` :bug:`3279` -* :doc:`/plugins/discogs`: The field now collects the "style" field. +* :doc:`/plugins/discogs`: The plugin now collects the "style" field. Thanks to :user:`thedevilisinthedetails`. :bug:`2579` :bug:`3251` * :doc:`/plugins/absubmit`: By default, the plugin now avoids re-analyzing - files that already have AB data. + files that already have AcousticBrainz data. There are new ``force`` and ``pretend`` options to help control this new behavior. Thanks to :user:`SusannaMaria`. @@ -116,24 +119,21 @@ Other new things: Windows. Thanks to :user:`MartyLake`. :bug:`3331` :bug:`3334` -* The 'data_source' field is now also applied as an album-level flexible - attribute during imports, allowing for more refined album level searches. +* The `data_source` field, which indicates which metadata source was used + during an autotagging import, is now also applied as an album-level flexible + attribute. :bug:`3350` :bug:`1693` -* :doc:`/plugins/deezer`: Added Deezer plugin as an import metadata provider: - you can now match tracks and albums using the `Deezer`_ database. - Thanks to :user:`rhlahuja`. - :bug:`3355` -* :doc:`/plugins/beatport`: The plugin now gets the musical key, BPM and the +* :doc:`/plugins/beatport`: The plugin now gets the musical key, BPM, and genre for each track. :bug:`2080` -* :doc:`/plugins/beatport`: Fix default assignment of the musical key. +* :doc:`/plugins/beatport`: Fix the default assignment of the musical key. :bug:`3377` * :doc:`/plugins/bpsync`: Add `bpsync` plugin to sync metadata changes from the Beatport database. * :doc:`/plugins/beatport`: Fix assignment of `genre` and rename `musical_key` to `initial_key`. :bug:`3387` -* :doc:`/plugins/hook` now treats non-zero exit codes as errors. +* :doc:`/plugins/hook`: The plugin now treats non-zero exit codes as errors. :bug:`3409` * :doc:`/plugins/subsonicupdate`: A new ``url`` configuration replaces the older (and now deprecated) separate ``host``, ``port``, and ``contextpath`` @@ -148,27 +148,24 @@ Other new things: :bug:`3459` * :doc:`/plugins/fetchart`: Album art can now be fetched from `last.fm`_. :bug:`3530` -* The classes ``AlbumInfo`` and ``TrackInfo`` now have flexible attributes, - allowing to solve :bug:`1547`. - Thanks to :user:`dosoe`. * :doc:`/plugins/web`: The query API now interprets backslashes as path separators to support path queries. Thanks to :user:`nmeum`. :bug:`3567` * ``beet import`` now handles tar archives with bzip2 or gzip compression. :bug:`3606` -* :doc:`/plugins/plexupdate`: Add option to use secure connection to Plex - server, and to ignore certificate validation errors if necessary. +* :doc:`/plugins/plexupdate`: Added an option to use a secure connection to + Plex server, and to ignore certificate validation errors if necessary. :bug:`2871` -* :doc:`/plugins/lyrics`: Improved searching Genius backend when artist - contained special characters. +* :doc:`/plugins/lyrics`: Improved searching on the Genius backend when the + artist contains special characters. :bug:`3634` * :doc:`/plugins/parentwork`: Also get the composition date of the parent work, instead of just the child work. Thanks to :user:`aereaux`. :bug:`3650` * :doc:`/plugins/lyrics`: Fix a bug in the heuristic for detecting valid - lyrics in the Google source of the lyrics plugin + lyrics in the Google source. :bug:`2969` * :doc:`/plugins/thumbnails`: Fix a bug where pathlib expected a string instead of bytes for a path. @@ -375,6 +372,10 @@ For plugin developers: * ``Item.keys`` also has a ``with_album`` argument now, defaulting to ``True``. * A ``revision`` attribute has been added to ``Database``. It is increased on every transaction that mutates it. :bug:`2988` +* The classes ``AlbumInfo`` and ``TrackInfo`` now convey arbitrary attributes + instead of a fixed, built-in set of field names (which was important to + address :bug:`1547`). + Thanks to :user:`dosoe`. For packagers: From 553e38fc11adff4342aa9d9a6e91618788211d32 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 8 Mar 2021 16:40:04 +0000 Subject: [PATCH 056/130] Do not do backslash substitution on regex queries As discussed in bug #3867, backslash replacement in query strings is a bit of a hack but it is useful (see #3566 and #3567 for more discussion). However, it breaks many regular expressions so this patch stops the replacement if the query term contains '::', indicating it is a regex match. This commit fixes #3867. Signed-off-by: Graham R. Cobb --- beetsplug/web/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index e80c8c29e..c8f979fa6 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -244,7 +244,9 @@ class QueryConverter(PathConverter): def to_python(self, value): queries = value.split('/') - return [query.replace('\\', os.sep) for query in queries] + """Do not do path substitution on regex value tests""" + return [query if '::' in query else query.replace('\\', os.sep) + for query in queries] def to_url(self, value): return ','.join([v.replace(os.sep, '\\') for v in value]) From 38fb1b5f08af38ccb1e919f797e0ee28013d5a0e Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 8 Mar 2021 17:00:41 +0000 Subject: [PATCH 057/130] Add tests for several web queries, including the regex queries fixed in bugfix #3867. Signed-off-by: Graham R. Cobb --- test/test_web.py | 102 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index 88be31365..b673e4045 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -26,12 +26,12 @@ class WebPluginTest(_common.LibTestCase): # Add library elements. Note that self.lib.add overrides any "id=" # and assigns the next free id number. # The following adds will create items #1, #2 and #3 - self.lib.add(Item(title=u'title', path='/path_1', album_id=2)) - self.lib.add(Item(title=u'another title', path='/path_2')) - self.lib.add(Item(title=u'and a third')) + self.lib.add(Item(title=u'title', path='/path_1', album_id=2, artist='AAA Singers')) + self.lib.add(Item(title=u'another title', path='/somewhere/path_2', artist='AAA Singers')) + self.lib.add(Item(title=u'and a third', testattr='ABC', path='/somewhere-else/path_2', album_id=2)) # The following adds will create albums #1 and #2 - self.lib.add(Album(album=u'album')) - self.lib.add(Album(album=u'other album', artpath='/art_path_2')) + self.lib.add(Album(album=u'album', albumtest='xyz')) + self.lib.add(Album(album=u'other album', artpath='/somewhere-else/art_path_2')) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -46,13 +46,17 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) self.assertEqual(res_json['path'], u'/path_1') + web.app.config['INCLUDE_PATHS'] = False + def test_config_include_artpaths_true(self): web.app.config['INCLUDE_PATHS'] = True response = self.client.get('/album/2') res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(res_json['artpath'], u'/art_path_2') + self.assertEqual(res_json['artpath'], u'/somewhere-else/art_path_2') + + web.app.config['INCLUDE_PATHS'] = False def test_config_include_paths_false(self): web.app.config['INCLUDE_PATHS'] = False @@ -91,8 +95,8 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 200) self.assertEqual(len(res_json['items']), 2) - response_titles = [item['title'] for item in res_json['items']] - assertCountEqual(self, response_titles, [u'title', u'another title']) + response_titles = {item['title'] for item in res_json['items']} + self.assertEqual(response_titles, {u'title', u'another title'}) def test_get_single_item_not_found(self): response = self.client.get('/item/4') @@ -116,6 +120,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(response.status_code, 404) def test_get_item_empty_query(self): + """ testing item query: """ response = self.client.get('/item/query/') res_json = json.loads(response.data.decode('utf-8')) @@ -123,6 +128,7 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(len(res_json['items']), 3) def test_get_simple_item_query(self): + """ testing item query: another """ response = self.client.get('/item/query/another') res_json = json.loads(response.data.decode('utf-8')) @@ -131,6 +137,49 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(res_json['results'][0]['title'], u'another title') + def test_query_item_string(self): + """ testing item query: testattr:ABC """ + response = self.client.get('/item/query/testattr%3aABC') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['title'], + u'and a third') + + def test_query_item_regex(self): + """ testing item query: testattr::[A-C]+ """ + response = self.client.get('/item/query/testattr%3a%3a[A-C]%2b') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['title'], + u'and a third') + + def test_query_item_regex_backslash(self): + #""" testing item query: testattr::\w+ """ + response = self.client.get('/item/query/testattr%3a%3a%5cw%2b') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['title'], + u'and a third') + + def test_query_item_path(self): + #""" testing item query: path:\somewhere """ + """ Note: path queries are special: the query item must match the path + from the root all the way to a directory, so this matches 1 item """ + """ Note: filesystem separators must be specified as '\' """ + response = self.client.get('/item/query/path:\\somewhere') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['title'], + u'another title') + def test_get_all_albums(self): response = self.client.get('/album/') res_json = json.loads(response.data.decode('utf-8')) @@ -177,10 +226,43 @@ class WebPluginTest(_common.LibTestCase): res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(len(res_json['items']), 1) + self.assertEqual(len(res_json['items']), 2) self.assertEqual(res_json['items'][0]['album'], u'other album') - self.assertEqual(res_json['items'][0]['id'], 1) + self.assertEqual(res_json['items'][1]['album'], + u'other album') + response_track_titles = {item['title'] for item in res_json['items']} + self.assertEqual(response_track_titles, {u'title', u'and a third'}) + + def test_query_album_string(self): + """ testing query: albumtest:xy """ + response = self.client.get('/album/query/albumtest%3axy') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['album'], + u'album') + + def test_query_album_artpath_regex(self): + """ testing query: artpath::art_ """ + response = self.client.get('/album/query/artpath%3a%3aart_') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['album'], + u'other album') + + def test_query_album_regex_backslash(self): + #""" testing query: albumtest::\w+ """ + response = self.client.get('/album/query/albumtest%3a%3a%5cw%2b') + res_json = json.loads(response.data.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + self.assertEqual(res_json['results'][0]['album'], + u'album') def test_get_stats(self): response = self.client.get('/stats') From 75f331e64bf47a688bb8842c4a025f353bede611 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 8 Mar 2021 17:08:36 +0000 Subject: [PATCH 058/130] Update changelog with web regex query backslash fix for #3867. Signed-off-by: Graham R. Cobb --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b9020621b..1e21ac37f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -186,6 +186,8 @@ New features: Fixes: +* :bug:`/plugins/web`: Allow use of backslash in regex web queries. + :bug:`3867` * :bug:`/plugins/web`: Fixed a small bug which caused album artpath to be redacted even when ``include_paths`` option is set. :bug:`3866` From 4d5c5c02ae38f991a797babf8f9b73427d891fba Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 8 Mar 2021 17:26:04 +0000 Subject: [PATCH 059/130] Lint fixes Signed-off-by: Graham R. Cobb --- test/test_web.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index b673e4045..9d085650b 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -26,12 +26,22 @@ class WebPluginTest(_common.LibTestCase): # Add library elements. Note that self.lib.add overrides any "id=" # and assigns the next free id number. # The following adds will create items #1, #2 and #3 - self.lib.add(Item(title=u'title', path='/path_1', album_id=2, artist='AAA Singers')) - self.lib.add(Item(title=u'another title', path='/somewhere/path_2', artist='AAA Singers')) - self.lib.add(Item(title=u'and a third', testattr='ABC', path='/somewhere-else/path_2', album_id=2)) + self.lib.add(Item(title=u'title', + path='/path_1', + album_id=2, + artist='AAA Singers')) + self.lib.add(Item(title=u'another title', + path='/somewhere/path_2', + artist='AAA Singers')) + self.lib.add(Item(title=u'and a third', + testattr='ABC', + path='/somewhere-else/path_2', + album_id=2)) # The following adds will create albums #1 and #2 - self.lib.add(Album(album=u'album', albumtest='xyz')) - self.lib.add(Album(album=u'other album', artpath='/somewhere-else/art_path_2')) + self.lib.add(Album(album=u'album', + albumtest='xyz')) + self.lib.add(Album(album=u'other album', + artpath='/somewhere-else/art_path_2')) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -158,7 +168,7 @@ class WebPluginTest(_common.LibTestCase): u'and a third') def test_query_item_regex_backslash(self): - #""" testing item query: testattr::\w+ """ + # """ testing item query: testattr::\w+ """ response = self.client.get('/item/query/testattr%3a%3a%5cw%2b') res_json = json.loads(response.data.decode('utf-8')) @@ -168,7 +178,7 @@ class WebPluginTest(_common.LibTestCase): u'and a third') def test_query_item_path(self): - #""" testing item query: path:\somewhere """ + # """ testing item query: path:\somewhere """ """ Note: path queries are special: the query item must match the path from the root all the way to a directory, so this matches 1 item """ """ Note: filesystem separators must be specified as '\' """ @@ -255,7 +265,7 @@ class WebPluginTest(_common.LibTestCase): u'other album') def test_query_album_regex_backslash(self): - #""" testing query: albumtest::\w+ """ + # """ testing query: albumtest::\w+ """ response = self.client.get('/album/query/albumtest%3a%3a%5cw%2b') res_json = json.loads(response.data.decode('utf-8')) From f19faae1dc437e779a3d81b0d23c446eddd78757 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 9 Mar 2021 11:19:54 +0000 Subject: [PATCH 060/130] Attempt to make path-related tests portable to Windows Signed-off-by: Graham R. Cobb --- test/test_web.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index 9d085650b..528cbf36d 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -27,21 +27,21 @@ class WebPluginTest(_common.LibTestCase): # and assigns the next free id number. # The following adds will create items #1, #2 and #3 self.lib.add(Item(title=u'title', - path='/path_1', + path=os.sep + os.path.join('path_1'), album_id=2, artist='AAA Singers')) self.lib.add(Item(title=u'another title', - path='/somewhere/path_2', + path=os.sep + os.path.join('somewhere', 'a'), artist='AAA Singers')) self.lib.add(Item(title=u'and a third', testattr='ABC', - path='/somewhere-else/path_2', + path=os.sep + os.path.join('somewhere', 'abc'), album_id=2)) # The following adds will create albums #1 and #2 self.lib.add(Album(album=u'album', albumtest='xyz')) self.lib.add(Album(album=u'other album', - artpath='/somewhere-else/art_path_2')) + artpath=os.sep + os.path.join('somewhere-else', 'art_path_2'))) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -54,7 +54,7 @@ class WebPluginTest(_common.LibTestCase): res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(res_json['path'], u'/path_1') + self.assertEqual(res_json['path'], os.path.join(os.sep, u'path_1')) web.app.config['INCLUDE_PATHS'] = False @@ -64,7 +64,7 @@ class WebPluginTest(_common.LibTestCase): res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(res_json['artpath'], u'/somewhere-else/art_path_2') + self.assertEqual(res_json['artpath'], os.path.join(os.sep, u'somewhere-else', u'art_path_2')) web.app.config['INCLUDE_PATHS'] = False @@ -180,9 +180,9 @@ class WebPluginTest(_common.LibTestCase): def test_query_item_path(self): # """ testing item query: path:\somewhere """ """ Note: path queries are special: the query item must match the path - from the root all the way to a directory, so this matches 1 item """ - """ Note: filesystem separators must be specified as '\' """ - response = self.client.get('/item/query/path:\\somewhere') + from the root all the way to a directory, so this matches just 1 item """ + """ Note: filesystem separators in the query must be specified as '\' """ + response = self.client.get('/item/query/path:\\somewhere\\a') res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) From ab2e858bce14fb9176dc3afbe0329714aba64210 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 9 Mar 2021 11:40:14 +0000 Subject: [PATCH 061/130] Aaargh! Forgot to run lint again Signed-off-by: Graham R. Cobb --- test/test_web.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index 528cbf36d..a4c10a193 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -41,7 +41,8 @@ class WebPluginTest(_common.LibTestCase): self.lib.add(Album(album=u'album', albumtest='xyz')) self.lib.add(Album(album=u'other album', - artpath=os.sep + os.path.join('somewhere-else', 'art_path_2'))) + artpath=os.sep + + os.path.join('somewhere2', 'art_path_2'))) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -64,7 +65,8 @@ class WebPluginTest(_common.LibTestCase): res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) - self.assertEqual(res_json['artpath'], os.path.join(os.sep, u'somewhere-else', u'art_path_2')) + self.assertEqual(res_json['artpath'], + os.path.join(os.sep, u'somewhere2', u'art_path_2')) web.app.config['INCLUDE_PATHS'] = False @@ -180,8 +182,8 @@ class WebPluginTest(_common.LibTestCase): def test_query_item_path(self): # """ testing item query: path:\somewhere """ """ Note: path queries are special: the query item must match the path - from the root all the way to a directory, so this matches just 1 item """ - """ Note: filesystem separators in the query must be specified as '\' """ + from the root all the way to a directory, so this matches 1 item """ + """ Note: filesystem separators in the query must be '\' """ response = self.client.get('/item/query/path:\\somewhere\\a') res_json = json.loads(response.data.decode('utf-8')) From 45cf9d00c73eb755e752d0497e6b945e2a4ee932 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 9 Mar 2021 18:58:05 +0000 Subject: [PATCH 062/130] Improve handling of paths for Windows tests Signed-off-by: Graham R. Cobb --- test/test_web.py | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/test/test_web.py b/test/test_web.py index a4c10a193..606f1e243 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -13,11 +13,22 @@ from test import _common from beets.library import Item, Album from beetsplug import web +import platform + +from beets import logging + class WebPluginTest(_common.LibTestCase): def setUp(self): + super(WebPluginTest, self).setUp() + self.log = logging.getLogger('beets.web') + + if platform.system() == 'Windows': + self.path_prefix = u'C:' + else: + self.path_prefix = u'' # Add fixtures for track in self.lib.items(): @@ -26,23 +37,30 @@ class WebPluginTest(_common.LibTestCase): # Add library elements. Note that self.lib.add overrides any "id=" # and assigns the next free id number. # The following adds will create items #1, #2 and #3 + path1 = self.path_prefix + os.sep + \ + os.path.join(b'path_1').decode('utf-8') self.lib.add(Item(title=u'title', - path=os.sep + os.path.join('path_1'), + path=path1, album_id=2, artist='AAA Singers')) + path2 = self.path_prefix + os.sep + \ + os.path.join(b'somewhere', b'a').decode('utf-8') self.lib.add(Item(title=u'another title', - path=os.sep + os.path.join('somewhere', 'a'), + path=path2, artist='AAA Singers')) + path3 = self.path_prefix + os.sep + \ + os.path.join(b'somewhere', b'abc').decode('utf-8') self.lib.add(Item(title=u'and a third', testattr='ABC', - path=os.sep + os.path.join('somewhere', 'abc'), + path=path3, album_id=2)) # The following adds will create albums #1 and #2 self.lib.add(Album(album=u'album', albumtest='xyz')) + path4 = self.path_prefix + os.sep + \ + os.path.join(b'somewhere2', b'art_path_2').decode('utf-8') self.lib.add(Album(album=u'other album', - artpath=os.sep - + os.path.join('somewhere2', 'art_path_2'))) + artpath=path4)) web.app.config['TESTING'] = True web.app.config['lib'] = self.lib @@ -53,9 +71,11 @@ class WebPluginTest(_common.LibTestCase): web.app.config['INCLUDE_PATHS'] = True response = self.client.get('/item/1') res_json = json.loads(response.data.decode('utf-8')) + expected_path = self.path_prefix + os.sep \ + + os.path.join(b'path_1').decode('utf-8') self.assertEqual(response.status_code, 200) - self.assertEqual(res_json['path'], os.path.join(os.sep, u'path_1')) + self.assertEqual(res_json['path'], expected_path) web.app.config['INCLUDE_PATHS'] = False @@ -63,10 +83,11 @@ class WebPluginTest(_common.LibTestCase): web.app.config['INCLUDE_PATHS'] = True response = self.client.get('/album/2') res_json = json.loads(response.data.decode('utf-8')) + expected_path = self.path_prefix + os.sep \ + + os.path.join(b'somewhere2', b'art_path_2').decode('utf-8') self.assertEqual(response.status_code, 200) - self.assertEqual(res_json['artpath'], - os.path.join(os.sep, u'somewhere2', u'art_path_2')) + self.assertEqual(res_json['artpath'], expected_path) web.app.config['INCLUDE_PATHS'] = False @@ -180,11 +201,14 @@ class WebPluginTest(_common.LibTestCase): u'and a third') def test_query_item_path(self): - # """ testing item query: path:\somewhere """ + # """ testing item query: path:\somewhere\a """ """ Note: path queries are special: the query item must match the path from the root all the way to a directory, so this matches 1 item """ """ Note: filesystem separators in the query must be '\' """ - response = self.client.get('/item/query/path:\\somewhere\\a') + + response = self.client.get('/item/query/path:' + + self.path_prefix + + '\\somewhere\\a') res_json = json.loads(response.data.decode('utf-8')) self.assertEqual(response.status_code, 200) From 374707bb7a6379ca283393ce961697711fd525d7 Mon Sep 17 00:00:00 2001 From: David Dembeck <71412966+djdembeck@users.noreply.github.com> Date: Wed, 10 Mar 2021 12:59:48 -0600 Subject: [PATCH 063/130] remove LyricsWiki from default sources in docs https://github.com/beetbox/beets/pull/3766 missed removing this --- docs/plugins/lyrics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index b71764042..b49909678 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -58,7 +58,7 @@ configuration file. The available options are: sources known to be scrapeable. - **sources**: List of sources to search for lyrics. An asterisk ``*`` expands to all available sources. - Default: ``google lyricwiki musixmatch genius``, i.e., all the + Default: ``google musixmatch genius``, i.e., all the available sources. The ``google`` source will be automatically deactivated if no ``google_API_key`` is setup. Both it and the ``genius`` source will only be enabled if BeautifulSoup is From 79858975a9bc007680a8241e666c8b6e8f4a35a0 Mon Sep 17 00:00:00 2001 From: Aksh Gupta Date: Wed, 10 Mar 2021 13:07:43 +0530 Subject: [PATCH 064/130] chore: refactor code quality issues --- beets/dbcore/db.py | 4 ++-- beets/importer.py | 2 +- beets/ui/__init__.py | 2 +- beetsplug/bpd/__init__.py | 3 ++- beetsplug/export.py | 2 +- beetsplug/fetchart.py | 2 +- beetsplug/fish.py | 4 ++-- beetsplug/importadded.py | 2 +- beetsplug/info.py | 2 +- beetsplug/mbsync.py | 2 +- beetsplug/missing.py | 2 +- beetsplug/replaygain.py | 6 +++--- beetsplug/subsonicplaylist.py | 4 ++-- 13 files changed, 19 insertions(+), 18 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 409ecc9af..b87b41644 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -729,10 +729,10 @@ class Results(object): def _get_indexed_flex_attrs(self): """ Index flexible attributes by the entity id they belong to """ - flex_values = dict() + flex_values = {} for row in self.flex_rows: if row['entity_id'] not in flex_values: - flex_values[row['entity_id']] = dict() + flex_values[row['entity_id']] = {} flex_values[row['entity_id']][row['key']] = row['value'] diff --git a/beets/importer.py b/beets/importer.py index c5701ff30..b7bfdb156 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -187,7 +187,7 @@ class ImportSession(object): self.logger = self._setup_logging(loghandler) self.paths = paths self.query = query - self._is_resuming = dict() + self._is_resuming = {} self._merged_items = set() self._merged_dirs = set() diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 362e8752a..3d7cffb51 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -791,7 +791,7 @@ def _store_dict(option, opt_str, value, parser): if option_values is None: # This is the first supplied ``key=value`` pair of option. # Initialize empty dictionary and get a reference to it. - setattr(parser.values, dest, dict()) + setattr(parser.values, dest, {}) option_values = getattr(parser.values, dest) try: diff --git a/beetsplug/bpd/__init__.py b/beetsplug/bpd/__init__.py index dad864b8b..628353942 100644 --- a/beetsplug/bpd/__init__.py +++ b/beetsplug/bpd/__init__.py @@ -21,6 +21,7 @@ use of the wide range of MPD clients. from __future__ import division, absolute_import, print_function import re +import sys from string import Template import traceback import random @@ -334,7 +335,7 @@ class BaseServer(object): def cmd_kill(self, conn): """Exits the server process.""" - exit(0) + sys.exit(0) def cmd_close(self, conn): """Closes the connection.""" diff --git a/beetsplug/export.py b/beetsplug/export.py index 957180db2..3ce9ab887 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -33,7 +33,7 @@ from beetsplug.info import make_key_filter, library_data, tag_data class ExportEncoder(json.JSONEncoder): """Deals with dates because JSON doesn't have a standard""" def default(self, o): - if isinstance(o, datetime) or isinstance(o, date): + if isinstance(o, (datetime, date)): return o.isoformat() return json.JSONEncoder.default(self, o) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 1bf8ad428..14323507a 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -504,7 +504,7 @@ class FanartTV(RemoteArtSource): matches = [] # can there be more than one releasegroupid per response? - for mbid, art in data.get(u'albums', dict()).items(): + for mbid, art in data.get(u'albums', {}).items(): # there might be more art referenced, e.g. cdart, and an albumcover # might not be present, even if the request was successful if album.mb_releasegroupid == mbid and u'albumcover' in art: diff --git a/beetsplug/fish.py b/beetsplug/fish.py index 12f474e88..b4c28c283 100644 --- a/beetsplug/fish.py +++ b/beetsplug/fish.py @@ -110,7 +110,7 @@ class FishPlugin(BeetsPlugin): # Collect commands, their aliases, and their help text cmd_names_help = [] for cmd in beetcmds: - names = [alias for alias in cmd.aliases] + names = list(cmd.aliases) names.append(cmd.name) for name in names: cmd_names_help.append((name, cmd.help)) @@ -238,7 +238,7 @@ def get_all_commands(beetcmds): # Formatting for Fish to complete command options word = "" for cmd in beetcmds: - names = [alias for alias in cmd.aliases] + names = list(cmd.aliases) names.append(cmd.name) for name in names: name = _escape(name) diff --git a/beetsplug/importadded.py b/beetsplug/importadded.py index 29aeeab0f..37e881e96 100644 --- a/beetsplug/importadded.py +++ b/beetsplug/importadded.py @@ -27,7 +27,7 @@ class ImportAddedPlugin(BeetsPlugin): # album.path for old albums that were replaced by a reimported album self.replaced_album_paths = None # item path in the library to the mtime of the source file - self.item_mtime = dict() + self.item_mtime = {} register = self.register_listener register('import_task_created', self.check_config) diff --git a/beetsplug/info.py b/beetsplug/info.py index b8a0c9375..05d215e7b 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -235,7 +235,7 @@ def make_key_filter(include): matchers.append(re.compile(key + '$')) def filter_(data): - filtered = dict() + filtered = {} for key, value in data.items(): if any([m.match(key) for m in matchers]): filtered[key] = value diff --git a/beetsplug/mbsync.py b/beetsplug/mbsync.py index a2b3bc4aa..ee2c4b5bd 100644 --- a/beetsplug/mbsync.py +++ b/beetsplug/mbsync.py @@ -123,7 +123,7 @@ class MBSyncPlugin(BeetsPlugin): # Map release track and recording MBIDs to their information. # Recordings can appear multiple times on a release, so each MBID # maps to a list of TrackInfo objects. - releasetrack_index = dict() + releasetrack_index = {} track_index = defaultdict(list) for track_info in album_info.tracks: releasetrack_index[track_info.release_track_id] = track_info diff --git a/beetsplug/missing.py b/beetsplug/missing.py index 8f0790f2b..247ccc999 100644 --- a/beetsplug/missing.py +++ b/beetsplug/missing.py @@ -216,7 +216,7 @@ class MissingPlugin(BeetsPlugin): """Query MusicBrainz to determine items missing from `album`. """ item_mbids = [x.mb_trackid for x in album.items()] - if len([i for i in album.items()]) < album.albumtotal: + if len(list(album.items())) < album.albumtotal: # fetch missing items # TODO: Implement caching that without breaking other stuff album_info = hooks.album_for_mbid(album.mb_albumid) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 5060c8efe..8b9447bdb 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1139,7 +1139,7 @@ class ReplayGainPlugin(BeetsPlugin): tag_vals = self.tag_specific_values(album.items()) store_track_gain, store_album_gain, target_level, peak = tag_vals - discs = dict() + discs = {} if self.per_disc: for item in album.items(): if discs.get(item.disc) is None: @@ -1172,7 +1172,7 @@ class ReplayGainPlugin(BeetsPlugin): self._apply( self.backend_instance.compute_album_gain, args=(), kwds={ - "items": [i for i in items], + "items": list(items), "target_level": target_level, "peak": peak }, @@ -1288,7 +1288,7 @@ class ReplayGainPlugin(BeetsPlugin): try: self._log.info('interrupted') self.terminate_pool() - exit(0) + sys.exit(0) except SystemExit: # Silence raised SystemExit ~ exit(0) pass diff --git a/beetsplug/subsonicplaylist.py b/beetsplug/subsonicplaylist.py index 732b51c2f..5d64708ad 100644 --- a/beetsplug/subsonicplaylist.py +++ b/beetsplug/subsonicplaylist.py @@ -148,7 +148,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): def send(self, endpoint, params=None): if params is None: - params = dict() + params = {} a, b = self.generate_token() params['u'] = self.config['username'] params['t'] = a @@ -163,7 +163,7 @@ class SubsonicPlaylistPlugin(BeetsPlugin): return resp def get_playlists(self, ids): - output = dict() + output = {} for playlist_id in ids: name, tracks = self.get_playlist(playlist_id) for track in tracks: From 4a9652a9e44e8e3aa56f85e6b275d8c87332c4a4 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Wed, 10 Mar 2021 18:56:30 +0000 Subject: [PATCH 065/130] Only allow DELETE or PATCH operations if "readonly" is set to true. Note: default is false which is a **NOT BACKWARDS COMPATIBLE** change. Signed-off-by: Graham R. Cobb --- beetsplug/web/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index c8f979fa6..e4d83f8d2 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -116,12 +116,18 @@ def resource(name, patchable=False): entities = [entity for entity in entities if entity] if get_method() == "DELETE": + if app.config.get('READONLY', True): + return flask.abort(405) + for entity in entities: entity.remove(delete=is_delete()) return flask.make_response(jsonify({'deleted': True}), 200) elif get_method() == "PATCH" and patchable: + if app.config.get('READONLY', True): + return flask.abort(405) + for entity in entities: entity.update(flask.request.get_json()) entity.try_sync(True, False) # write, don't move @@ -162,12 +168,18 @@ def resource_query(name, patchable=False): entities = query_func(queries) if get_method() == "DELETE": + if app.config.get('READONLY', True): + return flask.abort(405) + for entity in entities: entity.remove(delete=is_delete()) return flask.make_response(jsonify({'deleted': True}), 200) elif get_method() == "PATCH" and patchable: + if app.config.get('READONLY', True): + return flask.abort(405) + for entity in entities: entity.update(flask.request.get_json()) entity.try_sync(True, False) # write, don't move From 51d22b765ef24841fa751893c2fbb732b60f0614 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Wed, 10 Mar 2021 23:31:34 +0000 Subject: [PATCH 066/130] Add tests for delete operations Signed-off-by: Graham R. Cobb --- beetsplug/web/__init__.py | 2 + test/test_web.py | 305 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 307 insertions(+) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index e4d83f8d2..88f7e72cd 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -116,6 +116,7 @@ def resource(name, patchable=False): entities = [entity for entity in entities if entity] if get_method() == "DELETE": + if app.config.get('READONLY', True): return flask.abort(405) @@ -168,6 +169,7 @@ def resource_query(name, patchable=False): entities = query_func(queries) if get_method() == "DELETE": + if app.config.get('READONLY', True): return flask.abort(405) diff --git a/test/test_web.py b/test/test_web.py index 606f1e243..ed6bed8ed 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -8,6 +8,7 @@ import json import unittest import os.path from six import assertCountEqual +import shutil from test import _common from beets.library import Item, Album @@ -65,6 +66,7 @@ class WebPluginTest(_common.LibTestCase): web.app.config['TESTING'] = True web.app.config['lib'] = self.lib web.app.config['INCLUDE_PATHS'] = False + # Default value of READONLY is True self.client = web.app.test_client() def test_config_include_paths_true(self): @@ -308,6 +310,309 @@ class WebPluginTest(_common.LibTestCase): self.assertEqual(res_json['items'], 3) self.assertEqual(res_json['albums'], 2) + def test_delete_item_id(self): + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create a temporary item + item_id = self.lib.add(Item(title=u'test_delete_item_id', + test_delete_item_id=1)) + + # Check we can find the temporary item we just created + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + + # Delete item by id + response = self.client.delete('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + + # Check the item has gone + response = self.client.get('/item/' + str(item_id)) + self.assertEqual(response.status_code, 404) + # Note: if this fails, the item may still be around + # and may cause other tests to fail + + def test_delete_item_without_file(self): + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create an item with a file + ipath = os.path.join(self.temp_dir, b'testfile1.mp3') + shutil.copy(os.path.join(_common.RSRC, b'full.mp3'), ipath) + self.assertTrue(os.path.exists(ipath)) + item_id = self.lib.add(Item.from_path(ipath)) + + # Check we can find the temporary item we just created + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + + # Delete item by id, without deleting file + response = self.client.delete('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + + # Check the item has gone + response = self.client.get('/item/' + str(item_id)) + self.assertEqual(response.status_code, 404) + + # Check the file has not gone + self.assertTrue(os.path.exists(ipath)) + os.remove(ipath) + + def test_delete_item_with_file(self): + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create an item with a file + ipath = os.path.join(self.temp_dir, b'testfile2.mp3') + shutil.copy(os.path.join(_common.RSRC, b'full.mp3'), ipath) + self.assertTrue(os.path.exists(ipath)) + item_id = self.lib.add(Item.from_path(ipath)) + + # Check we can find the temporary item we just created + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + + # Delete item by id, with file + response = self.client.delete('/item/' + str(item_id) + '?delete') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + + # Check the item has gone + response = self.client.get('/item/' + str(item_id)) + self.assertEqual(response.status_code, 404) + + # Check the file has gone + self.assertFalse(os.path.exists(ipath)) + + def test_delete_item_query(self): + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create a temporary item + self.lib.add(Item(title=u'test_delete_item_query', + test_delete_item_query=1)) + + # Check we can find the temporary item we just created + response = self.client.get('/item/query/test_delete_item_query') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + + # Delete item by query + response = self.client.delete('/item/query/test_delete_item_query') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + + # Check the item has gone + response = self.client.get('/item/query/test_delete_item_query') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 0) + + def test_delete_item_all_fails(self): + """ DELETE is not supported for list all """ + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Delete all items + response = self.client.delete('/item/') + self.assertEqual(response.status_code, 405) + + # Note: if this fails, all items have gone and rest of + # tests wil fail! + + def test_delete_item_id_readonly(self): + + # Default value of READONLY is True + del web.app.config['READONLY'] + + # Create a temporary item + item_id = self.lib.add(Item(title=u'test_delete_item_id_ro', + test_delete_item_id_ro=1)) + + # Check we can find the temporary item we just created + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + + # Try to delete item by id + response = self.client.delete('/item/' + str(item_id)) + self.assertEqual(response.status_code, 405) + + # Check the item has not gone + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + + # Remove it + self.lib.get_item(item_id).remove() + + def test_delete_item_query_readonly(self): + + # Default value of READONLY is True + del web.app.config['READONLY'] + + # Create a temporary item + item_id = self.lib.add(Item(title=u'test_delete_item_q_ro', + test_delete_item_q_ro=1)) + + # Check we can find the temporary item we just created + response = self.client.get('/item/query/test_delete_item_q_ro') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + + # Try to delete item by query + response = self.client.delete('/item/query/test_delete_item_q_ro') + self.assertEqual(response.status_code, 405) + + # Check the item has not gone + response = self.client.get('/item/query/test_delete_item_q_ro') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + + # Remove it + self.lib.get_item(item_id).remove() + + def test_delete_album_id(self): + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create a temporary album + album_id = self.lib.add(Album(album=u'test_delete_album_id', + test_delete_album_id=1)) + + # Check we can find the temporary album we just created + response = self.client.get('/album/' + str(album_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], album_id) + + # Delete album by id + response = self.client.delete('/album/' + str(album_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + + # Check the album has gone + response = self.client.get('/album/' + str(album_id)) + self.assertEqual(response.status_code, 404) + # Note: if this fails, the album may still be around + # and may cause other tests to fail + + def test_delete_album_query(self): + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create a temporary album + self.lib.add(Album(album=u'test_delete_album_query', + test_delete_album_query=1)) + + # Check we can find the temporary album we just created + response = self.client.get('/album/query/test_delete_album_query') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + + # Delete album + response = self.client.delete('/album/query/test_delete_album_query') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + + # Check the album has gone + response = self.client.get('/album/query/test_delete_album_query') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 0) + + def test_delete_album_all_fails(self): + """ DELETE is not supported for list all """ + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Delete all albums + response = self.client.delete('/album/') + self.assertEqual(response.status_code, 405) + + # Note: if this fails, all albums have gone and rest of + # tests wil fail! + + def test_delete_album_id_readonly(self): + + # Default value of READONLY is True + del web.app.config['READONLY'] + + # Create a temporary album + album_id = self.lib.add(Album(album=u'test_delete_album_id_ro', + test_delete_album_id_ro=1)) + + # Check we can find the temporary album we just created + response = self.client.get('/album/' + str(album_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], album_id) + + # Try to delete album by id + response = self.client.delete('/album/' + str(album_id)) + self.assertEqual(response.status_code, 405) + + # Check the item has not gone + response = self.client.get('/album/' + str(album_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], album_id) + + # Remove it + self.lib.get_album(album_id).remove() + + def test_delete_album_query_readonly(self): + + # Default value of READONLY is True + del web.app.config['READONLY'] + + # Create a temporary album + album_id = self.lib.add(Album(album=u'test_delete_album_query_ro', + test_delete_album_query_ro=1)) + + # Check we can find the temporary album we just created + response = self.client.get('/album/query/test_delete_album_query_ro') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + + # Try to delete album + response = self.client.delete( + '/album/query/test_delete_album_query_ro' + ) + self.assertEqual(response.status_code, 405) + + # Check the album has not gone + response = self.client.get('/album/query/test_delete_album_query_ro') + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(res_json['results']), 1) + + # Remove it + self.lib.get_album(album_id).remove() + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 23c8d61104d1a01e52b38704c5d48ac64c6ed844 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Thu, 11 Mar 2021 22:55:42 +0000 Subject: [PATCH 067/130] Add tests for patch operations Signed-off-by: Graham R. Cobb --- test/test_web.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/test/test_web.py b/test/test_web.py index ed6bed8ed..01d5a0794 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -613,6 +613,78 @@ class WebPluginTest(_common.LibTestCase): # Remove it self.lib.get_album(album_id).remove() + def test_patch_item_id(self): + # Note: PATCH is currently only implemented for track items, not albums + + # Default value of READONLY is True + web.app.config['READONLY'] = False + + # Create a temporary item + item_id = self.lib.add(Item(title=u'test_patch_item_id', + test_patch_f1=1, + test_patch_f2="Old")) + + # Check we can find the temporary item we just created + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + self.assertEqual( + [res_json['test_patch_f1'], res_json['test_patch_f2']], + ['1', 'Old']) + + # Patch item by id + # patch_json = json.JSONEncoder().encode({"test_patch_f2": "New"}]}) + response = self.client.patch('/item/' + str(item_id), + json={"test_patch_f2": "New"}) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + self.assertEqual( + [res_json['test_patch_f1'], res_json['test_patch_f2']], + ['1', 'New']) + + # Check the update has really worked + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + self.assertEqual( + [res_json['test_patch_f1'], res_json['test_patch_f2']], + ['1', 'New']) + + # Remove the item + self.lib.get_item(item_id).remove() + + def test_patch_item_id_readonly(self): + # Note: PATCH is currently only implemented for track items, not albums + + # Default value of READONLY is True + del web.app.config['READONLY'] + + # Create a temporary item + item_id = self.lib.add(Item(title=u'test_patch_item_id_ro', + test_patch_f1=2, + test_patch_f2="Old")) + + # Check we can find the temporary item we just created + response = self.client.get('/item/' + str(item_id)) + res_json = json.loads(response.data.decode('utf-8')) + self.assertEqual(response.status_code, 200) + self.assertEqual(res_json['id'], item_id) + self.assertEqual( + [res_json['test_patch_f1'], res_json['test_patch_f2']], + ['2', 'Old']) + + # Patch item by id + # patch_json = json.JSONEncoder().encode({"test_patch_f2": "New"}) + response = self.client.patch('/item/' + str(item_id), + json={"test_patch_f2": "New"}) + self.assertEqual(response.status_code, 405) + + # Remove the item + self.lib.get_item(item_id).remove() + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 680ccdcd9653e1cdf486cd1058ec99d93455721e Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Thu, 11 Mar 2021 23:09:51 +0000 Subject: [PATCH 068/130] Documentation and changelog for web ``readonly`` option fixing #3870. Signed-off-by: Graham R. Cobb --- docs/changelog.rst | 3 +++ docs/plugins/web.rst | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f39c41584..7338282f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -335,6 +335,9 @@ Fixes: * :doc:`/plugins/chroma`: Fixed submitting AcoustID information for tracks that already have a fingerprint. :bug:`3834` +* :doc:`/plugins/web`: DELETE and PATCH methods are disallowed by default. + Set ``readonly: no`` web config option to enable them. + :bug:`3870` For plugin developers: diff --git a/docs/plugins/web.rst b/docs/plugins/web.rst index 16dd43174..3a7e6d122 100644 --- a/docs/plugins/web.rst +++ b/docs/plugins/web.rst @@ -66,6 +66,8 @@ configuration file. The available options are: Default: false. - **include_paths**: If true, includes paths in item objects. Default: false. +- **readonly**: If true, DELETE and PATCH operations are not allowed. Only GET is permitted. + Default: true. Implementation -------------- @@ -189,6 +191,8 @@ code. Removes the item with id *6* from the beets library. If the *?delete* query string is included, the matching file will be deleted from disk. +Only allowed if ``readonly`` configuration option is set to ``no``. + ``PATCH /item/6`` ++++++++++++++++++ @@ -203,6 +207,8 @@ Returns the updated JSON representation. :: ... } +Only allowed if ``readonly`` configuration option is set to ``no``. + ``GET /item/6,12,13`` +++++++++++++++++++++ @@ -279,6 +285,7 @@ or ``/album/5,7``. In addition we can request the cover art of an album with ``GET /album/5/art``. You can also add the '?expand' flag to get the individual items of an album. +``DELETE`` is only allowed if ``readonly`` configuration option is set to ``no``. ``GET /stats`` ++++++++++++++ From 4ffe9a2c45e370a58e894f348613f89fd2e2833e Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Fri, 12 Mar 2021 17:58:35 +0000 Subject: [PATCH 069/130] Fixed bug where `readonly` value was not being read from config file. Also simplified the setup of the `readonly` value in the tests which fixes a test ordering issue found using --random-order. Signed-off-by: Graham R. Cobb --- beetsplug/web/__init__.py | 2 ++ test/test_web.py | 26 ++++++-------------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 88f7e72cd..c74cd0748 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -442,6 +442,7 @@ class WebPlugin(BeetsPlugin): 'cors_supports_credentials': False, 'reverse_proxy': False, 'include_paths': False, + 'readonly': True, }) def commands(self): @@ -461,6 +462,7 @@ class WebPlugin(BeetsPlugin): app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False app.config['INCLUDE_PATHS'] = self.config['include_paths'] + app.config['READONLY'] = self.config['readonly'] # Enable CORS if required. if self.config['cors']: diff --git a/test/test_web.py b/test/test_web.py index 01d5a0794..570a6447c 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -66,7 +66,7 @@ class WebPluginTest(_common.LibTestCase): web.app.config['TESTING'] = True web.app.config['lib'] = self.lib web.app.config['INCLUDE_PATHS'] = False - # Default value of READONLY is True + web.app.config['READONLY'] = True self.client = web.app.test_client() def test_config_include_paths_true(self): @@ -312,7 +312,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_id(self): - # Default value of READONLY is True web.app.config['READONLY'] = False # Create a temporary item @@ -338,7 +337,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_without_file(self): - # Default value of READONLY is True web.app.config['READONLY'] = False # Create an item with a file @@ -368,7 +366,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_with_file(self): - # Default value of READONLY is True web.app.config['READONLY'] = False # Create an item with a file @@ -397,7 +394,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_query(self): - # Default value of READONLY is True web.app.config['READONLY'] = False # Create a temporary item @@ -424,7 +420,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_all_fails(self): """ DELETE is not supported for list all """ - # Default value of READONLY is True web.app.config['READONLY'] = False # Delete all items @@ -436,8 +431,7 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_id_readonly(self): - # Default value of READONLY is True - del web.app.config['READONLY'] + web.app.config['READONLY'] = True # Create a temporary item item_id = self.lib.add(Item(title=u'test_delete_item_id_ro', @@ -464,8 +458,7 @@ class WebPluginTest(_common.LibTestCase): def test_delete_item_query_readonly(self): - # Default value of READONLY is True - del web.app.config['READONLY'] + web.app.config['READONLY'] = True # Create a temporary item item_id = self.lib.add(Item(title=u'test_delete_item_q_ro', @@ -492,7 +485,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_album_id(self): - # Default value of READONLY is True web.app.config['READONLY'] = False # Create a temporary album @@ -518,7 +510,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_album_query(self): - # Default value of READONLY is True web.app.config['READONLY'] = False # Create a temporary album @@ -545,7 +536,6 @@ class WebPluginTest(_common.LibTestCase): def test_delete_album_all_fails(self): """ DELETE is not supported for list all """ - # Default value of READONLY is True web.app.config['READONLY'] = False # Delete all albums @@ -557,8 +547,7 @@ class WebPluginTest(_common.LibTestCase): def test_delete_album_id_readonly(self): - # Default value of READONLY is True - del web.app.config['READONLY'] + web.app.config['READONLY'] = True # Create a temporary album album_id = self.lib.add(Album(album=u'test_delete_album_id_ro', @@ -585,8 +574,7 @@ class WebPluginTest(_common.LibTestCase): def test_delete_album_query_readonly(self): - # Default value of READONLY is True - del web.app.config['READONLY'] + web.app.config['READONLY'] = True # Create a temporary album album_id = self.lib.add(Album(album=u'test_delete_album_query_ro', @@ -616,7 +604,6 @@ class WebPluginTest(_common.LibTestCase): def test_patch_item_id(self): # Note: PATCH is currently only implemented for track items, not albums - # Default value of READONLY is True web.app.config['READONLY'] = False # Create a temporary item @@ -659,8 +646,7 @@ class WebPluginTest(_common.LibTestCase): def test_patch_item_id_readonly(self): # Note: PATCH is currently only implemented for track items, not albums - # Default value of READONLY is True - del web.app.config['READONLY'] + web.app.config['READONLY'] = True # Create a temporary item item_id = self.lib.add(Item(title=u'test_patch_item_id_ro', From 0c612f408b7b328c467eb8f29a9fd9c7325892ee Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 00:55:14 +0000 Subject: [PATCH 070/130] Create plugin for "bare-ASCII" matching query Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 51 +++++++++++++++++++++++++ test/test_bareasc.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 beetsplug/bareasc.py create mode 100644 test/test_bareasc.py diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py new file mode 100644 index 000000000..9a23ac6d5 --- /dev/null +++ b/beetsplug/bareasc.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2016, Philippe Mongeau. +# Copyright 2021, Graham R. Cobb +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and ascociated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# This module is adapted from Fuzzy in accordance to the licence of +# that module + +"""Provides a bare-ASCII matching query. +""" + +from __future__ import division, absolute_import, print_function + +from beets.plugins import BeetsPlugin +from beets.dbcore.query import StringFieldQuery +from unidecode import unidecode + + +class BareascQuery(StringFieldQuery): + @classmethod + def string_match(cls, pattern, val): + print('In BareascQuery' + ' ' + pattern + ' ' + val) + # smartcase + if pattern.islower(): + val = val.lower() + pattern = unidecode(pattern) + val = unidecode(val) + return pattern == val + + +class BareascPlugin(BeetsPlugin): + def __init__(self): + super(BareascPlugin, self).__init__() + self.config.add({ + 'prefix': '#', + }) + + def queries(self): + prefix = self.config['prefix'].as_str() + return {prefix: BareascQuery} diff --git a/test/test_bareasc.py b/test/test_bareasc.py new file mode 100644 index 000000000..20aceac3f --- /dev/null +++ b/test/test_bareasc.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +"""Tests for the 'bareasc' plugin""" + +from __future__ import division, absolute_import, print_function + +import unittest + +from test.helper import TestHelper + +from beets import logging + + +class BareascPluginTest(unittest.TestCase, TestHelper): + + def setUp(self): + self.setup_beets() + self.log = logging.getLogger('beets.web') + self.config['bareasc']['prefix'] = u'#' + self.load_plugins('bareasc') + + # Add library elements. Note that self.lib.add overrides any "id=" + # and assigns the next free id number. + self.add_item(title=u'with accents', + album_id=2, + artist=u'dvořák') + self.add_item(title=u'without accents', + artist=u'dvorak') + self.add_item(title=u'with umlaut', + album_id=2, + artist=u'Brüggen') + self.add_item(title=u'without umlaut', + artist=u'Bruggen') + + def test_search_normal_noaccent(self): + items = self.lib.items('dvorak') + + self.assertEqual(len(items), 1) + self.assertEqual([items[0].title], [u'without accents']) + + def test_search_normal_accent(self): + items = self.lib.items('dvořák') + + self.assertEqual(len(items), 1) + self.assertEqual([items[0].title], [u'with accents']) + + def test_search_bareasc_noaccent(self): + items = self.lib.items('#dvorak') + + self.assertEqual(len(items), 2) + self.assertEqual( + {items[0].title, items[1].title}, + {u'without accents', u'with accents'} + ) + + def test_search_bareasc_accent(self): + items = self.lib.items('#dvořák') + + self.assertEqual(len(items), 2) + self.assertEqual( + {items[0].title, items[1].title}, + {u'without accents', u'with accents'} + ) + + def test_search_bareasc_noumlaut(self): + items = self.lib.items('#Bruggen') + + self.assertEqual(len(items), 2) + self.assertEqual( + {items[0].title, items[1].title}, + {u'without umlaut', u'with umlaut'} + ) + + def test_search_bareasc_umlaut(self): + items = self.lib.items('#Brüggen') + + self.assertEqual(len(items), 2) + self.assertEqual( + {items[0].title, items[1].title}, + {u'without umlaut', u'with umlaut'} + ) + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 7dd1ee3fab34f6fafc7f9be83d50fbe905d38d65 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 01:09:50 +0000 Subject: [PATCH 071/130] Remove debugging print. Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index 9a23ac6d5..1419a5ba6 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -30,7 +30,6 @@ from unidecode import unidecode class BareascQuery(StringFieldQuery): @classmethod def string_match(cls, pattern, val): - print('In BareascQuery' + ' ' + pattern + ' ' + val) # smartcase if pattern.islower(): val = val.lower() From 80048e7153b709b6980871055395c5e4eb2678eb Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 09:11:17 +0000 Subject: [PATCH 072/130] Specify unicode, for python2 Signed-off-by: Graham R. Cobb --- test/test_bareasc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_bareasc.py b/test/test_bareasc.py index 20aceac3f..6b2b9bee9 100644 --- a/test/test_bareasc.py +++ b/test/test_bareasc.py @@ -33,19 +33,19 @@ class BareascPluginTest(unittest.TestCase, TestHelper): artist=u'Bruggen') def test_search_normal_noaccent(self): - items = self.lib.items('dvorak') + items = self.lib.items(u'dvorak') self.assertEqual(len(items), 1) self.assertEqual([items[0].title], [u'without accents']) def test_search_normal_accent(self): - items = self.lib.items('dvořák') + items = self.lib.items(u'dvořák') self.assertEqual(len(items), 1) self.assertEqual([items[0].title], [u'with accents']) def test_search_bareasc_noaccent(self): - items = self.lib.items('#dvorak') + items = self.lib.items(u'#dvorak') self.assertEqual(len(items), 2) self.assertEqual( @@ -54,7 +54,7 @@ class BareascPluginTest(unittest.TestCase, TestHelper): ) def test_search_bareasc_accent(self): - items = self.lib.items('#dvořák') + items = self.lib.items(u'#dvořák') self.assertEqual(len(items), 2) self.assertEqual( @@ -63,7 +63,7 @@ class BareascPluginTest(unittest.TestCase, TestHelper): ) def test_search_bareasc_noumlaut(self): - items = self.lib.items('#Bruggen') + items = self.lib.items(u'#Bruggen') self.assertEqual(len(items), 2) self.assertEqual( @@ -72,7 +72,7 @@ class BareascPluginTest(unittest.TestCase, TestHelper): ) def test_search_bareasc_umlaut(self): - items = self.lib.items('#Brüggen') + items = self.lib.items(u'#Brüggen') self.assertEqual(len(items), 2) self.assertEqual( From dcbe622b7604aadf66d0a1f881ef1ee7a7651096 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 10:25:19 +0000 Subject: [PATCH 073/130] Oops, use `in` for matching! (It was late last night!) Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 8 ++++---- test/test_bareasc.py | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index 1419a5ba6..f0c43fdfd 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # This file is part of beets. # Copyright 2016, Philippe Mongeau. -# Copyright 2021, Graham R. Cobb +# Copyright 2021, Graham R. Cobb. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and ascociated documentation files (the @@ -17,8 +17,7 @@ # This module is adapted from Fuzzy in accordance to the licence of # that module -"""Provides a bare-ASCII matching query. -""" +"""Provides a bare-ASCII matching query.""" from __future__ import division, absolute_import, print_function @@ -28,6 +27,7 @@ from unidecode import unidecode class BareascQuery(StringFieldQuery): + """Matches items using bare ASCII, without accents etc.""" @classmethod def string_match(cls, pattern, val): # smartcase @@ -35,7 +35,7 @@ class BareascQuery(StringFieldQuery): val = val.lower() pattern = unidecode(pattern) val = unidecode(val) - return pattern == val + return pattern in val class BareascPlugin(BeetsPlugin): diff --git a/test/test_bareasc.py b/test/test_bareasc.py index 6b2b9bee9..4b004ff62 100644 --- a/test/test_bareasc.py +++ b/test/test_bareasc.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2021, Graham R. Cobb. -"""Tests for the 'bareasc' plugin""" +"""Tests for the 'bareasc' plugin.""" from __future__ import division, absolute_import, print_function @@ -23,9 +25,9 @@ class BareascPluginTest(unittest.TestCase, TestHelper): # and assigns the next free id number. self.add_item(title=u'with accents', album_id=2, - artist=u'dvořák') + artist=u'Antonín dvořák') self.add_item(title=u'without accents', - artist=u'dvorak') + artist=u'Antonín dvorak') self.add_item(title=u'with umlaut', album_id=2, artist=u'Brüggen') From fab7a27e9ff54aecb9c8565ac7b718cb965978a9 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 12:43:58 +0000 Subject: [PATCH 074/130] Add a couple more tests and make lint happy. Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 10 +++++++- test/test_bareasc.py | 54 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index f0c43fdfd..bdae32284 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -27,9 +27,14 @@ from unidecode import unidecode class BareascQuery(StringFieldQuery): - """Matches items using bare ASCII, without accents etc.""" + """Compare items using bare ASCII, without accents etc.""" @classmethod def string_match(cls, pattern, val): + """Convert both pattern and string to plain ASCII before matching. + + If pattern is all lower case, also convert string to lower case so + match is also case insensitive + """ # smartcase if pattern.islower(): val = val.lower() @@ -39,12 +44,15 @@ class BareascQuery(StringFieldQuery): class BareascPlugin(BeetsPlugin): + """Plugin to provide bare-ASCII option for beets matching.""" def __init__(self): + """Default prefix for selecting bare-ASCII matching is #.""" super(BareascPlugin, self).__init__() self.config.add({ 'prefix': '#', }) def queries(self): + """Reguster bare-ASCII matching.""" prefix = self.config['prefix'].as_str() return {prefix: BareascQuery} diff --git a/test/test_bareasc.py b/test/test_bareasc.py index 4b004ff62..d49ea4448 100644 --- a/test/test_bareasc.py +++ b/test/test_bareasc.py @@ -14,8 +14,10 @@ from beets import logging class BareascPluginTest(unittest.TestCase, TestHelper): + """Test bare ASCII query matching.""" def setUp(self): + """Set up test environment for bare ASCII query matching.""" self.setup_beets() self.log = logging.getLogger('beets.web') self.config['bareasc']['prefix'] = u'#' @@ -31,22 +33,36 @@ class BareascPluginTest(unittest.TestCase, TestHelper): self.add_item(title=u'with umlaut', album_id=2, artist=u'Brüggen') - self.add_item(title=u'without umlaut', + self.add_item(title=u'without umlaut or e', artist=u'Bruggen') + self.add_item(title=u'without umlaut with e', + artist=u'Brueggen') def test_search_normal_noaccent(self): + """Normal search, no accents, not using bare-ASCII match. + + Finds just the unaccented entry. + """ items = self.lib.items(u'dvorak') self.assertEqual(len(items), 1) self.assertEqual([items[0].title], [u'without accents']) def test_search_normal_accent(self): + """Normal search, with accents, not using bare-ASCII match. + + Finds just the accented entry. + """ items = self.lib.items(u'dvořák') self.assertEqual(len(items), 1) self.assertEqual([items[0].title], [u'with accents']) def test_search_bareasc_noaccent(self): + """Bare-ASCII search, no accents. + + Finds both entries. + """ items = self.lib.items(u'#dvorak') self.assertEqual(len(items), 2) @@ -56,6 +72,10 @@ class BareascPluginTest(unittest.TestCase, TestHelper): ) def test_search_bareasc_accent(self): + """Bare-ASCII search, with accents. + + Finds both entries. + """ items = self.lib.items(u'#dvořák') self.assertEqual(len(items), 2) @@ -64,26 +84,54 @@ class BareascPluginTest(unittest.TestCase, TestHelper): {u'without accents', u'with accents'} ) + def test_search_bareasc_wrong_accent(self): + """Bare-ASCII search, with incorrect accent. + + Finds both entries. + """ + items = self.lib.items(u'#dvořäk') + + self.assertEqual(len(items), 2) + self.assertEqual( + {items[0].title, items[1].title}, + {u'without accents', u'with accents'} + ) + def test_search_bareasc_noumlaut(self): + """Bare-ASCII search, with no umlaut. + + Finds entry with 'u' not 'ue', although German speaker would + normally replace ü with ue. + + This is expected behaviour for this simple plugin. + """ items = self.lib.items(u'#Bruggen') self.assertEqual(len(items), 2) self.assertEqual( {items[0].title, items[1].title}, - {u'without umlaut', u'with umlaut'} + {u'without umlaut or e', u'with umlaut'} ) def test_search_bareasc_umlaut(self): + """Bare-ASCII search, with umlaut. + + Finds entry with 'u' not 'ue', although German speaker would + normally replace ü with ue. + + This is expected behaviour for this simple plugin. + """ items = self.lib.items(u'#Brüggen') self.assertEqual(len(items), 2) self.assertEqual( {items[0].title, items[1].title}, - {u'without umlaut', u'with umlaut'} + {u'without umlaut or e', u'with umlaut'} ) def suite(): + """loader.""" return unittest.TestLoader().loadTestsFromName(__name__) if __name__ == '__main__': From 341a0a0adf0b86769fa111aa32fdb860d8ba7769 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 15:34:58 +0000 Subject: [PATCH 075/130] Added documentation and changelog for bareasc. Signed-off-by: Graham R. Cobb --- docs/changelog.rst | 5 +++++ docs/plugins/bareasc.rst | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 docs/plugins/bareasc.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 7338282f5..07fa573f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -38,6 +38,11 @@ Major new features: `Deezer`_ database. Thanks to :user:`rhlahuja`. :bug:`3355` +* A new :doc:`/plugins/bareasc` provides a new query type: `bare ASCII` + which ignores accented characters, treating them as though they + were the base ASCII character. To perform `bare ASCII` searches, use + the ``#`` prefix with :ref:`list-cmd` or other commands. + :bug:`3882` Other new things: diff --git a/docs/plugins/bareasc.rst b/docs/plugins/bareasc.rst new file mode 100644 index 000000000..114b5d8f9 --- /dev/null +++ b/docs/plugins/bareasc.rst @@ -0,0 +1,37 @@ +Bare-ASCII Search Plugin +======================== + +The ``bareasc`` plugin provides a prefixed query that searches your library using +simple ASCII character matching, with accented characters folded to their base +ASCII character. This can be useful if you want to find a track with accented +characters in the title or artist, particularly if you are not confident +you have the accents correct. It is also not unknown for the accents +to not be correct in the database entry or wrong in the CD information. + +First, enable the plugin named ``bareasc`` (see :ref:`using-plugins`). +You'll then be able to use the ``#`` prefix to use bare-ASCII matching:: + + $ beet ls '#dvorak' + István Kertész - REQUIEM - Dvořàk: Requiem, op.89 - Confutatis maledictis + +Notes +----- + +If the query string is all in lower case, the comparison ignores case as well as +accents. + +The default ``bareasc`` prefix (``#``) is used as a comment character in some shells +so may need to be protected (for example in quotes) when typed into the command line. + +The bare ASCII transformation is quite simple. It may not work perfectly for all +languages and does not handle transformations which change the number of letters. +For example, German u-umlaut ``ü`` is transformed into ASCII ``u``, not into ``ue``. + +Configuration +------------- + +To configure the plugin, make a ``bareasc:`` section in your configuration +file. The only available option is: + +- **prefix**: The character used to designate bare-ASCII queries. + Default: ``#``, which may need to be escaped in some shells. From f4fccfa05c960d91472d01ceb58a470794707c80 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 15:53:14 +0000 Subject: [PATCH 076/130] Add documentation for bareasc to the plugins index. Signed-off-by: Graham R. Cobb --- docs/plugins/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index ae14b8166..14dd5137b 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -63,6 +63,7 @@ following to your configuration:: acousticbrainz aura badfiles + bareasc beatport bpd bpm @@ -218,6 +219,7 @@ Interoperability Miscellaneous ------------- +* :doc:`bareasc`: Search albums and tracks with bare ASCII string matching. * :doc:`bpd`: A music player for your beets library that emulates `MPD`_ and is compatible with `MPD clients`_. * :doc:`convert`: Transcode music and embed album art while exporting to From 06b6b72e0eff1171c9eea1095af027a38aa0a96c Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Mon, 15 Mar 2021 22:52:14 +0000 Subject: [PATCH 077/130] Add credit to Unidecode library into bareasc docs. Signed-off-by: Graham R. Cobb --- docs/plugins/bareasc.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/plugins/bareasc.rst b/docs/plugins/bareasc.rst index 114b5d8f9..63aca2317 100644 --- a/docs/plugins/bareasc.rst +++ b/docs/plugins/bareasc.rst @@ -35,3 +35,10 @@ file. The only available option is: - **prefix**: The character used to designate bare-ASCII queries. Default: ``#``, which may need to be escaped in some shells. + +Credits +------- + +The hard work in this plugin is done in Sean Burke's Unidecode library. +Thanks are due to Sean and to all the people who created the Python +version and the beets extensible query architecture. From d1ec7b4b70f91ed5fa932db35716057a589911fa Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 16 Mar 2021 11:57:52 +0000 Subject: [PATCH 078/130] Add ``bareasc`` command to display entries with the unidecode transformation applied. Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 27 ++++++++++++++++++++++++++- docs/plugins/bareasc.rst | 30 +++++++++++++++++++++++++++--- test/test_bareasc.py | 13 ++++++++++--- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index bdae32284..138da8f82 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -21,6 +21,8 @@ from __future__ import division, absolute_import, print_function +from beets import ui +from beets.ui import print_, decargs from beets.plugins import BeetsPlugin from beets.dbcore.query import StringFieldQuery from unidecode import unidecode @@ -53,6 +55,29 @@ class BareascPlugin(BeetsPlugin): }) def queries(self): - """Reguster bare-ASCII matching.""" + """Register bare-ASCII matching.""" prefix = self.config['prefix'].as_str() return {prefix: BareascQuery} + + def commands(self): + """Add bareasc command as unidecode version of 'list'.""" + cmd = ui.Subcommand('bareasc', + help='unidecode version of beet list command') + cmd.parser.usage += u"\n" \ + u'Example: %prog -f \'$album: $title\' artist:beatles' + cmd.parser.add_all_common_options() + cmd.func = self.unidecode_list + return [cmd] + + def unidecode_list(self, lib, opts, args): + """Emulate normal 'list' command but with unidecode output.""" + query = decargs(args) + album = opts.album + fmt = u'' + # Copied from commands.py - list_items + if album: + for album in lib.albums(query): + print_(unidecode(format(album, fmt))) + else: + for item in lib.items(query): + print_(unidecode(format(item, fmt))) diff --git a/docs/plugins/bareasc.rst b/docs/plugins/bareasc.rst index 63aca2317..765c9a81a 100644 --- a/docs/plugins/bareasc.rst +++ b/docs/plugins/bareasc.rst @@ -14,6 +14,23 @@ You'll then be able to use the ``#`` prefix to use bare-ASCII matching:: $ beet ls '#dvorak' István Kertész - REQUIEM - Dvořàk: Requiem, op.89 - Confutatis maledictis +Command +------- + +In addition to the query prefix, the plugin provides a utility ``bareasc`` command. +This command is **exactly** the same as the ``beet list`` command except that +the output is passed through the bare-ASCII transformation before being printed. +This allows you to easily check what the library data looks like in bare ASCII, +which can be useful if you are trying to work out why a query is not matching. + +Using the same example track as above:: + + $ beet bareasc 'Dvořàk' + Istvan Kertesz - REQUIEM - Dvorak: Requiem, op.89 - Confutatis maledictis + +Note: the ``bareasc`` command does NOT automatically use bare-ASCII queries. +If you want a bare-ASCII query you still need to specify the ``#`` prefix. + Notes ----- @@ -23,9 +40,16 @@ accents. The default ``bareasc`` prefix (``#``) is used as a comment character in some shells so may need to be protected (for example in quotes) when typed into the command line. -The bare ASCII transformation is quite simple. It may not work perfectly for all -languages and does not handle transformations which change the number of letters. -For example, German u-umlaut ``ü`` is transformed into ASCII ``u``, not into ``ue``. +The bare ASCII transformation is quite simple. It may not give the expected output +for all languages. For example, German u-umlaut ``ü`` is transformed into ASCII ``u``, +not into ``ue``. + +The bare ASCII transformation also changes Unicode punctuation like double quotes, +apostrophes and even some hyphens. It is often best to leave out punctuation +in the queries. Note that the punctuation changes are often not even visible +with normal terminal fonts. You can always use the ``bareasc`` command to print the +transformed entries and use a command like ``diff`` to compare with the output +from the ``list`` command. Configuration ------------- diff --git a/test/test_bareasc.py b/test/test_bareasc.py index d49ea4448..abb42a730 100644 --- a/test/test_bareasc.py +++ b/test/test_bareasc.py @@ -8,7 +8,7 @@ from __future__ import division, absolute_import, print_function import unittest -from test.helper import TestHelper +from test.helper import capture_stdout, TestHelper from beets import logging @@ -27,9 +27,9 @@ class BareascPluginTest(unittest.TestCase, TestHelper): # and assigns the next free id number. self.add_item(title=u'with accents', album_id=2, - artist=u'Antonín dvořák') + artist=u'Antonín Dvořák') self.add_item(title=u'without accents', - artist=u'Antonín dvorak') + artist=u'Antonín Dvorak') self.add_item(title=u'with umlaut', album_id=2, artist=u'Brüggen') @@ -129,6 +129,13 @@ class BareascPluginTest(unittest.TestCase, TestHelper): {u'without umlaut or e', u'with umlaut'} ) + def test_bareasc_list_output(self): + """Bare-ASCII version of list command - check output.""" + with capture_stdout() as output: + self.run_command('bareasc', 'with accents') + + self.assertIn('Antonin Dvorak', output.getvalue()) + def suite(): """loader.""" From 0078b02085047eb8e2d95364212f6a4c35d69a82 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 16 Mar 2021 14:50:51 +0000 Subject: [PATCH 079/130] Python2 support for bareasc command Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index 138da8f82..43d83005a 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -26,6 +26,7 @@ from beets.ui import print_, decargs from beets.plugins import BeetsPlugin from beets.dbcore.query import StringFieldQuery from unidecode import unidecode +import six class BareascQuery(StringFieldQuery): @@ -80,4 +81,4 @@ class BareascPlugin(BeetsPlugin): print_(unidecode(format(album, fmt))) else: for item in lib.items(query): - print_(unidecode(format(item, fmt))) + print_(six.ensure_str(unidecode(format(item, fmt)))) From 2df3765521c334de16b5a9203d0d409878bcc8be Mon Sep 17 00:00:00 2001 From: Graham Cobb Date: Tue, 16 Mar 2021 15:29:35 +0000 Subject: [PATCH 080/130] Tidy doc as suggested by @sampsyo Co-authored-by: Adrian Sampson --- docs/plugins/bareasc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/bareasc.rst b/docs/plugins/bareasc.rst index 765c9a81a..9be102e27 100644 --- a/docs/plugins/bareasc.rst +++ b/docs/plugins/bareasc.rst @@ -28,7 +28,7 @@ Using the same example track as above:: $ beet bareasc 'Dvořàk' Istvan Kertesz - REQUIEM - Dvorak: Requiem, op.89 - Confutatis maledictis -Note: the ``bareasc`` command does NOT automatically use bare-ASCII queries. +Note: the ``bareasc`` command does *not* automatically use bare-ASCII queries. If you want a bare-ASCII query you still need to specify the ``#`` prefix. Notes From 4b9c9d0a5fd551fdcd52737cd88e2aa09b89665a Mon Sep 17 00:00:00 2001 From: Graham Cobb Date: Tue, 16 Mar 2021 15:30:43 +0000 Subject: [PATCH 081/130] doc improvement as suggested by @sampsyo Co-authored-by: Adrian Sampson --- docs/plugins/bareasc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/bareasc.rst b/docs/plugins/bareasc.rst index 9be102e27..a22063805 100644 --- a/docs/plugins/bareasc.rst +++ b/docs/plugins/bareasc.rst @@ -40,7 +40,7 @@ accents. The default ``bareasc`` prefix (``#``) is used as a comment character in some shells so may need to be protected (for example in quotes) when typed into the command line. -The bare ASCII transformation is quite simple. It may not give the expected output +The bare ASCII transliteration is quite simple. It may not give the expected output for all languages. For example, German u-umlaut ``ü`` is transformed into ASCII ``u``, not into ``ue``. From cad2c055c5e1e595b76e013598fe454caa957bf2 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 16 Mar 2021 16:29:57 +0000 Subject: [PATCH 082/130] Make unicode handling explicit, to support python 2 and 3. Add link to Unidecode library in docs. Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 6 ++++-- docs/plugins/bareasc.rst | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index 43d83005a..d17cc4ac1 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -78,7 +78,9 @@ class BareascPlugin(BeetsPlugin): # Copied from commands.py - list_items if album: for album in lib.albums(query): - print_(unidecode(format(album, fmt))) + bare = unidecode(six.ensure_text(format(album, fmt))) + print_(six.ensure_str(bare)) else: for item in lib.items(query): - print_(six.ensure_str(unidecode(format(item, fmt)))) + bare = unidecode(six.ensure_text(format(item, fmt))) + print_(six.ensure_str(bare)) diff --git a/docs/plugins/bareasc.rst b/docs/plugins/bareasc.rst index a22063805..0c8d6636c 100644 --- a/docs/plugins/bareasc.rst +++ b/docs/plugins/bareasc.rst @@ -63,6 +63,7 @@ file. The only available option is: Credits ------- -The hard work in this plugin is done in Sean Burke's Unidecode library. +The hard work in this plugin is done in Sean Burke's +`Unidecode `__ library. Thanks are due to Sean and to all the people who created the Python version and the beets extensible query architecture. From b0110fa224e2af73125edb88a8b6cce0d6da5b12 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 16 Mar 2021 16:40:43 +0000 Subject: [PATCH 083/130] Try again to fix unidecode_list for python2 support Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index d17cc4ac1..2bdad98fc 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -79,8 +79,8 @@ class BareascPlugin(BeetsPlugin): if album: for album in lib.albums(query): bare = unidecode(six.ensure_text(format(album, fmt))) - print_(six.ensure_str(bare)) + print_(six.ensure_text(bare)) else: for item in lib.items(query): bare = unidecode(six.ensure_text(format(item, fmt))) - print_(six.ensure_str(bare)) + print_(six.ensure_text(bare)) From c3485b5b0415fb4e4bdf64b1f0bf6cf0840c4567 Mon Sep 17 00:00:00 2001 From: "Graham R. Cobb" Date: Tue, 16 Mar 2021 22:36:42 +0000 Subject: [PATCH 084/130] Remove unnecessary call to format. Signed-off-by: Graham R. Cobb --- beetsplug/bareasc.py | 5 ++--- test/test_bareasc.py | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/beetsplug/bareasc.py b/beetsplug/bareasc.py index 2bdad98fc..4d574c756 100644 --- a/beetsplug/bareasc.py +++ b/beetsplug/bareasc.py @@ -74,13 +74,12 @@ class BareascPlugin(BeetsPlugin): """Emulate normal 'list' command but with unidecode output.""" query = decargs(args) album = opts.album - fmt = u'' # Copied from commands.py - list_items if album: for album in lib.albums(query): - bare = unidecode(six.ensure_text(format(album, fmt))) + bare = unidecode(six.ensure_text(str(album))) print_(six.ensure_text(bare)) else: for item in lib.items(query): - bare = unidecode(six.ensure_text(format(item, fmt))) + bare = unidecode(six.ensure_text(str(item))) print_(six.ensure_text(bare)) diff --git a/test/test_bareasc.py b/test/test_bareasc.py index abb42a730..1ce4e6176 100644 --- a/test/test_bareasc.py +++ b/test/test_bareasc.py @@ -136,6 +136,15 @@ class BareascPluginTest(unittest.TestCase, TestHelper): self.assertIn('Antonin Dvorak', output.getvalue()) + def test_bareasc_format_output(self): + """Bare-ASCII version of list -f command - check output.""" + with capture_stdout() as output: + self.run_command('bareasc', 'with accents', + '-f', '$artist:: $title') + + self.assertEqual('Antonin Dvorak:: with accents\n', + output.getvalue()) + def suite(): """loader.""" From 6dc1f6414d2e229ad0502b69e7655361d3f51115 Mon Sep 17 00:00:00 2001 From: Nick Sellen Date: Wed, 17 Mar 2021 09:06:35 +0000 Subject: [PATCH 085/130] Remove ascii encoding step now library is fixed --- beetsplug/discogs.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/beetsplug/discogs.py b/beetsplug/discogs.py index 05abb80c2..d52973b6f 100644 --- a/beetsplug/discogs.py +++ b/beetsplug/discogs.py @@ -239,13 +239,10 @@ class DiscogsPlugin(BeetsPlugin): # cause a query to return no results, even if they match the artist or # album title. Use `re.UNICODE` flag to avoid stripping non-english # word characters. - # FIXME: Encode as ASCII to work around a bug: - # https://github.com/beetbox/beets/issues/1051 - # When the library is fixed, we should encode as UTF-8. - query = re.sub(r'(?u)\W+', ' ', query).encode('ascii', "replace") + query = re.sub(r'(?u)\W+', ' ', query) # Strip medium information from query, Things like "CD1" and "disk 1" # can also negate an otherwise positive result. - query = re.sub(br'(?i)\b(CD|disc)\s*\d+', b'', query) + query = re.sub(r'(?i)\b(CD|disc)\s*\d+', '', query) self.request_start() try: From 5183fc53addf6741f4c31b27e9f13f1e0ad16a5e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 12:29:39 +0100 Subject: [PATCH 086/130] when iterating over item keys, do not return duplicates ... for keys that exist both as Item and Album fields --- beets/library.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index 3ba79d069..00cf00c79 100644 --- a/beets/library.py +++ b/beets/library.py @@ -623,7 +623,9 @@ class Item(LibModel): """ keys = super(Item, self).keys(computed=computed) if with_album and self._cached_album: - keys += self._cached_album.keys(computed=computed) + keys = set(keys) + keys.update(self._cached_album.keys(computed=computed)) + keys = list(keys) return keys def get(self, key, default=None, with_album=True): From a3a26784945a6a91bcea4dd49d5ac99d5e5211a8 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 12:38:29 +0100 Subject: [PATCH 087/130] slight performance optimization for ui.show_model_changes I didn't actually benchmark this, but creating new FormattedItemMapping views in each iteration of the loop appeared like a rather inefficient way of doing things. In reality, this is probably completely irrelevant, since one would usually not show huge numbers of changed Models at a time. --- beets/ui/__init__.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 3d7cffb51..228305db7 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -666,10 +666,10 @@ def term_width(): FLOAT_EPSILON = 0.01 -def _field_diff(field, old, new): - """Given two Model objects, format their values for `field` and - highlight changes among them. Return a human-readable string. If the - value has not changed, return None instead. +def _field_diff(field, old, old_fmt, new, new_fmt): + """Given two Model objects and their formatted views, format their values + for `field` and highlight changes among them. Return a human-readable + string. If the value has not changed, return None instead. """ oldval = old.get(field) newval = new.get(field) @@ -682,8 +682,8 @@ def _field_diff(field, old, new): return None # Get formatted values for output. - oldstr = old.formatted().get(field, u'') - newstr = new.formatted().get(field, u'') + oldstr = old_fmt.get(field, u'') + newstr = new_fmt.get(field, u'') # For strings, highlight changes. For others, colorize the whole # thing. @@ -708,6 +708,11 @@ def show_model_changes(new, old=None, fields=None, always=False): """ old = old or new._db._get(type(new), new.id) + # Keep the formatted views around instead of re-creating them in each + # iteration step + old_fmt = old.formatted() + new_fmt = new.formatted() + # Build up lines showing changed fields. changes = [] for field in old: @@ -716,7 +721,7 @@ def show_model_changes(new, old=None, fields=None, always=False): continue # Detect and show difference for this field. - line = _field_diff(field, old, new) + line = _field_diff(field, old, old_fmt, new, new_fmt) if line: changes.append(u' {0}: {1}'.format(field, line)) @@ -727,7 +732,7 @@ def show_model_changes(new, old=None, fields=None, always=False): changes.append(u' {0}: {1}'.format( field, - colorize('text_highlight', new.formatted()[field]) + colorize('text_highlight', new_fmt[field]) )) # Print changes. From feb5041f57dc3a4fcf2747d5dbcf2f6e10e31212 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 14:29:28 +0100 Subject: [PATCH 088/130] replaygain: remove debugging leftovers --- beetsplug/replaygain.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 8b9447bdb..2ea5318ad 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1059,40 +1059,30 @@ class ReplayGainPlugin(BeetsPlugin): (not item.rg_album_gain or not item.rg_album_peak) for item in album.items()]) - def _store(self, item): - """Store an item to the database. - When testing, item.store() sometimes fails non-destructively with - sqlite.OperationalError. - This method is here to be patched to a retry-once helper function - in test_replaygain.py, so that it can still fail appropriately - outside of these tests. - """ - item.store() - def store_track_gain(self, item, track_gain): item.rg_track_gain = track_gain.gain item.rg_track_peak = track_gain.peak - self._store(item) + item.store() self._log.debug(u'applied track gain {0} LU, peak {1} of FS', item.rg_track_gain, item.rg_track_peak) def store_album_gain(self, item, album_gain): item.rg_album_gain = album_gain.gain item.rg_album_peak = album_gain.peak - self._store(item) + item.store() self._log.debug(u'applied album gain {0} LU, peak {1} of FS', item.rg_album_gain, item.rg_album_peak) def store_track_r128_gain(self, item, track_gain): item.r128_track_gain = track_gain.gain - self._store(item) + item.store() self._log.debug(u'applied r128 track gain {0} LU', item.r128_track_gain) def store_album_r128_gain(self, item, album_gain): item.r128_album_gain = album_gain.gain - self._store(item) + item.store() self._log.debug(u'applied r128 album gain {0} LU', item.r128_album_gain) From 3d5142433fa28891a94ea7740f6b1b338a514ee7 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 17:56:43 +0100 Subject: [PATCH 089/130] avoid deadlocks in Item.__repr__ --- beets/library.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/beets/library.py b/beets/library.py index 3ba79d069..170f0fd28 100644 --- a/beets/library.py +++ b/beets/library.py @@ -617,6 +617,16 @@ class Item(LibModel): return self._cached_album[key] raise + def __repr__(self): + # This must not use `with_album=True`, because that might access + # the database. When debugging, that is not guaranteed to succeed, and + # can even deadlock due to the database lock. + return '{0}({1})'.format( + type(self).__name__, + ', '.join('{0}={1!r}'.format(k, self[k]) + for k in self.keys(with_album=False)), + ) + def keys(self, computed=False, with_album=True): """Get a list of available field names. `with_album` controls whether the album's fields are included. From 0125fdf41553914f95ce86e9ac46324211c6d523 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 18:31:00 +0100 Subject: [PATCH 090/130] replaygain: don't nest functions when there's no need to --- beetsplug/replaygain.py | 66 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 8b9447bdb..3b01987cd 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1333,41 +1333,41 @@ class ReplayGainPlugin(BeetsPlugin): self.config['overwrite'].get(bool) ) + def command_func(self, lib, opts, args): + try: + write = ui.should_write(opts.write) + force = opts.force + + # Bypass self.open_pool() if called with `--threads 0` + if opts.threads != 0: + threads = opts.threads or self.config['threads'].get(int) + self.open_pool(threads) + + if opts.album: + albums = lib.albums(ui.decargs(args)) + self._log.info( + "Analyzing {} albums ~ {} backend..." + .format(len(albums), self.backend_name) + ) + for album in albums: + self.handle_album(album, write, force) + else: + items = lib.items(ui.decargs(args)) + self._log.info( + "Analyzing {} tracks ~ {} backend..." + .format(len(items), self.backend_name) + ) + for item in items: + self.handle_track(item, write, force) + + self.close_pool() + except (SystemExit, KeyboardInterrupt): + # Silence interrupt exceptions + pass + def commands(self): """Return the "replaygain" ui subcommand. """ - def func(lib, opts, args): - try: - write = ui.should_write(opts.write) - force = opts.force - - # Bypass self.open_pool() if called with `--threads 0` - if opts.threads != 0: - threads = opts.threads or self.config['threads'].get(int) - self.open_pool(threads) - - if opts.album: - albums = lib.albums(ui.decargs(args)) - self._log.info( - "Analyzing {} albums ~ {} backend..." - .format(len(albums), self.backend_name) - ) - for album in albums: - self.handle_album(album, write, force) - else: - items = lib.items(ui.decargs(args)) - self._log.info( - "Analyzing {} tracks ~ {} backend..." - .format(len(items), self.backend_name) - ) - for item in items: - self.handle_track(item, write, force) - - self.close_pool() - except (SystemExit, KeyboardInterrupt): - # Silence interrupt exceptions - pass - cmd = ui.Subcommand('replaygain', help=u'analyze for ReplayGain') cmd.parser.add_album_option() cmd.parser.add_option( @@ -1385,5 +1385,5 @@ class ReplayGainPlugin(BeetsPlugin): cmd.parser.add_option( "-W", "--nowrite", dest="write", action="store_false", help=u"don't write metadata (opposite of -w)") - cmd.func = func + cmd.func = self.command_func return [cmd] From d3ec2106cd9e00be7fd52954c288e612de1d6153 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Thu, 18 Mar 2021 18:37:15 +0100 Subject: [PATCH 091/130] replaygain: fix write/force flags on import This restores old behaviour. --- beetsplug/replaygain.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/beetsplug/replaygain.py b/beetsplug/replaygain.py index 8b9447bdb..d85a95022 100644 --- a/beetsplug/replaygain.py +++ b/beetsplug/replaygain.py @@ -1321,17 +1321,9 @@ class ReplayGainPlugin(BeetsPlugin): """ if self.config['auto']: if task.is_album: - self.handle_album( - task.album, - self.config['auto'].get(bool), - self.config['overwrite'].get(bool) - ) + self.handle_album(task.album, False) else: - self.handle_track( - task.item, - self.config['auto'].get(bool), - self.config['overwrite'].get(bool) - ) + self.handle_track(task.item, False) def commands(self): """Return the "replaygain" ui subcommand. From d7c556f7d17e0aef5c9e81aa07de6acf97e66f4a Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sat, 20 Mar 2021 09:04:03 -0400 Subject: [PATCH 092/130] Remove stray `cue` plugin (fix #3892) --- beetsplug/cue.py | 58 ------------------------------------------------ 1 file changed, 58 deletions(-) delete mode 100644 beetsplug/cue.py diff --git a/beetsplug/cue.py b/beetsplug/cue.py deleted file mode 100644 index 1ff817b2b..000000000 --- a/beetsplug/cue.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2016 Bruno Cauet -# Split an album-file in tracks thanks a cue file - -from __future__ import division, absolute_import, print_function - -import subprocess -from os import path -from glob import glob - -from beets.util import command_output, displayable_path -from beets.plugins import BeetsPlugin -from beets.autotag import TrackInfo - - -class CuePlugin(BeetsPlugin): - def __init__(self): - super(CuePlugin, self).__init__() - # this does not seem supported by shnsplit - self.config.add({ - 'keep_before': .1, - 'keep_after': .9, - }) - - # self.register_listener('import_task_start', self.look_for_cues) - - def candidates(self, items, artist, album, va_likely, extra_tags=None): - import pdb - pdb.set_trace() - - def item_candidates(self, item, artist, album): - dir = path.dirname(item.path) - cues = glob.glob(path.join(dir, "*.cue")) - if not cues: - return - if len(cues) > 1: - self._log.info(u"Found multiple cue files doing nothing: {0}", - list(map(displayable_path, cues))) - - cue_file = cues[0] - self._log.info("Found {} for {}", displayable_path(cue_file), item) - - try: - # careful: will ask for input in case of conflicts - command_output(['shnsplit', '-f', cue_file, item.path]) - except (subprocess.CalledProcessError, OSError): - self._log.exception(u'shnsplit execution failed') - return - - tracks = glob(path.join(dir, "*.wav")) - self._log.info("Generated {0} tracks", len(tracks)) - for t in tracks: - title = "dunno lol" - track_id = "wtf" - index = int(path.basename(t)[len("split-track"):-len(".wav")]) - yield TrackInfo(title=title, track_id=track_id, index=index, - artist=artist) - # generate TrackInfo instances From 01bc32e50edbe60d43d4db7e6643f94fadbe322c Mon Sep 17 00:00:00 2001 From: Jack Wilsdon Date: Wed, 24 Apr 2019 20:04:52 +0100 Subject: [PATCH 093/130] Allow equals within import --set value --- beets/ui/__init__.py | 5 ++++- docs/changelog.rst | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 228305db7..8d980d549 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -799,8 +799,11 @@ def _store_dict(option, opt_str, value, parser): setattr(parser.values, dest, {}) option_values = getattr(parser.values, dest) + # Decode the argument using the platform's argument encoding. + value = util.text_string(value, util.arg_encoding()) + try: - key, value = map(lambda s: util.text_string(s), value.split('=')) + key, value = value.split('=', 1) if not (key and value): raise ValueError except ValueError: diff --git a/docs/changelog.rst b/docs/changelog.rst index 07fa573f9..46c9f4b3f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -343,6 +343,8 @@ Fixes: * :doc:`/plugins/web`: DELETE and PATCH methods are disallowed by default. Set ``readonly: no`` web config option to enable them. :bug:`3870` +* Allow equals within ``--set`` value when importing. + :bug:`2984` For plugin developers: From 89624cf0a1c3f561bf4dbedad16dbe28d14cf16f Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 22 Mar 2021 19:04:49 +0100 Subject: [PATCH 094/130] setup.py: add 3.9 classifier, add note about ffmpeg dep for replaygain --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec1d2c819..8191d3888 100755 --- a/setup.py +++ b/setup.py @@ -178,8 +178,10 @@ setup( # embedart: ImageMagick # absubmit: extractor binary from https://acousticbrainz.org/download # keyfinder: KeyFinder - # replaygain: python-gi and GStreamer 1.0+ or mp3gain/aacgain + # replaygain: python-gi and GStreamer 1.0+ + # or mp3gain/aacgain # or Python Audio Tools + # or ffmpeg # ipfs: go-ipfs classifiers=[ @@ -196,6 +198,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', ], ) From 68fa03a5eb241225933431d21cd7347a760c0cd2 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 22 Mar 2021 19:07:34 +0100 Subject: [PATCH 095/130] ci: install ffmpeg for replaygain tests; show skipped tests We lost the ability to show skipped tests when transitioning to gh actions. As it turns out, all of the replaygain tests were being skipped, so as a start, try to install at least one backend. --- .github/workflows/ci.yaml | 4 ++++ tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a8b3a784..2b71b15c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,10 @@ jobs: python -m pip install --upgrade pip python -m pip install tox sphinx + - name: Install optional dependencies + run: | + sudo apt-get install ffmpeg # For replaygain + - name: Test with tox if: matrix.python-version != '3.9' run: | diff --git a/tox.ini b/tox.ini index 64acf1402..16a569344 100644 --- a/tox.ini +++ b/tox.ini @@ -18,8 +18,8 @@ deps = {test,cov}: {[_test]deps} lint: {[_lint]deps} commands = - test: python -bb -m pytest {posargs} - cov: coverage run -m pytest {posargs} + test: python -bb -m pytest -rs {posargs} + cov: coverage run -m pytest -rs {posargs} lint: python -m flake8 {posargs} {[_lint]files} [testenv:docs] From 78808e4654458465f0f7ceff404c6470e5992207 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 22 Mar 2021 20:02:25 +0100 Subject: [PATCH 096/130] ci: also run on latest Python alpha, but allow failure --- .github/workflows/ci.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b71b15c5..012617769 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,7 +6,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10-dev] env: PY_COLORS: 1 @@ -28,16 +28,25 @@ jobs: run: | sudo apt-get install ffmpeg # For replaygain - - name: Test with tox - if: matrix.python-version != '3.9' + - name: Test older Python versions with tox + if: matrix.python-version != '3.9' && matrix.python-version != '3.10-dev' run: | tox -e py-test - - name: Test with tox and get coverage + - name: Test latest Python version with tox and get coverage if: matrix.python-version == '3.9' run: | tox -vv -e py-cov + - name: Test nightly Python version with tox + if: matrix.python-version == '3.10-dev' + # continue-on-error is not ideal since it doesn't give a visible + # warning, but there doesn't seem to be anything better: + # https://github.com/actions/toolkit/issues/399 + continue-on-error: true + run: | + tox -e py-test + - name: Upload code coverage if: matrix.python-version == '3.9' run: | From 64dc3ec2a2e55078632830359224af0c68a45f13 Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Mon, 22 Mar 2021 21:48:30 +0100 Subject: [PATCH 097/130] plugins: Python 3.10 compatibility In 766c8b190bb782a5ef7bafbfa657007d4f158872 from [1] a slight hack was introduced that allowed to add new arguments to plugin events without breaking legacy plugins that did not feature those arguments in the signature of their handler functions. When improving logging management for plugins in 327b62b6103771bd19465f10a6d4f0412c2b9f1c from [2] this hack was removed, to be restored with 7f34c101d70126497afbf448c9d7ffb6faf6655a from [3] Now in addition to inspecting the function signature, an assumption is made about the string message of the TypeError that occurs for legacy handler functions (namely, that it start with func.__name__). In Python 3.10, the TypeError uses func.__qualname__ [4], however, due to the patch [5]. Thus, checking whether the TypeError is due to an outdated function signature or something internal to the handler fais. As a fix, instead of using __name__ or __qualname__ depending on Python version, let's just revert to the old hack from [1], I'm not seeing a significant advantage in [3]. The latter might be a bit faster since it doesn't construct a new dict for each call, but only for legacy calls in the excption handler. I doubt that really matters, and if it does, we should try to think of a better fix than inspecting error messages with no guarantees about their content in such a central place in the code. [1] https://github.com/beetbox/beets/pull/652 [2] https://github.com/beetbox/beets/pull/1320 [3] https://github.com/beetbox/beets/pull/1359 [4] https://docs.python.org/3.10/glossary.html#term-qualified-name [5] https://bugs.python.org/issue40679 --- beets/plugins.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 3abd911c9..23f938169 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -130,29 +130,30 @@ class BeetsPlugin(object): be sent for backwards-compatibility. """ if six.PY2: - func_args = inspect.getargspec(func).args + argspec = inspect.getargspec(func) + func_args = argspec.args + has_varkw = argspec.keywords is not None else: - func_args = inspect.getfullargspec(func).args + argspec = inspect.getfullargspec(func) + func_args = argspec.args + has_varkw = argspec.varkw is not None @wraps(func) def wrapper(*args, **kwargs): assert self._log.level == logging.NOTSET + verbosity = beets.config['verbose'].get(int) log_level = max(logging.DEBUG, base_log_level - 10 * verbosity) self._log.setLevel(log_level) + if not has_varkw: + kwargs = dict((k, v) for k, v in kwargs.items() + if k in func_args) + try: - try: - return func(*args, **kwargs) - except TypeError as exc: - if exc.args[0].startswith(func.__name__): - # caused by 'func' and not stuff internal to 'func' - kwargs = dict((arg, val) for arg, val in kwargs.items() - if arg in func_args) - return func(*args, **kwargs) - else: - raise + return func(*args, **kwargs) finally: self._log.setLevel(logging.NOTSET) + return wrapper def queries(self): From 07b5e69f4097744f891e186483abe5ba7cad96ae Mon Sep 17 00:00:00 2001 From: David Swarbrick Date: Sun, 19 Apr 2020 13:28:38 +0100 Subject: [PATCH 098/130] fetchart/artresizer: add max_filesize support (#3560) Squashed from the PR, relevant commit messages follow below: Added file size option to artresizer - In line with comments on PR, adjusted the ArtResizer API to add functionality to "resize to X bytes" through `max_filesize` arg - Adjustment to changelog.rst to include max_filesize change to ArtResizer and addition of new plugin. Added explicit tests for PIL & Imagemagick Methods - Checks new resizing functions do reduce the filesize of images Expose max_filesize logic to fetchart plugin - Add syspath escaping for OS cross compatibility - Return smaller PIL image even if max filesize not reached. - Test resize logic against known smaller filesize (//2) - Pass integer (not float) quality argument to PIL - Remove Pillow from dependencies - Implement "max_filesize" fetchart option, including logic to resize and rescale if maxwidth is also set. Added tests & documentation for fetchart additions. Tests now check that a target filesize is reached with a higher initial quality (a difficult check to pass). With a starting quality of 95% PIL takes 4 iterations to succeed in lowering the example cover image to 90% its original size. To cover all bases, the PIL loop has been changed to 5 iterations in the worst case, and the documentation altered to reflect the 50% loss in quality this implies. This seems reasonable as users concerned about performance would most likely be persuaded to install ImageMagick, or remove the maximum filesize constraint. The previous 30% figure was arbitrary. --- beets/util/artresizer.py | 51 ++++++++++++++++-- beetsplug/fetchart.py | 47 ++++++++++++---- docs/changelog.rst | 6 +-- docs/plugins/fetchart.rst | 7 +++ test/test_art.py | 30 ++++++++++- test/test_art_resize.py | 110 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 231 insertions(+), 20 deletions(-) create mode 100644 test/test_art_resize.py diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index c57918f16..0053dd7c0 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -64,12 +64,13 @@ def temp_file_for(path): return util.bytestring_path(f.name) -def pil_resize(maxwidth, path_in, path_out=None, quality=0): +def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """Resize using Python Imaging Library (PIL). Return the output path of resized image. """ path_out = path_out or temp_file_for(path_in) from PIL import Image + log.debug(u'artresizer: PIL resizing {0} to {1}', util.displayable_path(path_in), util.displayable_path(path_out)) @@ -83,14 +84,43 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0): quality = -1 im.save(util.py3_path(path_out), quality=quality) - return path_out + if max_filesize > 0: + # If maximum filesize is set, we attempt to lower the quality of + # jpeg conversion by a proportional amount, up to 3 attempts + # First, set the maximum quality to either provided, or 95 + if quality > 0: + lower_qual = quality + else: + lower_qual = 95 + for i in range(5): + # 3 attempts is an abitrary choice + filesize = os.stat(util.syspath(path_out)).st_size + log.debug(u"PIL Pass {0} : Output size: {1}B", i, filesize) + if filesize <= max_filesize: + return path_out + # The relationship between filesize & quality will be + # image dependent. + lower_qual -= 10 + # Restrict quality dropping below 10 + if lower_qual < 10: + lower_qual = 10 + # Use optimize flag to improve filesize decrease + im.save( + util.py3_path(path_out), quality=lower_qual, optimize=True + ) + log.warning(u"PIL Failed to resize file to below {0}B", + max_filesize) + return path_out + + else: + return path_out except IOError: log.error(u"PIL cannot create thumbnail for '{0}'", util.displayable_path(path_in)) return path_in -def im_resize(maxwidth, path_in, path_out=None, quality=0): +def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): """Resize using ImageMagick. Use the ``magick`` program or ``convert`` on older versions. Return @@ -103,6 +133,9 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. + + # "-define jpeg:extent=SIZEb" sets the target filesize + # for imagemagick to SIZE in bytes cmd = ArtResizer.shared.im_convert_cmd + [ util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), @@ -111,6 +144,9 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0): if quality > 0: cmd += ['-quality', '{0}'.format(quality)] + if max_filesize > 0: + cmd += ['-define', 'jpeg:extent={0}b'.format(max_filesize)] + cmd.append(util.syspath(path_out, prefix=False)) try: @@ -131,6 +167,7 @@ BACKEND_FUNCS = { def pil_getsize(path_in): from PIL import Image + try: im = Image.open(util.syspath(path_in)) return im.size @@ -171,6 +208,7 @@ class Shareable(type): lazily-created shared instance of ``MyClass`` while calling ``MyClass()`` to construct a new object works as usual. """ + def __init__(cls, name, bases, dict): super(Shareable, cls).__init__(name, bases, dict) cls._instance = None @@ -205,7 +243,9 @@ class ArtResizer(six.with_metaclass(Shareable, object)): self.im_convert_cmd = ['magick'] self.im_identify_cmd = ['magick', 'identify'] - def resize(self, maxwidth, path_in, path_out=None, quality=0): + def resize( + self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0 + ): """Manipulate an image file according to the method, returning a new path. For PIL or IMAGEMAGIC methods, resizes the image to a temporary file and encodes with the specified quality level. @@ -213,7 +253,8 @@ class ArtResizer(six.with_metaclass(Shareable, object)): """ if self.local: func = BACKEND_FUNCS[self.method[0]] - return func(maxwidth, path_in, path_out, quality=quality) + return func(maxwidth, path_in, path_out, + quality=quality, max_filesize=max_filesize) else: return path_in diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 1bf8ad428..37db35870 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -51,6 +51,7 @@ class Candidate(object): CANDIDATE_BAD = 0 CANDIDATE_EXACT = 1 CANDIDATE_DOWNSCALE = 2 + CANDIDATE_DOWNSIZE = 3 MATCH_EXACT = 0 MATCH_FALLBACK = 1 @@ -71,12 +72,14 @@ class Candidate(object): Return `CANDIDATE_BAD` if the file is unusable. Return `CANDIDATE_EXACT` if the file is usable as-is. - Return `CANDIDATE_DOWNSCALE` if the file must be resized. + Return `CANDIDATE_DOWNSCALE` if the file must be rescaled. + Return `CANDIDATE_DOWNSIZE` if the file must be resized. """ if not self.path: return self.CANDIDATE_BAD - if not (plugin.enforce_ratio or plugin.minwidth or plugin.maxwidth): + if (not (plugin.enforce_ratio or plugin.minwidth or plugin.maxwidth + or plugin.max_filesize)): return self.CANDIDATE_EXACT # get_size returns None if no local imaging backend is available @@ -94,7 +97,7 @@ class Candidate(object): short_edge = min(self.size) long_edge = max(self.size) - # Check minimum size. + # Check minimum dimension. if plugin.minwidth and self.size[0] < plugin.minwidth: self._log.debug(u'image too small ({} < {})', self.size[0], plugin.minwidth) @@ -122,22 +125,44 @@ class Candidate(object): self.size[0], self.size[1]) return self.CANDIDATE_BAD - # Check maximum size. + # Check maximum dimension. + downscale = False if plugin.maxwidth and self.size[0] > plugin.maxwidth: - self._log.debug(u'image needs resizing ({} > {})', + self._log.debug(u'image needs rescaling ({} > {})', self.size[0], plugin.maxwidth) - return self.CANDIDATE_DOWNSCALE + downscale = True - return self.CANDIDATE_EXACT + # Check filesize. + filesize = os.stat(syspath(self.path)).st_size + downsize = False + if plugin.max_filesize and filesize > plugin.max_filesize: + self._log.debug(u'image needs resizing ({}B > {}B)', + filesize, plugin.max_filesize) + downsize = True + + if downscale: + return self.CANDIDATE_DOWNSCALE + elif downsize: + return self.CANDIDATE_DOWNSIZE + else: + return self.CANDIDATE_EXACT def validate(self, plugin): self.check = self._validate(plugin) return self.check def resize(self, plugin): - if plugin.maxwidth and self.check == self.CANDIDATE_DOWNSCALE: - self.path = ArtResizer.shared.resize(plugin.maxwidth, self.path, - quality=plugin.quality) + if self.check == self.CANDIDATE_DOWNSCALE: + self.path = \ + ArtResizer.shared.resize(plugin.maxwidth, self.path, + quality=plugin.quality, + max_filesize=plugin.max_filesize) + elif self.check == self.CANDIDATE_DOWNSIZE: + # dimensions are correct, so maxwidth is set to maximum dimension + self.path = \ + ArtResizer.shared.resize(max(self.size), self.path, + quality=plugin.quality, + max_filesize=plugin.max_filesize) def _logged_get(log, *args, **kwargs): @@ -892,6 +917,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'minwidth': 0, 'maxwidth': 0, 'quality': 0, + 'max_filesize': 0, 'enforce_ratio': False, 'cautious': False, 'cover_names': ['cover', 'front', 'art', 'album', 'folder'], @@ -910,6 +936,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): self.minwidth = self.config['minwidth'].get(int) self.maxwidth = self.config['maxwidth'].get(int) + self.max_filesize = self.config['max_filesize'].get(int) self.quality = self.config['quality'].get(int) # allow both pixel and percentage-based margin specifications diff --git a/docs/changelog.rst b/docs/changelog.rst index e2a62e6d8..ea032bb95 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,9 +25,9 @@ New features: * A new :ref:`extra_tags` configuration option allows more tagged metadata to be included in MusicBrainz queries. * A new :doc:`/plugins/fish` adds `Fish shell`_ tab autocompletion to beets -* :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added a new ``quality`` - option that controls the quality of the image output when the image is - resized. +* :doc:`plugins/fetchart` and :doc:`plugins/embedart`: Added new ``quality`` option + that controls the quality of the output when an image is resized. Additionally, + the new ``max_filesize`` option for fetchart can be used to target a maximum image filesize. * :doc:`plugins/keyfinder`: Added support for `keyfinder-cli`_ Thanks to :user:`BrainDamage`. * :doc:`plugins/fetchart`: Added a new ``high_resolution`` config option to diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 168ca0fa0..6344c1562 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -49,6 +49,13 @@ file. The available options are: estimate the input image quality and uses 92 if it cannot be determined, and PIL defaults to 75. Default: 0 (disabled) +- **max_filesize**: The maximum size of a target piece of cover art in bytes. + When using an ImageMagick backend this sets + ``-define jpeg:extent=max_filesize``. Using PIL this will reduce JPG quality + by up to 50% to attempt to reach the target filesize. Neither method is + *guaranteed* to reach the target size, however in most cases it should + succeed. + Default: 0 (disabled) - **enforce_ratio**: Only images with a width:height ratio of 1:1 are considered as valid album art candidates if set to ``yes``. It is also possible to specify a certain deviation to the exact ratio to diff --git a/test/test_art.py b/test/test_art.py index 51e5a9fe8..d84ca4a91 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -742,14 +742,17 @@ class ArtImporterTest(UseThePlugin): class ArtForAlbumTest(UseThePlugin): - """ Tests that fetchart.art_for_album respects the size - configuration (e.g., minwidth, enforce_ratio) + """ Tests that fetchart.art_for_album respects the scale & filesize + configurations (e.g., minwidth, enforce_ratio, max_filesize) """ IMG_225x225 = os.path.join(_common.RSRC, b'abbey.jpg') IMG_348x348 = os.path.join(_common.RSRC, b'abbey-different.jpg') IMG_500x490 = os.path.join(_common.RSRC, b'abbey-similar.jpg') + IMG_225x225_SIZE = os.stat(util.syspath(IMG_225x225)).st_size + IMG_348x348_SIZE = os.stat(util.syspath(IMG_348x348)).st_size + def setUp(self): super(ArtForAlbumTest, self).setUp() @@ -839,6 +842,29 @@ class ArtForAlbumTest(UseThePlugin): self._assertImageResized(self.IMG_225x225, False) self._assertImageResized(self.IMG_348x348, True) + def test_fileresize(self): + self._require_backend() + self.plugin.max_filesize = self.IMG_225x225_SIZE // 2 + self._assertImageResized(self.IMG_225x225, True) + + def test_fileresize_if_necessary(self): + self._require_backend() + self.plugin.max_filesize = self.IMG_225x225_SIZE + self._assertImageResized(self.IMG_225x225, False) + self._assertImageIsValidArt(self.IMG_225x225, True) + + def test_fileresize_no_scale(self): + self._require_backend() + self.plugin.maxwidth = 300 + self.plugin.max_filesize = self.IMG_225x225_SIZE // 2 + self._assertImageResized(self.IMG_225x225, True) + + def test_fileresize_and_scale(self): + self._require_backend() + self.plugin.maxwidth = 200 + self.plugin.max_filesize = self.IMG_225x225_SIZE // 2 + self._assertImageResized(self.IMG_225x225, True) + class DeprecatedConfigTest(_common.TestCase): """While refactoring the plugin, the remote_priority option was deprecated, diff --git a/test/test_art_resize.py b/test/test_art_resize.py new file mode 100644 index 000000000..f7090d5e7 --- /dev/null +++ b/test/test_art_resize.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# This file is part of beets. +# Copyright 2020, David Swarbrick. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Tests for image resizing based on filesize.""" + +from __future__ import division, absolute_import, print_function + + +import unittest +import os + +from test import _common +from test.helper import TestHelper +from beets.util import syspath +from beets.util.artresizer import ( + pil_resize, + im_resize, + get_im_version, + get_pil_version, +) + + +class ArtResizerFileSizeTest(_common.TestCase, TestHelper): + """Unittest test case for Art Resizer to a specific filesize.""" + + IMG_225x225 = os.path.join(_common.RSRC, b"abbey.jpg") + IMG_225x225_SIZE = os.stat(syspath(IMG_225x225)).st_size + + def setUp(self): + """Called before each test, setting up beets.""" + self.setup_beets() + + def tearDown(self): + """Called after each test, unloading all plugins.""" + self.teardown_beets() + + def _test_img_resize(self, resize_func): + """Test resizing based on file size, given a resize_func.""" + # Check quality setting unaffected by new parameter + im_95_qual = resize_func( + 225, + self.IMG_225x225, + quality=95, + max_filesize=0, + ) + # check valid path returned - max_filesize hasn't broken resize command + self.assertExists(im_95_qual) + + # Attempt a lower filesize with same quality + im_a = resize_func( + 225, + self.IMG_225x225, + quality=95, + max_filesize=0.9 * os.stat(syspath(im_95_qual)).st_size, + ) + self.assertExists(im_a) + # target size was achieved + self.assertLess(os.stat(syspath(im_a)).st_size, + os.stat(syspath(im_95_qual)).st_size) + + # Attempt with lower initial quality + im_75_qual = resize_func( + 225, + self.IMG_225x225, + quality=75, + max_filesize=0, + ) + self.assertExists(im_75_qual) + + im_b = resize_func( + 225, + self.IMG_225x225, + quality=95, + max_filesize=0.9 * os.stat(syspath(im_75_qual)).st_size, + ) + self.assertExists(im_b) + # Check high (initial) quality still gives a smaller filesize + self.assertLess(os.stat(syspath(im_b)).st_size, + os.stat(syspath(im_75_qual)).st_size) + + @unittest.skipUnless(get_pil_version(), "PIL not available") + def test_pil_file_resize(self): + """Test PIL resize function is lowering file size.""" + self._test_img_resize(pil_resize) + + @unittest.skipUnless(get_im_version(), "ImageMagick not available") + def test_im_file_resize(self): + """Test IM resize function is lowering file size.""" + self._test_img_resize(im_resize) + + +def suite(): + """Run this suite of tests.""" + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == "__main__": + unittest.main(defaultTest="suite") From af70b75670b5a887d841bc096c4d75e2c598937f Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:57:01 +0100 Subject: [PATCH 099/130] fetchart/artresizer: Improve comments & docstrings, avoid os.stat if possible slight enhancements on top of https://github.com/beetbox/beets/pull/3560 --- beets/util/artresizer.py | 7 +++---- beetsplug/fetchart.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/beets/util/artresizer.py b/beets/util/artresizer.py index 0053dd7c0..bf6254c81 100644 --- a/beets/util/artresizer.py +++ b/beets/util/artresizer.py @@ -93,7 +93,7 @@ def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): else: lower_qual = 95 for i in range(5): - # 3 attempts is an abitrary choice + # 5 attempts is an abitrary choice filesize = os.stat(util.syspath(path_out)).st_size log.debug(u"PIL Pass {0} : Output size: {1}B", i, filesize) if filesize <= max_filesize: @@ -133,9 +133,6 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): # "-resize WIDTHx>" shrinks images with the width larger # than the given width while maintaining the aspect ratio # with regards to the height. - - # "-define jpeg:extent=SIZEb" sets the target filesize - # for imagemagick to SIZE in bytes cmd = ArtResizer.shared.im_convert_cmd + [ util.syspath(path_in, prefix=False), '-resize', '{0}x>'.format(maxwidth), @@ -144,6 +141,8 @@ def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0): if quality > 0: cmd += ['-quality', '{0}'.format(quality)] + # "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to + # SIZE in bytes. if max_filesize > 0: cmd += ['-define', 'jpeg:extent={0}b'.format(max_filesize)] diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 37db35870..d54e26102 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -73,7 +73,8 @@ class Candidate(object): Return `CANDIDATE_BAD` if the file is unusable. Return `CANDIDATE_EXACT` if the file is usable as-is. Return `CANDIDATE_DOWNSCALE` if the file must be rescaled. - Return `CANDIDATE_DOWNSIZE` if the file must be resized. + Return `CANDIDATE_DOWNSIZE` if the file must be resized, and possibly + also rescaled. """ if not self.path: return self.CANDIDATE_BAD @@ -90,8 +91,9 @@ class Candidate(object): if not self.size: self._log.warning(u'Could not get size of image (please see ' u'documentation for dependencies). ' - u'The configuration options `minwidth` and ' - u'`enforce_ratio` may be violated.') + u'The configuration options `minwidth`, ' + u'`enforce_ratio` and `max_filesize` ' + u'may be violated.') return self.CANDIDATE_EXACT short_edge = min(self.size) @@ -133,12 +135,13 @@ class Candidate(object): downscale = True # Check filesize. - filesize = os.stat(syspath(self.path)).st_size downsize = False - if plugin.max_filesize and filesize > plugin.max_filesize: - self._log.debug(u'image needs resizing ({}B > {}B)', - filesize, plugin.max_filesize) - downsize = True + if plugin.max_filesize: + filesize = os.stat(syspath(self.path)).st_size + if filesize > plugin.max_filesize: + self._log.debug(u'image needs resizing ({}B > {}B)', + filesize, plugin.max_filesize) + downsize = True if downscale: return self.CANDIDATE_DOWNSCALE From 7a0599efe20091b989f96dc653ecbb467a7ef09e Mon Sep 17 00:00:00 2001 From: wisp3rwind <17089248+wisp3rwind@users.noreply.github.com> Date: Tue, 23 Mar 2021 19:41:29 +0100 Subject: [PATCH 100/130] Python 3.5 is EOL and we should drop support thus add notice to changelog The final release 3.5.10 was on 2020-09-05 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4940064b3..d7c6f682d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ As a technical detail, it also introduces two new external libraries: `MediaFile`_ and `Confuse`_ used to be part of beets but are now reusable dependencies---packagers, please take note. Finally, this is the last version of beets where we intend to support Python -2.x; future releases will soon require Python 3.5. +2.x and 3.5; future releases will soon require Python 3.6. Major new features: From e96b5a7eba2bb7a76ada3b18fe93ae8cac046852 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Tue, 23 Mar 2021 21:55:51 -0500 Subject: [PATCH 101/130] Check files during import, show checker warnings and errors along with tagging choices. --- beets/importer.py | 1 + beets/ui/commands.py | 23 ++++++++++------- beetsplug/badfiles.py | 60 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index 3220b260f..c16350c68 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -472,6 +472,7 @@ class ImportTask(BaseImportTask): def __init__(self, toppath, paths, items): super(ImportTask, self).__init__(toppath, paths, items) self.choice_flag = None + self.skip_summary_judgement = False self.cur_album = None self.cur_artist = None self.candidates = [] diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 4d010f4b1..8d169129a 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -698,14 +698,18 @@ class TerminalImportSession(importer.ImportSession): print_(displayable_path(task.paths, u'\n') + u' ({0} items)'.format(len(task.items))) - # Take immediate action if appropriate. - action = _summary_judgment(task.rec) - if action == importer.action.APPLY: - match = task.candidates[0] - show_change(task.cur_artist, task.cur_album, match) - return match - elif action is not None: - return action + # Call here so plugins have the option to skip `_summary_judgement` + choices = self._get_choices(task) + + if not task.skip_summary_judgement: + # Take immediate action if appropriate. + action = _summary_judgment(task.rec) + if action == importer.action.APPLY: + match = task.candidates[0] + show_change(task.cur_artist, task.cur_album, match) + return match + elif action is not None: + return action # Loop until we have a choice. while True: @@ -713,7 +717,6 @@ class TerminalImportSession(importer.ImportSession): # `choose_candidate` may be an `importer.action`, an # `AlbumMatch` object for a specific selection, or a # `PromptChoice`. - choices = self._get_choices(task) choice = choose_candidate( task.candidates, False, task.rec, task.cur_artist, task.cur_album, itemcount=len(task.items), choices=choices @@ -742,6 +745,8 @@ class TerminalImportSession(importer.ImportSession): assert isinstance(choice, autotag.AlbumMatch) return choice + choices = self._get_choices(task) + def choose_item(self, task): """Ask the user for a choice about tagging a single item. Returns either an action constant or a TrackMatch object. diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 36b45de3a..c54190eaf 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -30,6 +30,7 @@ from beets.plugins import BeetsPlugin from beets.ui import Subcommand from beets.util import displayable_path, par_map from beets import ui +from beets import importer class CheckerCommandException(Exception): @@ -54,6 +55,11 @@ class BadFiles(BeetsPlugin): super(BadFiles, self).__init__() self.verbose = False + self.register_listener('import_task_start', + self.on_import_task_start) + self.register_listener('before_choose_candidate', + self.on_before_choose_candidate) + def run_command(self, cmd): self._log.debug(u"running command: {}", displayable_path(list2cmdline(cmd))) @@ -115,7 +121,7 @@ class BadFiles(BeetsPlugin): if not checker: self._log.error(u"no checker specified in the config for {}", ext) - return + return [] path = item.path if not isinstance(path, six.text_type): path = item.path.decode(sys.getfilesystemencoding()) @@ -130,25 +136,59 @@ class BadFiles(BeetsPlugin): ) else: self._log.error(u"error invoking {}: {}", e.checker, e.msg) - return + return [] + + error_lines = [] + if status > 0: - ui.print_(u"{}: checker exited with status {}" - .format(ui.colorize('text_error', dpath), status)) + error_lines.append(u"{}: checker exited with status {}" + .format(ui.colorize('text_error', dpath), status)) for line in output: - ui.print_(u" {}".format(line)) + error_lines.append(u" {}".format(line)) + elif errors > 0: - ui.print_(u"{}: checker found {} errors or warnings" - .format(ui.colorize('text_warning', dpath), errors)) + error_lines.append(u"{}: checker found {} errors or warnings" + .format(ui.colorize('text_warning', dpath), errors)) for line in output: - ui.print_(u" {}".format(line)) + error_lines.append(u" {}".format(line)) elif self.verbose: - ui.print_(u"{}: ok".format(ui.colorize('text_success', dpath))) + error_lines.append(u"{}: ok".format(ui.colorize('text_success', dpath))) + + return error_lines + + def on_import_task_start(self, task, session): + checks_failed = [] + + for item in task.items: + error_lines = self.check_item(item) + if error_lines: + checks_failed.append(error_lines) + + if checks_failed: + task._badfiles_checks_failed = checks_failed + task.skip_summary_judgement = True + + def on_before_choose_candidate(self, task, session): + if hasattr(task, '_badfiles_checks_failed'): + ui.print_('{} one or more files failed checks:' + .format(ui.colorize('text_warning', 'BAD'))) + for error in task._badfiles_checks_failed: + for error_line in error: + ui.print_(error_line) + + ui.print_() + def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) self.verbose = opts.verbose - par_map(self.check_item, items) + + def check_and_print(item): + for error_line in self.check_item(item): + ui.print_(error_line) + + par_map(check_and_print, items) def commands(self): bad_command = Subcommand('bad', From 2c9f3d6464d851236bef8a9173ec8e17d0ea9983 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Wed, 24 Mar 2021 16:51:58 -0500 Subject: [PATCH 102/130] Add `check_on_import` config for badfiles. --- beetsplug/badfiles.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index c54190eaf..c9edbc8c8 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -157,6 +157,9 @@ class BadFiles(BeetsPlugin): return error_lines def on_import_task_start(self, task, session): + if not self.config['check_on_import'].get(False): + return + checks_failed = [] for item in task.items: From 2f8a8e2f8a5fc7859328196a3f14adb5c9ee2a4e Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Wed, 24 Mar 2021 16:55:28 -0500 Subject: [PATCH 103/130] Add docs for badfiles' `check_on_import` option. --- docs/plugins/badfiles.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/plugins/badfiles.rst b/docs/plugins/badfiles.rst index a59dbd0d1..796f991e1 100644 --- a/docs/plugins/badfiles.rst +++ b/docs/plugins/badfiles.rst @@ -17,6 +17,7 @@ install yourself: You can also add custom commands for a specific extension, like this:: badfiles: + check_on_import: yes commands: ogg: myoggchecker --opt1 --opt2 flac: flac --test --warnings-as-errors --silent @@ -25,6 +26,10 @@ Custom commands will be run once for each file of the specified type, with the path to the file as the last argument. Commands must return a status code greater than zero for a file to be considered corrupt. +You can run the checkers when importing files by using the `check_on_import` +option. When on, checkers will be run against every imported file and warnings +and errors will be presented when selecting a tagging option. + .. _mp3val: http://mp3val.sourceforge.net/ .. _flac: https://xiph.org/flac/ From 227d420efbfd08764b55abf28aff7188160b6344 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Wed, 24 Mar 2021 17:04:17 -0500 Subject: [PATCH 104/130] Add entry to changelog. --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7c6f682d..421c1e964 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -210,6 +210,8 @@ Other new things: * The :doc:`/plugins/aura` has arrived! * :doc:`plugins/fetchart`: The new ``max_filesize`` option for fetchart can be used to target a maximum image filesize. +* :doc:`/plugins/badfiles`: Checkers can now be run during import with the + ``check_on_import`` config option. Fixes: From 9c17d9a9243b82adbc1d4f3b96f42172840bd07c Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Wed, 24 Mar 2021 17:29:16 -0500 Subject: [PATCH 105/130] linting --- beetsplug/badfiles.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index c9edbc8c8..44c1ce812 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -30,7 +30,6 @@ from beets.plugins import BeetsPlugin from beets.ui import Subcommand from beets.util import displayable_path, par_map from beets import ui -from beets import importer class CheckerCommandException(Exception): @@ -141,18 +140,21 @@ class BadFiles(BeetsPlugin): error_lines = [] if status > 0: - error_lines.append(u"{}: checker exited with status {}" - .format(ui.colorize('text_error', dpath), status)) + error_lines.append( + u"{}: checker exited with status {}" + .format(ui.colorize('text_error', dpath), status)) for line in output: error_lines.append(u" {}".format(line)) elif errors > 0: - error_lines.append(u"{}: checker found {} errors or warnings" - .format(ui.colorize('text_warning', dpath), errors)) + error_lines.append( + u"{}: checker found {} errors or warnings" + .format(ui.colorize('text_warning', dpath), errors)) for line in output: error_lines.append(u" {}".format(line)) elif self.verbose: - error_lines.append(u"{}: ok".format(ui.colorize('text_success', dpath))) + error_lines.append( + u"{}: ok".format(ui.colorize('text_success', dpath))) return error_lines @@ -163,7 +165,7 @@ class BadFiles(BeetsPlugin): checks_failed = [] for item in task.items: - error_lines = self.check_item(item) + error_lines = self.check_item(item) if error_lines: checks_failed.append(error_lines) @@ -181,7 +183,6 @@ class BadFiles(BeetsPlugin): ui.print_() - def command(self, lib, opts, args): # Get items from arguments items = lib.items(ui.decargs(args)) From 1d4a2c3e2ae348d6d3884bf18823cdbe0e268662 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Sat, 27 Mar 2021 11:37:26 -0500 Subject: [PATCH 106/130] Add a new hook for badfiles pre-import user interaction, instead of flagging the task. --- beets/importer.py | 1 - beets/plugins.py | 16 ++++++++++++++++ beets/ui/commands.py | 28 +++++++++++++++------------- beetsplug/badfiles.py | 20 ++++++++++++++++---- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/beets/importer.py b/beets/importer.py index 491383622..b7bfdb156 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -472,7 +472,6 @@ class ImportTask(BaseImportTask): def __init__(self, toppath, paths, items): super(ImportTask, self).__init__(toppath, paths, items) self.choice_flag = None - self.skip_summary_judgement = False self.cur_album = None self.cur_artist = None self.candidates = [] diff --git a/beets/plugins.py b/beets/plugins.py index 23f938169..402a6db17 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -501,6 +501,22 @@ def send(event, **arguments): return results +def send_seq(event, **arguments): + """Like `send` but passes the result of the previous event handler to the + next, and returns only the result of the last non-None event handler. + + `event` is the name of the event to send, all other named arguments + are passed along to the handlers. + """ + log.debug(u'Sequentially sending event: {0}', event) + previous = None + for handler in event_handlers()[event]: + result = handler(previous=previous, **arguments) + previous = result or previous + + return previous + + def feat_tokens(for_artist=True): """Return a regular expression that matches phrases like "featuring" that separate a main artist or a song title from secondary artists. diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 8d169129a..c8f85a999 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -698,18 +698,21 @@ class TerminalImportSession(importer.ImportSession): print_(displayable_path(task.paths, u'\n') + u' ({0} items)'.format(len(task.items))) - # Call here so plugins have the option to skip `_summary_judgement` - choices = self._get_choices(task) + # Let plugins display info or prompt the user before we go through the + # process of selecting candidate. + action = plugins.send_seq('import_task_before_choice', + session=self, task=task) + if action is not None: + return action - if not task.skip_summary_judgement: - # Take immediate action if appropriate. - action = _summary_judgment(task.rec) - if action == importer.action.APPLY: - match = task.candidates[0] - show_change(task.cur_artist, task.cur_album, match) - return match - elif action is not None: - return action + # Take immediate action if appropriate. + action = _summary_judgment(task.rec) + if action == importer.action.APPLY: + match = task.candidates[0] + show_change(task.cur_artist, task.cur_album, match) + return match + elif action is not None: + return action # Loop until we have a choice. while True: @@ -717,6 +720,7 @@ class TerminalImportSession(importer.ImportSession): # `choose_candidate` may be an `importer.action`, an # `AlbumMatch` object for a specific selection, or a # `PromptChoice`. + choices = self._get_choices(task) choice = choose_candidate( task.candidates, False, task.rec, task.cur_artist, task.cur_album, itemcount=len(task.items), choices=choices @@ -745,8 +749,6 @@ class TerminalImportSession(importer.ImportSession): assert isinstance(choice, autotag.AlbumMatch) return choice - choices = self._get_choices(task) - def choose_item(self, task): """Ask the user for a choice about tagging a single item. Returns either an action constant or a TrackMatch object. diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 44c1ce812..0fede8f4b 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -30,6 +30,7 @@ from beets.plugins import BeetsPlugin from beets.ui import Subcommand from beets.util import displayable_path, par_map from beets import ui +from beets import importer class CheckerCommandException(Exception): @@ -56,8 +57,8 @@ class BadFiles(BeetsPlugin): self.register_listener('import_task_start', self.on_import_task_start) - self.register_listener('before_choose_candidate', - self.on_before_choose_candidate) + self.register_listener('import_task_before_choice', + self.on_import_task_before_choice) def run_command(self, cmd): self._log.debug(u"running command: {}", @@ -171,9 +172,8 @@ class BadFiles(BeetsPlugin): if checks_failed: task._badfiles_checks_failed = checks_failed - task.skip_summary_judgement = True - def on_before_choose_candidate(self, task, session): + def on_import_task_before_choice(self, task, session, previous): if hasattr(task, '_badfiles_checks_failed'): ui.print_('{} one or more files failed checks:' .format(ui.colorize('text_warning', 'BAD'))) @@ -182,6 +182,18 @@ class BadFiles(BeetsPlugin): ui.print_(error_line) ui.print_() + ui.print_('What would you like to do?') + + sel = ui.input_options(['aBort', 'skip', 'continue']) + + if sel == 's': + return importer.action.SKIP + elif sel == 'c': + return None + elif sel == 'b': + raise importer.ImportAbort() + else: + raise Exception('Unexpected selection: {}'.format(sel)) def command(self, lib, opts, args): # Get items from arguments From 6c4cb82c0317f5d11c99cbbe4a331e6c68302b33 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Sat, 27 Mar 2021 12:03:56 -0500 Subject: [PATCH 107/130] badfiles: Respect any previous decision to skip an import and don't re-prompt the user. --- beetsplug/badfiles.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index 0fede8f4b..c68d7224e 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -174,6 +174,10 @@ class BadFiles(BeetsPlugin): task._badfiles_checks_failed = checks_failed def on_import_task_before_choice(self, task, session, previous): + # Already skipping, so no need to notify the user of anything + if previous == importer.action.SKIP: + return None + if hasattr(task, '_badfiles_checks_failed'): ui.print_('{} one or more files failed checks:' .format(ui.colorize('text_warning', 'BAD'))) From 39b5a7636c33fcffe0001f8038b1dc28f0af28d5 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Sat, 27 Mar 2021 12:04:25 -0500 Subject: [PATCH 108/130] Add line in dev docs for new hook. --- docs/dev/plugins.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index a6aa3d6d7..8a75bc142 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -200,6 +200,14 @@ The events currently available are: import pipeline stage is a better choice (see :ref:`plugin-stage`). Parameters: ``task`` and ``session``. +* `import_task_before_choice`: called after candidate search for an import task + before any decision is made about how/if to import or tag. Can be used to + present information about the task or initiate interaction with the user + before importing occurs. Return an importer action to take a specific action. + Parameters: ``task``, ``session``, and ``previous`` (which is the action + specified by the previous event handler for the same task, allowing an early + exist if e.g. a previous plugin has already selected to skip the import) + * `import_task_choice`: called after a decision has been made about an import task. This event can be used to initiate further interaction with the user. Use ``task.choice_flag`` to determine or change the action to be From 6081e6a8dff95edbf636f105d2548473ff12e95b Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sun, 28 Mar 2021 13:08:00 -0500 Subject: [PATCH 109/130] CR comments --- beetsplug/lyrics.py | 8 ++++---- docs/plugins/lyrics.rst | 14 ++++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 182e753f6..08efd38ff 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -716,10 +716,10 @@ class LyricsPlugin(plugins.BeetsPlugin): def sanitize_bs_sources(self, sources): for source in self.BS_SOURCES: if source in sources: - self._log.warning(u'To use the %s lyrics source, you must ' - u'install the beautifulsoup4 module. See ' - u'the documentation for further details.' - % source) + self._log.debug(u'To use the %s lyrics source, you must ' + u'install the beautifulsoup4 module. See ' + u'the documentation for further details.' + % source) sources.remove(source) return sources diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index c8e12b526..f05ef6357 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -62,7 +62,8 @@ configuration file. The available options are: Default: ``google musixmatch genius tekstowo``, i.e., all the available sources. The ``google`` source will be automatically deactivated if no ``google_API_key`` is setup. - The following sources will only be enabled if BeatifulSoup is installed: ``[google, genius, tekstowo]`` + The ``google``, ``genius``, and ``tekstowo`` sources will only be enabled if + BeautifulSoup is installed. Here's an example of ``config.yaml``:: @@ -128,6 +129,11 @@ few suggestions. Activate Google Custom Search ------------------------------ +Using the Google backend requires `BeautifulSoup`_, which you can install +using `pip`_ by typing:: + + pip install beautifulsoup4 + You also need to `register for a Google API key`_. Set the ``google_API_key`` configuration option to your key. Then add ``google`` to the list of sources in your configuration (or use @@ -150,11 +156,11 @@ After that, the lyrics plugin will fall back on other declared data sources. .. _pip: https://pip.pypa.io .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ -Installing BeautifulSoup for Google, Genius, or Tekstowo.pl Backends +Activate Genius and Tekstowo.pl Lyrics -------------------------------------------------------------------- -Using the Google, Genius, or Tekstowo.pl backends requires `BeautifulSoup`_, -which you can install using `pip`_ by typing:: +Using the Genius or Tekstowo.pl backends requires `BeautifulSoup`_, which +you can install using `pip`_ by typing:: pip install beautifulsoup4 From f067180c4c6a8cac441da46f0e8af13a80dda368 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 28 Mar 2021 14:23:38 -0400 Subject: [PATCH 110/130] Shorten a ReST underline --- docs/plugins/lyrics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index f05ef6357..7c5a3773c 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -157,7 +157,7 @@ After that, the lyrics plugin will fall back on other declared data sources. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/bs4/doc/ Activate Genius and Tekstowo.pl Lyrics --------------------------------------------------------------------- +-------------------------------------- Using the Genius or Tekstowo.pl backends requires `BeautifulSoup`_, which you can install using `pip`_ by typing:: From 79616b42ed5111f3a69420200da5b0ad08b5201f Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Sun, 28 Mar 2021 16:53:01 -0500 Subject: [PATCH 111/130] Use simpler approach of asserting that at most one handler of `import_task_before_choice` returns an action. --- beets/plugins.py | 16 ---------------- beets/ui/commands.py | 14 ++++++++++---- beetsplug/badfiles.py | 6 +----- docs/dev/plugins.rst | 5 ++--- 4 files changed, 13 insertions(+), 28 deletions(-) diff --git a/beets/plugins.py b/beets/plugins.py index 402a6db17..23f938169 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -501,22 +501,6 @@ def send(event, **arguments): return results -def send_seq(event, **arguments): - """Like `send` but passes the result of the previous event handler to the - next, and returns only the result of the last non-None event handler. - - `event` is the name of the event to send, all other named arguments - are passed along to the handlers. - """ - log.debug(u'Sequentially sending event: {0}', event) - previous = None - for handler in event_handlers()[event]: - result = handler(previous=previous, **arguments) - previous = result or previous - - return previous - - def feat_tokens(for_artist=True): """Return a regular expression that matches phrases like "featuring" that separate a main artist or a song title from secondary artists. diff --git a/beets/ui/commands.py b/beets/ui/commands.py index c8f85a999..9d3e3dda8 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -700,10 +700,16 @@ class TerminalImportSession(importer.ImportSession): # Let plugins display info or prompt the user before we go through the # process of selecting candidate. - action = plugins.send_seq('import_task_before_choice', - session=self, task=task) - if action is not None: - return action + results = plugins.send('import_task_before_choice', + session=self, task=task) + actions = [action for action in results if action] + + if len(actions) == 1: + return actions[0] + elif len(actions) > 1: + raise Exception( + u'Only one handler for `import_task_before_choice` may return ' + u'an action.') # Take immediate action if appropriate. action = _summary_judgment(task.rec) diff --git a/beetsplug/badfiles.py b/beetsplug/badfiles.py index c68d7224e..5eaecc054 100644 --- a/beetsplug/badfiles.py +++ b/beetsplug/badfiles.py @@ -173,11 +173,7 @@ class BadFiles(BeetsPlugin): if checks_failed: task._badfiles_checks_failed = checks_failed - def on_import_task_before_choice(self, task, session, previous): - # Already skipping, so no need to notify the user of anything - if previous == importer.action.SKIP: - return None - + def on_import_task_before_choice(self, task, session): if hasattr(task, '_badfiles_checks_failed'): ui.print_('{} one or more files failed checks:' .format(ui.colorize('text_warning', 'BAD'))) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 8a75bc142..d81461f4d 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -204,9 +204,8 @@ The events currently available are: before any decision is made about how/if to import or tag. Can be used to present information about the task or initiate interaction with the user before importing occurs. Return an importer action to take a specific action. - Parameters: ``task``, ``session``, and ``previous`` (which is the action - specified by the previous event handler for the same task, allowing an early - exist if e.g. a previous plugin has already selected to skip the import) + Only one handler may return a non-None result. + Parameters: ``task`` and ``session`` * `import_task_choice`: called after a decision has been made about an import task. This event can be used to initiate further interaction with the user. From c8fa1b28094110a63f757a9a3966545ac6043a42 Mon Sep 17 00:00:00 2001 From: Ryan Lanny Jenkins Date: Mon, 29 Mar 2021 18:12:30 -0500 Subject: [PATCH 112/130] Raise PluginConflictException instead of the generic Exception for new hook. --- beets/ui/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/ui/commands.py b/beets/ui/commands.py index 9d3e3dda8..38687b3a9 100755 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -707,7 +707,7 @@ class TerminalImportSession(importer.ImportSession): if len(actions) == 1: return actions[0] elif len(actions) > 1: - raise Exception( + raise plugins.PluginConflictException( u'Only one handler for `import_task_before_choice` may return ' u'an action.') From a8fe8ce9a78ec320cd52a70b0ee2e7502ca8fc91 Mon Sep 17 00:00:00 2001 From: Evgeniy Gurevich Date: Thu, 1 Apr 2021 14:21:49 +0800 Subject: [PATCH 113/130] Fix for Musixmatch multi-part lyrics Sometimes Musixmatch lyrics come in 2 parts. Also, sometime non-existent lyrics return some content --- beetsplug/lyrics.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 08efd38ff..84fe6c6bb 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -291,14 +291,25 @@ class MusiXmatch(Backend): self._log.warning(u'we are blocked at MusixMatch: url %s failed' % url) return - html_part = html.split('

1: + html_part1 = html_parts[-2] + lyrics_part1 = extract_text_between(html_part1, '>', '

') + else: + lyrics_part1 = '' + html_part2 = html_parts[-1] + lyrics_part2 = extract_text_between(html_part2, '>', '

') + lyrics = lyrics_part1 + '\n' + lyrics_part2 lyrics = lyrics.strip(',"').replace('\\n', '\n') # another odd case: sometimes only that string remains, for # missing songs. this seems to happen after being blocked # above, when filling in the CAPTCHA. if "Instant lyrics for all your music." in lyrics: return + # sometimes there are non-existent lyrics with some content + if 'Think is wrong? You can always add the lyrics' in lyrics: + return return lyrics From afc072801c4a254e6b4114d89133d6bebc3e34b9 Mon Sep 17 00:00:00 2001 From: Evgeniy Gurevich Date: Fri, 2 Apr 2021 10:35:53 +0800 Subject: [PATCH 114/130] Simplify Musixmatch multi-part lyrics parser --- beetsplug/lyrics.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 84fe6c6bb..125d1330d 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -292,15 +292,11 @@ class MusiXmatch(Backend): % url) return html_parts = html.split('

Date: Wed, 7 Apr 2021 20:12:52 +0800 Subject: [PATCH 115/130] Update changelog.rst --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d9e30e1fd..6aca3ac8e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -215,6 +215,8 @@ Other new things: Fixes: +* :bug:`/plugins/lyrics`: Fixed Musixmatch fetch lyrics divided into multiple elements on the web-page +* :bug:`/plugins/lyrics`: Fixed Musixmatch fetch for non-existing lyrics * :bug:`/plugins/web`: Allow use of backslash in regex web queries. :bug:`3867` * :bug:`/plugins/web`: Fixed a small bug which caused album artpath to be From 31f4df3ff3d23a49d5206345d549aa3916e96478 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sun, 11 Apr 2021 17:30:45 -0500 Subject: [PATCH 116/130] Fix falso positive lyric pages for Tekstowo --- beetsplug/lyrics.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 125d1330d..b154d58ce 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -416,12 +416,16 @@ class Tekstowo(Backend): def fetch(self, artist, title): url = self.build_url(title, artist) + print(url) search_results = self.fetch_url(url) song_page_url = self.parse_search_results(search_results) if not song_page_url: return None + print('--------------------') + print(song_page_url) + song_page_html = self.fetch_url(song_page_url) return self.extract_lyrics(song_page_html) @@ -437,8 +441,14 @@ class Tekstowo(Backend): except HTMLParseError: return None - song_row = html.find("div", class_="content"). \ - find_all("div", class_="box-przeboje")[0] + song_rows = html.find("div", class_="content"). \ + find("div", class_="card"). \ + find_all("div", class_="box-przeboje") + + if len(song_rows) < 1: + return None + + song_row = song_rows[0] if not song_row: return None From 0dd569a884baa2b8b6083a1614de069270e3ba65 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sun, 11 Apr 2021 17:34:46 -0500 Subject: [PATCH 117/130] Add changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6aca3ac8e..88338f7a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -353,6 +353,8 @@ Fixes: :bug:`3870` * Allow equals within ``--set`` value when importing. :bug:`2984` +* :doc:`/plugins/lyrics`: Fix crashes for Tekstowo false positives + :bug:`3904` For plugin developers: From 8b979f2a50f526350b91461f230b8f5f0e171a19 Mon Sep 17 00:00:00 2001 From: Xavier Hocquet Date: Sun, 11 Apr 2021 18:38:01 -0500 Subject: [PATCH 118/130] Review comments --- beetsplug/lyrics.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index b154d58ce..f0290f74a 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -416,16 +416,12 @@ class Tekstowo(Backend): def fetch(self, artist, title): url = self.build_url(title, artist) - print(url) search_results = self.fetch_url(url) song_page_url = self.parse_search_results(search_results) if not song_page_url: return None - print('--------------------') - print(song_page_url) - song_page_html = self.fetch_url(song_page_url) return self.extract_lyrics(song_page_html) @@ -445,7 +441,7 @@ class Tekstowo(Backend): find("div", class_="card"). \ find_all("div", class_="box-przeboje") - if len(song_rows) < 1: + if not song_rows: return None song_row = song_rows[0] From 50163b373f527d1b1f8b2442240ca547e846744e Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Sat, 17 Apr 2021 09:51:03 -0400 Subject: [PATCH 119/130] ci: scope ci workflow (#3907) --- .github/workflows/ci.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 012617769..91706f89d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,11 @@ name: ci -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + branches: + - master jobs: test: runs-on: ${{ matrix.platform }} From e1a901dce5c7b867a5c84a438044c4093bcbed05 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Mon, 19 Apr 2021 19:37:00 -0400 Subject: [PATCH 120/130] chore: makes discussions more visible (#3807) --- .github/ISSUE_TEMPLATE/config.yml | 8 +++++ .github/ISSUE_TEMPLATE/feature-request.md | 39 ++++++++++++++--------- README.rst | 21 ++++++++---- 3 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..f673e8e5c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: 💡 Have an idea for a new feature? + url: https://github.com/beetbox/beets/discussions + about: Create a new idea discussion! + - name: 🙇 Need help with beets? + url: https://github.com/beetbox/beets/discussions + about: Create a new help discussion if it hasn't been asked before! diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 5eb651aac..b15fb4fa7 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -1,26 +1,35 @@ --- name: "\U0001F680 Feature request" -about: Suggest a new idea for beets +about: "Formalize a feature request from GitHub Discussions" --- -### Use case - -I'm trying to use beets to... - - -### Solution +### Proposed solution + + + +### Objective + + + +#### Goals + + + +#### Non-goals -### Alternatives + +#### Anti-goals + + diff --git a/README.rst b/README.rst index 4a2d8cb10..6cdcd3054 100644 --- a/README.rst +++ b/README.rst @@ -81,8 +81,8 @@ Install ------- You can install beets by typing ``pip install beets``. -Beets has also been packaged in the `software repositories`_ of several distributions. -Check out the `Getting Started`_ guide for more information. +Beets has also been packaged in the `software repositories`_ of several +distributions. Check out the `Getting Started`_ guide for more information. .. _Getting Started: https://beets.readthedocs.org/page/guides/main.html .. _software repositories: https://repology.org/project/beets/versions @@ -90,7 +90,9 @@ Check out the `Getting Started`_ guide for more information. Contribute ---------- -Thank you for considering contributing to ``beets``! Whether you're a programmer or not, you should be able to find all the info you need at `CONTRIBUTING.rst`_. +Thank you for considering contributing to ``beets``! Whether you're a +programmer or not, you should be able to find all the info you need at +`CONTRIBUTING.rst`_. .. _CONTRIBUTING.rst: https://github.com/beetbox/beets/blob/master/CONTRIBUTING.rst @@ -105,12 +107,19 @@ news and updates. Contact ------- -* Encountered a bug you'd like to report or have an idea for a new feature? Check out our `issue tracker`_! If your issue or feature hasn't already been reported, please `open a new ticket`_ and we'll be in touch with you shortly. If you'd like to vote on a feature/bug, simply give a :+1: on issues you'd like to see prioritized over others. -* Need help/support, would like to start a discussion, or would just like to introduce yourself to the team? Check out our `forums`_! +* Encountered a bug you'd like to report? Check out our `issue tracker`_! + * If your issue hasn't already been reported, please `open a new ticket`_ + and we'll be in touch with you shortly. + * If you'd like to vote on a feature/bug, simply give a :+1: on issues + you'd like to see prioritized over others. +* Need help/support, would like to start a discussion, have an idea for a new + feature, or would just like to introduce yourself to the team? Check out + `GitHub Discussions`_ or `Discourse`_! +.. _GitHub Discussions: https://github.com/beetbox/beets/discussions .. _issue tracker: https://github.com/beetbox/beets/issues .. _open a new ticket: https://github.com/beetbox/beets/issues/new/choose -.. _forums: https://discourse.beets.io/ +.. _Discourse: https://discourse.beets.io/ Authors ------- From 2fa37177319c5da90413b456f28bc86cbf0b2c79 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 1 May 2021 22:38:47 +0200 Subject: [PATCH 121/130] Optimise FormattedMapping when querying a specific set of fields This changes greatly improves the speed of `beet export` and `beet info` when the `--include-keys` option is used. It also removes the globbing feature of `--include-keys` that was added in #1295. (See #3762 for discussion). Listing all fields for an item requires querying the database to find any flex attributes. This is slow when done for every item being exported. We already have a way for the user to specify a fixed set of keys, but we previously queried everything and filtered it afterwards. The new approach is more efficient. Code that iterates through all fields now have to handle invalid field names. The export and info plugins output invalid fields as None. Timings before: > /usr/bin/time beet export -i title,path,artist -l Bob Dylan 13.26user 20.22system 0:34.01elapsed 98%CPU (0avgtext+0avgdata 52544maxresident)k > /usr/bin/time beet export -l Bob Dylan 12.93user 20.15system 0:33.58elapsed 98%CPU (0avgtext+0avgdata 53632maxresident)k Timings after: > /usr/bin/time beet export -l Bob Dylan 13.33user 20.17system 0:34.02elapsed 98%CPU (0avgtext+0avgdata 53500maxresident)k > /usr/bin/time beet export -i title,path,artist -l Bob Dylan 0.49user 0.07system 0:00.56elapsed 98%CPU (0avgtext+0avgdata 50496maxresident)k Notice the dramatic speedup in the last example! --- beets/dbcore/db.py | 21 ++++++++++---- beets/library.py | 44 +++++++++++++++++++---------- beetsplug/export.py | 13 +++++---- beetsplug/info.py | 64 +++++++++++++++---------------------------- docs/changelog.rst | 2 ++ docs/plugins/info.rst | 10 +++---- test/test_info.py | 11 -------- 7 files changed, 80 insertions(+), 85 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index b87b41644..ae6151f34 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -52,15 +52,24 @@ class FormattedMapping(Mapping): The accessor `mapping[key]` returns the formatted version of `model[key]` as a unicode string. + The `included_keys` parameter allows filtering the fields that are + returned. By default all fields are returned. Limiting to specific keys can + avoid expensive per-item database queries. + If `for_path` is true, all path separators in the formatted values are replaced. """ - def __init__(self, model, for_path=False, compute_keys=True): + ALL_KEYS = '*' + + def __init__(self, model, included_keys=ALL_KEYS, for_path=False): self.for_path = for_path self.model = model - if compute_keys: - self.model_keys = model.keys(True) + if included_keys == self.ALL_KEYS: + # Performance note: this triggers a database query. + self.model_keys = self.model.keys(True) + else: + self.model_keys = included_keys def __getitem__(self, key): if key in self.model_keys: @@ -605,11 +614,11 @@ class Model(object): _formatter = FormattedMapping - def formatted(self, for_path=False): + def formatted(self, included_keys=_formatter.ALL_KEYS, for_path=False): """Get a mapping containing all values on this object formatted as human-readable unicode strings. """ - return self._formatter(self, for_path) + return self._formatter(self, included_keys, for_path) def evaluate_template(self, template, for_path=False): """Evaluate a template (a string or a `Template` object) using @@ -619,7 +628,7 @@ class Model(object): # Perform substitution. if isinstance(template, six.string_types): template = functemplate.template(template) - return template.substitute(self.formatted(for_path), + return template.substitute(self.formatted(for_path=for_path), self._template_funcs()) # Parsing. diff --git a/beets/library.py b/beets/library.py index df20c7a3d..9e459a3f2 100644 --- a/beets/library.py +++ b/beets/library.py @@ -374,12 +374,19 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): Album-level fields take precedence if `for_path` is true. """ - def __init__(self, item, for_path=False): + ALL_KEYS = '*' + + def __init__(self, item, included_keys=ALL_KEYS, for_path=False): # We treat album and item keys specially here, # so exclude transitive album keys from the model's keys. - super(FormattedItemMapping, self).__init__(item, for_path, - compute_keys=False) - self.model_keys = item.keys(computed=True, with_album=False) + super(FormattedItemMapping, self).__init__(item, included_keys=[], + for_path=for_path) + self.included_keys = included_keys + if included_keys == self.ALL_KEYS: + # Performance note: this triggers a database query. + self.model_keys = item.keys(computed=True, with_album=False) + else: + self.model_keys = included_keys self.item = item @lazy_property @@ -390,10 +397,14 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): def album_keys(self): album_keys = [] if self.album: - for key in self.album.keys(computed=True): - if key in Album.item_keys \ - or key not in self.item._fields.keys(): - album_keys.append(key) + if self.included_keys == self.ALL_KEYS: + # Performance note: this triggers a database query. + for key in self.album.keys(computed=True): + if key in Album.item_keys \ + or key not in self.item._fields.keys(): + album_keys.append(key) + else: + album_keys = self.included_keys return album_keys @property @@ -422,12 +433,15 @@ class FormattedItemMapping(dbcore.db.FormattedMapping): # `artist` and `albumartist` fields fall back to one another. # This is helpful in path formats when the album artist is unset # on as-is imports. - if key == 'artist' and not value: - return self._get('albumartist') - elif key == 'albumartist' and not value: - return self._get('artist') - else: - return value + try: + if key == 'artist' and not value: + return self._get('albumartist') + elif key == 'albumartist' and not value: + return self._get('artist') + except KeyError: + pass + + return value def __iter__(self): return iter(self.all_keys) @@ -1703,7 +1717,7 @@ class DefaultTemplateFunctions(object): return res # Flatten disambiguation value into a string. - disam_value = album.formatted(True).get(disambiguator) + disam_value = album.formatted(for_path=True).get(disambiguator) # Return empty string if disambiguator is empty. if disam_value: diff --git a/beetsplug/export.py b/beetsplug/export.py index 3ce9ab887..9aabaea05 100644 --- a/beetsplug/export.py +++ b/beetsplug/export.py @@ -26,8 +26,9 @@ from xml.etree import ElementTree from datetime import datetime, date from beets.plugins import BeetsPlugin from beets import ui +from beets import util import mediafile -from beetsplug.info import make_key_filter, library_data, tag_data +from beetsplug.info import library_data, tag_data class ExportEncoder(json.JSONEncoder): @@ -129,16 +130,18 @@ class ExportPlugin(BeetsPlugin): for keys in opts.included_keys: included_keys.extend(keys.split(',')) - key_filter = make_key_filter(included_keys) - for data_emitter in data_collector(lib, ui.decargs(args)): try: - data, item = data_emitter() + data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue - data = key_filter(data) + for key, value in data.items(): + if isinstance(value, bytes): + data[key] = util.displayable_path(value) + + items += [data] if file_format_is_line_based: export_format.export(data, **format_options) diff --git a/beetsplug/info.py b/beetsplug/info.py index 05d215e7b..828b0a81a 100644 --- a/beetsplug/info.py +++ b/beetsplug/info.py @@ -19,7 +19,6 @@ from __future__ import division, absolute_import, print_function import os -import re from beets.plugins import BeetsPlugin from beets import ui @@ -42,15 +41,29 @@ def tag_data(lib, args): yield tag_data_emitter(item.path) +def tag_fields(): + fields = set(mediafile.MediaFile.readable_fields()) + fields.add('art') + return fields + + def tag_data_emitter(path): - def emitter(): - fields = list(mediafile.MediaFile.readable_fields()) - fields.remove('images') + def emitter(included_keys): + if included_keys == '*': + fields = tag_fields() + else: + fields = included_keys + if 'images' in fields: + # We can't serialize the image data. + fields.remove('images') mf = mediafile.MediaFile(syspath(path)) tags = {} for field in fields: - tags[field] = getattr(mf, field) - tags['art'] = mf.art is not None + if field == 'art': + tags[field] = mf.art is not None + else: + tags[field] = getattr(mf, field, None) + # create a temporary Item to take advantage of __format__ item = Item.from_path(syspath(path)) @@ -64,8 +77,8 @@ def library_data(lib, args): def library_data_emitter(item): - def emitter(): - data = dict(item.formatted()) + def emitter(included_keys): + data = dict(item.formatted(included_keys=included_keys)) return data, item return emitter @@ -185,18 +198,16 @@ class InfoPlugin(BeetsPlugin): included_keys.extend(keys.split(',')) # Drop path even if user provides it multiple times included_keys = [k for k in included_keys if k != 'path'] - key_filter = make_key_filter(included_keys) first = True summary = {} for data_emitter in data_collector(lib, ui.decargs(args)): try: - data, item = data_emitter() + data, item = data_emitter(included_keys or '*') except (mediafile.UnreadableFileError, IOError) as ex: self._log.error(u'cannot read file: {0}', ex) continue - data = key_filter(data) if opts.summarize: update_summary(summary, data) else: @@ -211,34 +222,3 @@ class InfoPlugin(BeetsPlugin): if opts.summarize: print_data(summary) - - -def make_key_filter(include): - """Return a function that filters a dictionary. - - The returned filter takes a dictionary and returns another - dictionary that only includes the key-value pairs where the key - glob-matches one of the keys in `include`. - """ - # By default, if no field inclusions are specified, include - # everything but `path`. - if not include: - def filter_(data): - return {k: v for k, v in data.items() - if k != 'path'} - return filter_ - - matchers = [] - for key in include: - key = re.escape(key) - key = key.replace(r'\*', '.*') - matchers.append(re.compile(key + '$')) - - def filter_(data): - filtered = {} - for key, value in data.items(): - if any([m.match(key) for m in matchers]): - filtered[key] = value - return filtered - - return filter_ diff --git a/docs/changelog.rst b/docs/changelog.rst index 88338f7a7..6caed75c6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -212,6 +212,8 @@ Other new things: used to target a maximum image filesize. * :doc:`/plugins/badfiles`: Checkers can now be run during import with the ``check_on_import`` config option. +* :doc:`/plugins/export`: big speedups when `--include-keys` option is used + Thanks to :user:`ssssam`. Fixes: diff --git a/docs/plugins/info.rst b/docs/plugins/info.rst index d628fb4ad..3950cf0aa 100644 --- a/docs/plugins/info.rst +++ b/docs/plugins/info.rst @@ -20,14 +20,12 @@ your library:: If you just want to see specific properties you can use the ``--include-keys`` option to filter them. The argument is a -comma-separated list of simple glob patterns where ``*`` matches any -string. For example:: +comma-separated list of field names. For example:: - $ beet info -i 'title,mb*' beatles + $ beet info -i 'title,mb_artistid' beatles -Will only show the ``title`` property and all properties starting with -``mb``. You can add the ``-i`` option multiple times to the command -line. +Will only show the ``title`` and ``mb_artistid`` properties. You can add the +``-i`` option multiple times to the command line. Additional command-line options include: diff --git a/test/test_info.py b/test/test_info.py index 814205527..0f29621d6 100644 --- a/test/test_info.py +++ b/test/test_info.py @@ -92,17 +92,6 @@ class InfoTest(unittest.TestCase, TestHelper): self.assertIn(u'title: [various]', out) self.remove_mediafile_fixtures() - def test_include_pattern(self): - item, = self.add_item_fixtures() - item.album = 'xxxx' - item.store() - - out = self.run_with_output('info', '--library', 'album:xxxx', - '--include-keys', '*lbu*') - self.assertIn(displayable_path(item.path), out) - self.assertNotIn(u'title:', out) - self.assertIn(u'album: xxxx', out) - def test_custom_format(self): self.add_item_fixtures() out = self.run_with_output('info', '--library', '--format', From a46c975387a08500af9730556d386c20a788974b Mon Sep 17 00:00:00 2001 From: Jelle Kaufmann Date: Mon, 3 May 2021 20:28:47 +0200 Subject: [PATCH 122/130] Add HTTPS support for Musicbrainz --- beets/autotag/mb.py | 3 ++- beets/config_default.yaml | 1 + docs/changelog.rst | 2 ++ docs/reference/config.rst | 6 ++++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 3ca5463c2..53fe80a3a 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -97,7 +97,8 @@ def configure(): from the beets configuration. This should be called at startup. """ hostname = config['musicbrainz']['host'].as_str() - musicbrainzngs.set_hostname(hostname) + use_https = config['musicbrainz']['use_https'].get(bool) + musicbrainzngs.set_hostname(hostname, use_https) musicbrainzngs.set_rate_limit( config['musicbrainz']['ratelimit_interval'].as_number(), config['musicbrainz']['ratelimit'].get(int), diff --git a/beets/config_default.yaml b/beets/config_default.yaml index dd140675f..06445a687 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -102,6 +102,7 @@ statefile: state.pickle musicbrainz: host: musicbrainz.org + use_https: yes ratelimit: 1 ratelimit_interval: 1.0 searchlimit: 5 diff --git a/docs/changelog.rst b/docs/changelog.rst index 6caed75c6..bc7eb367e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,6 +46,8 @@ Major new features: Other new things: +* Enable HTTPS support to Musicbrainz by default and add configuration option + `use_https`. * :doc:`/plugins/mpdstats`: Add a new `strip_path` option to help build the right local path from MPD information. * :doc:`/plugins/convert`: Conversion can now parallelize conversion jobs on diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 9dd7447a4..8c4b9c373 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -691,15 +691,17 @@ MusicBrainz Options ------------------- You can instruct beets to use `your own MusicBrainz database`_ instead of -the `main server`_. Use the ``host`` and ``ratelimit`` options under a -``musicbrainz:`` header, like so:: +the `main server`_. Use the ``host``, ``use_https`` and ``ratelimit`` options +under a ``musicbrainz:`` header, like so:: musicbrainz: host: localhost:5000 + use_https: no ratelimit: 100 The ``host`` key, of course, controls the Web server hostname (and port, optionally) that will be contacted by beets (default: musicbrainz.org). +The ``use_https`` key makes the client use https instead of http (default: yes). The server must have search indices enabled (see `Building search indexes`_). The ``ratelimit`` option, an integer, controls the number of Web service requests From 449617050dddaf833fb56f92815253a75dd82aff Mon Sep 17 00:00:00 2001 From: Jelle Kaufmann Date: Tue, 4 May 2021 16:07:08 +0200 Subject: [PATCH 123/130] Change behavior to HTTPS by default for musicbrainz.org, but HTTP by default for custom servers --- beets/autotag/mb.py | 7 +++++-- beets/config_default.yaml | 2 +- docs/changelog.rst | 2 +- docs/reference/config.rst | 7 ++++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 53fe80a3a..0334cff1e 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -97,8 +97,11 @@ def configure(): from the beets configuration. This should be called at startup. """ hostname = config['musicbrainz']['host'].as_str() - use_https = config['musicbrainz']['use_https'].get(bool) - musicbrainzngs.set_hostname(hostname, use_https) + https = config['musicbrainz']['https'].get(bool) + # Only call set_hostname when a custom server is configured. Since + # musicbrainz-ngs connects to musicbrainz.org with HTTPS by default + if hostname != "musicbrainz.org": + musicbrainzngs.set_hostname(hostname, https) musicbrainzngs.set_rate_limit( config['musicbrainz']['ratelimit_interval'].as_number(), config['musicbrainz']['ratelimit'].get(int), diff --git a/beets/config_default.yaml b/beets/config_default.yaml index 06445a687..74540891e 100644 --- a/beets/config_default.yaml +++ b/beets/config_default.yaml @@ -102,7 +102,7 @@ statefile: state.pickle musicbrainz: host: musicbrainz.org - use_https: yes + https: no ratelimit: 1 ratelimit_interval: 1.0 searchlimit: 5 diff --git a/docs/changelog.rst b/docs/changelog.rst index bc7eb367e..4e3d1b374 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,7 +47,7 @@ Major new features: Other new things: * Enable HTTPS support to Musicbrainz by default and add configuration option - `use_https`. + `https` for custom servers. * :doc:`/plugins/mpdstats`: Add a new `strip_path` option to help build the right local path from MPD information. * :doc:`/plugins/convert`: Conversion can now parallelize conversion jobs on diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 8c4b9c373..92570e882 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -691,17 +691,18 @@ MusicBrainz Options ------------------- You can instruct beets to use `your own MusicBrainz database`_ instead of -the `main server`_. Use the ``host``, ``use_https`` and ``ratelimit`` options +the `main server`_. Use the ``host``, ``https`` and ``ratelimit`` options under a ``musicbrainz:`` header, like so:: musicbrainz: host: localhost:5000 - use_https: no + https: no ratelimit: 100 The ``host`` key, of course, controls the Web server hostname (and port, optionally) that will be contacted by beets (default: musicbrainz.org). -The ``use_https`` key makes the client use https instead of http (default: yes). +The ``https`` key makes the client use https instead of http, this setting applies +only to custom servers. The official musicbrainz server always uses HTTPS (default: no). The server must have search indices enabled (see `Building search indexes`_). The ``ratelimit`` option, an integer, controls the number of Web service requests From 5460a81a4efa54a54156dd4107de2630db06ac34 Mon Sep 17 00:00:00 2001 From: Jelle Kaufmann <17139405+jellekaufmann@users.noreply.github.com> Date: Tue, 4 May 2021 17:01:25 +0200 Subject: [PATCH 124/130] Update docs/reference/config.rst Co-authored-by: Adrian Sampson --- docs/reference/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 92570e882..455639be0 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -701,8 +701,8 @@ under a ``musicbrainz:`` header, like so:: The ``host`` key, of course, controls the Web server hostname (and port, optionally) that will be contacted by beets (default: musicbrainz.org). -The ``https`` key makes the client use https instead of http, this setting applies -only to custom servers. The official musicbrainz server always uses HTTPS (default: no). +The ``https`` key makes the client use HTTPS instead of HTTP. This setting applies +only to custom servers. The official MusicBrainz server always uses HTTPS. (Default: no.) The server must have search indices enabled (see `Building search indexes`_). The ``ratelimit`` option, an integer, controls the number of Web service requests From 2eae5a8ecc1b151b9c94b9ccbac63ed94315177e Mon Sep 17 00:00:00 2001 From: Jelle Kaufmann <17139405+jellekaufmann@users.noreply.github.com> Date: Tue, 4 May 2021 17:01:52 +0200 Subject: [PATCH 125/130] Update docs/changelog.rst Co-authored-by: Adrian Sampson --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4e3d1b374..72a4b56d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,7 +46,7 @@ Major new features: Other new things: -* Enable HTTPS support to Musicbrainz by default and add configuration option +* Enable HTTPS for MusicBrainz by default and add configuration option `https` for custom servers. * :doc:`/plugins/mpdstats`: Add a new `strip_path` option to help build the right local path from MPD information. From 533559136e04bed7ce72077baf2fbf5e41705463 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Fri, 7 May 2021 22:04:46 +1000 Subject: [PATCH 126/130] Add 7z file support #3906 --- beets/importer.py | 6 ++++++ docs/changelog.rst | 4 ++++ setup.py | 1 + test/rsrc/archive.7z | Bin 0 -> 2395 bytes test/test_importer.py | 6 ++++++ 5 files changed, 17 insertions(+) create mode 100644 test/rsrc/archive.7z diff --git a/beets/importer.py b/beets/importer.py index b7bfdb156..38d2a4e62 100644 --- a/beets/importer.py +++ b/beets/importer.py @@ -1054,6 +1054,12 @@ class ArchiveImportTask(SentinelImportTask): pass else: cls._handlers.append((is_rarfile, RarFile)) + try: + from py7zr import is_7zfile, SevenZipFile + except ImportError: + pass + else: + cls._handlers.append((is_7zfile, SevenZipFile)) return cls._handlers diff --git a/docs/changelog.rst b/docs/changelog.rst index 6caed75c6..b215ae7b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -214,6 +214,10 @@ Other new things: ``check_on_import`` config option. * :doc:`/plugins/export`: big speedups when `--include-keys` option is used Thanks to :user:`ssssam`. +* Added 7z support via the `py7zr`_ library + Thanks to :user:`arogl`. :bug:`3906` + + .. _py7zr: https://github.com/miurahr/py7zr Fixes: diff --git a/setup.py b/setup.py index 8191d3888..84b391a78 100755 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ setup( 'responses>=0.3.0', 'requests_oauthlib', 'reflink', + 'py7zr', ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] diff --git a/test/rsrc/archive.7z b/test/rsrc/archive.7z new file mode 100644 index 0000000000000000000000000000000000000000..6a09fdf4b55e37f8466560276c8b09f6677b1732 GIT binary patch literal 2395 zcmV-h38eNndc3bE8~_A(X&9*C2mk;800015000000001X=6vJe3(E-ET>vJO0bdy< z9yfc~+#&B#G)7haKFjA8t)kzPI#V|s-nvDovM?#0e!A9F{}F-We1616Gvy;Wd~aQQCdJe2w7KJ|y!ldcMn>B{mWK7k zwy&ks+CBv-g>Dq}p!GD4bp)nzYfJ&hTY&@}h}f&SSYquk72pTb zJ_Of~k5I$!z0YQoN;|bH_{&vI+o1St5VBdL*B(1n%JH#KR%G$K*AQ~Aj{)Kj<+yZ& zS|Acg8R-y86cA;^#qb-CR1L|>Y9Y1rf4zKmx94sl0(FXKY4g?3iLURss?qjn400NV zKLSO4WB)j+&+K-xI)HC2VS&!bwdYo>vyVBwIno{}5pTOyVBv7r8yp{Q`{*b6RH_(B zKl!{Wc)o<*T{J*?*I-0!Zy5n^4!kwAOjT!=jh`$o1-|=ik+7CuB#!}JMOI!$thWzDV>rkR#%S8?pz;F|oh zE#nK~bG{HFMv(rSQpYcdDy+kX(*NIN75&MV^0_mU1OSL!)>*9R@-m8HY}Z{VW1&aM zFER61lB+f=9|9X~InBNf+JUA4B>h~4?)4?KIpk3xK-RU1lSsIzlRU!U> zXYTTdx@R4Vr~p8YEkgtSRw#rWx)&qmADbFkU1MB z6N4Z5w~fLN1=^Ymtil3)hMWQB>dpSw-1&q6CoU`L5t?ypu}HQo#c!dE@jJN=1E_fsczu}48|TP(_f8Ea zeqgNsmcRnC?@uJWg7HBEphYT1c9Ne!PKsLxPqstZjosY|*jG~S#m@X?h`wl|FTw@f z({v(=iAN;(U-DXAfgr%TA!H~#v~%@<|2C)_wT2R&lszU#MIH=7&I?-6ofl=}2>G>0 zOu5TwUARlr!R1YkM7Zx!*^ru>PdwX6%gLkd@%w<`^GXKXgTbGVmx_A)lHDka4Ha(< zlzK(6y1$YwM#h7D&}#KU;ii+wMAj; z%dv)v4kj8#Y zE?V1hod5zxsVmVW%#0r%<;ca>27^&X5`KwO?9d0z^&!^Z)jI0|@{j!(jd*c2dTP<6 zL7Z8nAu)X#BT}w`xx>{@2DJvl0sCxg^i7Tw69ys!1F$x>&2{q`Eju-*W`}_>Jk?2U zKmFI+qdQ1ZrDL#v%GL$s(VW=xewTBgK^zY+Wr01oe+R!~+G1}exyq~wSyi@Xudb!e zL+uJirLG`Bk$?M1X*wK!t1x0eOL8HaZdD@HaW#dRf)Ha@;x-~7rj2(&OnudZl~k9L zkdycftBJd2q@$J~|K^<#=orPga`f97Q!1QV9C`eXMRU9p9U1gYWZOz<`>8Qyu{uP| znMJ}yj;NInRZ8%B=R*gIjo+;A)v7c&cQbDe8BDLX<`GRHHbh?85G#10yQD$NV{=3%6Xl_?Wo?bXdk9ijJ3=3$`4%OfRWB$@{6jQyFjUb~``*uK-k)jz?Y z0F^pqXv@KcgB3boVOv2qAI^2{9d88zkD&)gp_Y8^_e)ub78 z%nh2&>PvLspgmc31H^s9B5dwe-5HqVuaxp*?R!+>Tm|ZJ&oG863Pl zx>!4iLb71=2MzZVjzd`wHH;?s`(n#VEM{;&txnb*bd$#j6QX>mG1HOa$xl$;A@OA` zaWT#OR#Z3S3#M?bzi9Zbl)G4A&r6)90JloWa~_Qp3sM{N8Ce~jjD2xTzv1(v5TN_- zFn8XQ3xOnk`}ZI)G~E&)r*m|Z=sKhPg3cdVbf7Pa!((ef`o7-HMWNFIT=!FD6ph$v zhY?$it-Mybz1;%Y@LUAHXVLrH4i0hH-oe%Bs`O1oAC#3)=t571d_`#qUB(&zBQs0C z8iMY?iO3po69E}XOGKJ)*!79$l2I)c?nuQmJsL1WQ^O7T|6F*`_gl2l837HMRJ^lm zz-pz)KyQs6<%CCAF|7sEXWt9BGa2U7)y=qaNx1)(-w8aUyVRQ%ASE1Hkc!swxAzXD{ zV6avQO_wl=*Wq`J+cp~reZqIf7Q(v#0R#pB0SSoV00#>J00AK(0RRk(%m4@q0d3K< zaR2}X0T~Ja000000000001*=aW&m{nYyfNkE&y!+Z~$Wf000>P6bb Date: Fri, 7 May 2021 22:30:14 +1000 Subject: [PATCH 127/130] fix up setup --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 84b391a78..8765d4b52 100755 --- a/setup.py +++ b/setup.py @@ -122,7 +122,6 @@ setup( 'responses>=0.3.0', 'requests_oauthlib', 'reflink', - 'py7zr', ] + ( # Tests for the thumbnails plugin need pathlib on Python 2 too. ['pathlib'] if (sys.version_info < (3, 4, 0)) else [] @@ -131,7 +130,9 @@ setup( ] + [ 'discogs-client' if (sys.version_info < (3, 0, 0)) else 'python3-discogs-client' - ], + ] + ( + ['py7zr'] if (sys.version_info > (3, 5, 0)) else [] + ), 'lint': [ 'flake8', 'flake8-coding', From 9ce93faa22cda8c5aba62dc655ead0c06857f215 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Fri, 7 May 2021 22:33:11 +1000 Subject: [PATCH 128/130] Flake8 setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8765d4b52..ca0dc819d 100755 --- a/setup.py +++ b/setup.py @@ -131,7 +131,7 @@ setup( 'discogs-client' if (sys.version_info < (3, 0, 0)) else 'python3-discogs-client' ] + ( - ['py7zr'] if (sys.version_info > (3, 5, 0)) else [] + ['py7zr'] if (sys.version_info > (3, 5, 0)) else [] ), 'lint': [ 'flake8', From cff4ed59bfedacf92960ac8b6fd969d0030d8d77 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sat, 8 May 2021 15:33:58 +1000 Subject: [PATCH 129/130] Update documentation --- docs/changelog.rst | 2 +- docs/reference/cli.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b215ae7b2..c81173209 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -217,7 +217,7 @@ Other new things: * Added 7z support via the `py7zr`_ library Thanks to :user:`arogl`. :bug:`3906` - .. _py7zr: https://github.com/miurahr/py7zr + .. _py7zr: https://pypi.org/project/py7zr/ Fixes: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 5d2b834b7..f7b68c4a8 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -67,7 +67,7 @@ albums (the latter case is true of typical Artist/Album organizations and many people's "downloads" folders). The path can also be a single song or an archive. Beets supports `zip` and `tar` archives out of the box. To extract `rar` files, install the `rarfile`_ package and the -`unrar` command. +`unrar` command. To extract `7z` files, install the `py7zr`_ package. Optional command flags: @@ -152,6 +152,7 @@ Optional command flags: beet import --set genre="Alternative Rock" --set mood="emotional" .. _rarfile: https://pypi.python.org/pypi/rarfile/2.2 +.. _py7zr: https://pypi.org/project/py7zr/ .. only:: html From a28099a35110b37dd1f65f042008b0a0e403da81 Mon Sep 17 00:00:00 2001 From: Andrew Rogl Date: Sat, 8 May 2021 19:18:11 +1000 Subject: [PATCH 130/130] Remove reference to rarfile version in link --- docs/changelog.rst | 1 + docs/reference/cli.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 72a4b56d5..6d99dbc89 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -359,6 +359,7 @@ Fixes: :bug:`2984` * :doc:`/plugins/lyrics`: Fix crashes for Tekstowo false positives :bug:`3904` +* :doc`/reference/cli`: Remove reference to rarfile version in link For plugin developers: diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 5d2b834b7..d450b299a 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -151,7 +151,7 @@ Optional command flags: beet import --set genre="Alternative Rock" --set mood="emotional" -.. _rarfile: https://pypi.python.org/pypi/rarfile/2.2 +.. _rarfile: https://pypi.python.org/pypi/rarfile/ .. only:: html