From 133c21a9c5157d90f3dfaa5863dce91092aa2ff6 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Sun, 18 Jan 2015 14:52:53 +0100 Subject: [PATCH 01/10] Fetchart: add fetching artwork from Wikipedia following the new class-based structure of the plugin --- beetsplug/fetchart.py | 75 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index d86d942ee..196be4b2d 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -158,6 +158,77 @@ class ITunesStore(ArtSource): self._log.debug(u'album not found in iTunes Store') +class Wikipedia(ArtSource): + # Art from Wikipedia (queried through DBpedia) + DBPEDIA_URL = 'http://dbpedia.org/sparql' + WIKIPEDIA_URL = 'http://en.wikipedia.org/w/api.php' + SPARQL_QUERY = '''PREFIX rdf: + PREFIX dbpprop: + PREFIX owl: + PREFIX rdfs: + SELECT DISTINCT ?coverFilename WHERE {{ + ?subject dbpprop:name ?name . + ?subject rdfs:label ?label . + {{ ?subject dbpprop:artist ?artist }} + UNION + {{ ?subject owl:artist ?artist }} + {{ ?artist rdfs:label "{artist}"@en }} + UNION + {{ ?artist dbpprop:name "{artist}"@en }} + ?subject rdf:type . + ?subject dbpprop:cover ?coverFilename . + FILTER ( regex(?name, "{album}", "i") ) + }} + Limit 1''' + + def get(self, album): + if not (album.albumartist and album.album): + return + + # Find the name of the cover art filename on DBpedia + cover_filename = None + dbpedia_response = requests.get( + self.DBPEDIA_URL, + params={ + 'format': 'application/sparql-results+json', + 'timeout': 2500, + 'query': self.SPARQL_QUERY.format(artist=album.albumartist, + album=album.album) + }, headers={'content-type': 'application/json'}) + try: + data = dbpedia_response.json() + results = data['results']['bindings'] + if results: + cover_filename = results[0]['coverFilename']['value'] + else: + self._log.debug(u'album not found on dbpedia') + except: + self._log.debug(u'error scraping dbpedia album page') + + # Ensure we have a filename before attempting to query wikipedia + if not cover_filename: + return + + # Find the absolute url of the cover art on Wikipedia + wikipedia_response = requests.get(self.WIKIPEDIA_URL, params={ + 'format': 'json', + 'action': 'query', + 'continue': '', + 'prop': 'imageinfo', + 'iiprop': 'url', + 'titles': ('File:' + cover_filename).encode('utf-8')}, + headers={'content-type': 'application/json'}) + try: + data = wikipedia_response.json() + results = data['query']['pages'] + for _, result in results.iteritems(): + image_url = result['imageinfo'][0]['url'] + yield image_url + except: + self._log.debug(u'error scraping wikipedia imageinfo') + return + + class FileSystem(ArtSource): """Art from the filesystem""" @staticmethod @@ -203,7 +274,8 @@ class FileSystem(ArtSource): # Try each source in turn. -SOURCES_ALL = [u'coverart', u'itunes', u'amazon', u'albumart', u'google'] +SOURCES_ALL = [u'coverart', u'itunes', u'amazon', u'albumart', u'google', + u'wikipedia'] ART_FUNCS = { u'coverart': CoverArtArchive, @@ -211,6 +283,7 @@ ART_FUNCS = { u'albumart': AlbumArtOrg, u'amazon': Amazon, u'google': GoogleImages, + u'wikipedia': Wikipedia, } # PLUGIN LOGIC ############################################################### From 1a799bb77fb10e3ede5da82f9a49a91411e724e8 Mon Sep 17 00:00:00 2001 From: Tom Jaspers Date: Sun, 18 Jan 2015 15:13:27 +0100 Subject: [PATCH 02/10] Fetchart: update documentation to reflect that Wikipedia is now also used as a source --- docs/plugins/fetchart.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 47aeabdc9..74057ce35 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -47,7 +47,7 @@ file. The available options are: found in the filesystem. - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. - Default: ``coverart itunes albumart amazon google``, i.e., all sources + Default: ``coverart itunes albumart amazon google wikipedia``, i.e., all sources Here's an example that makes plugin select only images that contain *front* or *back* keywords in their filenames and prioritizes the iTunes source over @@ -97,7 +97,7 @@ Album Art Sources By default, this plugin searches for art in the local filesystem as well as on the Cover Art Archive, the iTunes Store, Amazon, AlbumArt.org, -and Google Image Search, in that order. You can reorder the sources or remove +and Google Image Search, and Wikipedia, in that order. You can reorder the sources or remove some to speed up the process using the ``sources`` configuration option. When looking for local album art, beets checks for image files located in the From c86a5f9d97673e7979838a673062efbbfef88c08 Mon Sep 17 00:00:00 2001 From: Heinz Wiesinger Date: Sat, 17 Jan 2015 15:04:09 +0100 Subject: [PATCH 03/10] Make tracktotal an item-level field. This fixes tracktotal being stored incorrectly for multi-disc releases where the individual discs have a different number of tracks and per_disc_numbering is enabled. --- beets/library.py | 24 ++++++++++++++++++++++-- beetsplug/missing.py | 4 ++-- docs/changelog.rst | 15 +++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/beets/library.py b/beets/library.py index 9b3e4a238..e7a46e403 100644 --- a/beets/library.py +++ b/beets/library.py @@ -728,7 +728,6 @@ class Album(LibModel): 'year': types.PaddedInt(4), 'month': types.PaddedInt(2), 'day': types.PaddedInt(2), - 'tracktotal': types.PaddedInt(2), 'disctotal': types.PaddedInt(2), 'comp': types.BOOLEAN, 'mb_albumid': types.STRING, @@ -767,7 +766,6 @@ class Album(LibModel): 'year', 'month', 'day', - 'tracktotal', 'disctotal', 'comp', 'mb_albumid', @@ -797,6 +795,7 @@ class Album(LibModel): # the album's directory as `path`. getters = plugins.album_field_getters() getters['path'] = Album.item_dir + getters['albumtotal'] = Album.tracktotal return getters def items(self): @@ -884,6 +883,27 @@ class Album(LibModel): raise ValueError('empty album') return os.path.dirname(item.path) + def tracktotal(self): + """Return the total number of tracks on all discs on the album + """ + if self.disctotal == 1 or not beets.config['per_disc_numbering']: + return self.items()[0].tracktotal + + counted = [] + total = 0 + + for item in self.items(): + if item.disc in counted: + continue + + total += item.tracktotal + counted.append(item.disc) + + if len(counted) == self.disctotal: + break + + return total + def art_destination(self, image, item_dir=None): """Returns a path to the destination for the album art image for the album. `image` is the path of the image that will be diff --git a/beetsplug/missing.py b/beetsplug/missing.py index a27be65d1..cfcb3a958 100644 --- a/beetsplug/missing.py +++ b/beetsplug/missing.py @@ -23,7 +23,7 @@ from beets.ui import decargs, print_obj, Subcommand def _missing_count(album): """Return number of missing items in `album`. """ - return (album.tracktotal or 0) - len(album.items()) + return (album.albumtotal or 0) - len(album.items()) def _item(track_info, album_info, album_id): @@ -139,7 +139,7 @@ class MissingPlugin(BeetsPlugin): """ item_mbids = map(lambda x: x.mb_trackid, album.items()) - if len([i for i in album.items()]) < album.tracktotal: + if len([i for i in album.items()]) < album.albumtotal: # fetch missing items # TODO: Implement caching that without breaking other stuff album_info = hooks.album_for_mbid(album.mb_albumid) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3f82ff6f0..9ac4361bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,21 @@ Features: search results you wish to see when looking up releases at MusicBrainz during import. :bug:`1245` +Core improvements: + +* The ``tracktotal`` attribute is now a *track-level field* instead of an + album-level one. This field stores the total number of tracks on the + album, or if the ``per_disc_numbering`` config option is set, the total + number of tracks on a particular medium. With the latter option the + album-level incarnation of this field could not represent releases where + the total number of tracks differs per medium ---for example 20 tracks + on medium 1 and 21 tracks on medium 2. Now, tracktotal is correctly + handled also when ``per_disc_numbering`` is set. +* Complimentary to the change for ``tracktotal`` there is now an album-level + ``albumtotal`` attribute. This field always provides the total numbers of + tracks on the album. The ``per_disc_numbering`` config option has no + influence on this field. + Fixes: * :doc:`/plugins/lyrics`: Silence a warning about insecure requests in the new From b001a973a3dca5fc22932e583a4a1088f8eb74f7 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Fri, 23 Jan 2015 17:38:40 +0100 Subject: [PATCH 04/10] Improve embedart logging management - better logging levels - always use the preferred user formatting for items & albums - improved some messages wording Fix #1244 --- beetsplug/embedart.py | 91 ++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 48 deletions(-) diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 9ff4bf165..3192e8c04 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -19,7 +19,6 @@ import subprocess import platform from tempfile import NamedTemporaryFile -from beets import logging from beets.plugins import BeetsPlugin from beets import mediafile from beets import ui @@ -27,6 +26,12 @@ from beets.ui import decargs from beets.util import syspath, normpath, displayable_path from beets.util.artresizer import ArtResizer from beets import config +from beets.util.functemplate import Template + +__item_template = Template(ui._pick_format(False)) +fmt_item = lambda item: item.evaluate_template(__item_template) +__album_template = Template(ui._pick_format(True)) +fmt_album = lambda item: item.evaluate_template(__album_template) class EmbedCoverArtPlugin(BeetsPlugin): @@ -43,13 +48,13 @@ class EmbedCoverArtPlugin(BeetsPlugin): if self.config['maxwidth'].get(int) and not ArtResizer.shared.local: self.config['maxwidth'] = 0 - self._log.warn(u"ImageMagick or PIL not found; " - u"'maxwidth' option ignored") + self._log.warning(u"ImageMagick or PIL not found; " + u"'maxwidth' option ignored") if self.config['compare_threshold'].get(int) and not \ ArtResizer.shared.can_compare: self.config['compare_threshold'] = 0 - self._log.warn(u"ImageMagick 6.8.7 or higher not installed; " - u"'compare_threshold' option ignored") + self._log.warning(u"ImageMagick 6.8.7 or higher not installed; " + u"'compare_threshold' option ignored") self.register_listener('album_imported', self.album_imported) @@ -90,7 +95,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): def extract_func(lib, opts, args): outpath = normpath(opts.outpath or 'cover') item = lib.items(decargs(args)).get() - self.extract(outpath, item) + if item: + self.extract(outpath, item) extract_cmd.func = extract_func # Clear command. @@ -117,15 +123,10 @@ class EmbedCoverArtPlugin(BeetsPlugin): if compare_threshold: if not self.check_art_similarity(item, imagepath, compare_threshold): - self._log.warn(u'Image not similar; skipping.') + self._log.debug(u'Image not similar; skipping.') return - if ifempty: - art = self.get_art(item) - if not art: - pass - else: - self._log.debug(u'media file contained art already {0}', - displayable_path(imagepath)) + if ifempty and self.get_art(item): + self._log.debug(u'media file already contained art') return if maxwidth and not as_album: imagepath = self.resize_image(imagepath, maxwidth) @@ -134,7 +135,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): self._log.debug(u'embedding {0}', displayable_path(imagepath)) item['images'] = [self._mediafile_image(imagepath, maxwidth)] except IOError as exc: - self._log.error(u'could not read image file: {0}', exc) + self._log.warning(u'could not read image file: {0}', exc) else: # We don't want to store the image in the database. item.try_write(itempath) @@ -145,20 +146,16 @@ class EmbedCoverArtPlugin(BeetsPlugin): """ imagepath = album.artpath if not imagepath: - self._log.info(u'No album art present: {0} - {1}', - album.albumartist, album.album) + self._log.info(u'No album art present for {0}', fmt_album(album)) return if not os.path.isfile(syspath(imagepath)): - self._log.error(u'Album art not found at {0}', - displayable_path(imagepath)) + self._log.info(u'Album art not found at {0} for {1}', + displayable_path(imagepath), fmt_album(album)) return if maxwidth: imagepath = self.resize_image(imagepath, maxwidth) - self._log.log( - logging.DEBUG if quiet else logging.INFO, - u'Embedding album art into {0.albumartist} - {0.album}.', album - ) + self._log.info(u'Embedding album art into {0}', fmt_album(album)) for item in album.items(): thresh = self.config['compare_threshold'].get(int) @@ -169,7 +166,7 @@ class EmbedCoverArtPlugin(BeetsPlugin): def resize_image(self, imagepath, maxwidth): """Returns path to an image resized to maxwidth. """ - self._log.info(u'Resizing album art to {0} pixels wide', maxwidth) + self._log.debug(u'Resizing album art to {0} pixels wide', maxwidth) imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath)) return imagepath @@ -217,9 +214,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): out_str) return - self._log.info(u'compare PHASH score is {0}', phash_diff) - if phash_diff > compare_threshold: - return False + self._log.debug(u'compare PHASH score is {0}', phash_diff) + return phash_diff <= compare_threshold return True @@ -236,8 +232,8 @@ class EmbedCoverArtPlugin(BeetsPlugin): try: mf = mediafile.MediaFile(syspath(item.path)) except mediafile.UnreadableFileError as exc: - self._log.error(u'Could not extract art from {0}: {1}', - displayable_path(item.path), exc) + self._log.warning(u'Could not extract art from {0}: {1}', + displayable_path(item.path), exc) return return mf.art @@ -245,41 +241,40 @@ class EmbedCoverArtPlugin(BeetsPlugin): # 'extractart' command. def extract(self, outpath, item): - if not item: - self._log.error(u'No item matches query.') - return - art = self.get_art(item) if not art: - self._log.error(u'No album art present in {0} - {1}.', - item.artist, item.title) + self._log.info(u'No album art present in {0}, skipping.', + fmt_item(item)) return # Add an extension to the filename. ext = imghdr.what(None, h=art) if not ext: - self._log.error(u'Unknown image type.') + self._log.warning(u'Unknown image type in {0}.', + displayable_path(item.path)) return outpath += '.' + ext - self._log.info(u'Extracting album art from: {0.artist} - {0.title} ' - u'to: {1}', item, displayable_path(outpath)) + self._log.info(u'Extracting album art from: {0} to: {1}', + fmt_item(item), displayable_path(outpath)) with open(syspath(outpath), 'wb') as f: f.write(art) return outpath # 'clearart' command. def clear(self, lib, query): - self._log.info(u'Clearing album art from items:') - for item in lib.items(query): - self._log.info(u'{0} - {1}', item.artist, item.title) + id3v23 = config['id3v23'].get(bool) + + items = lib.items(query) + self._log.info(u'Clearing album art from {0} items', len(items)) + for item in items: + self._log.debug(u'Clearing art for {0}', fmt_item(item)) try: - mf = mediafile.MediaFile(syspath(item.path), - config['id3v23'].get(bool)) + mf = mediafile.MediaFile(syspath(item.path), id3v23) except mediafile.UnreadableFileError as exc: - self._log.error(u'Could not clear art from {0}: {1}', - displayable_path(item.path), exc) - continue - del mf.art - mf.save() + self._log.warning(u'Could not read file {0}: {1}', + displayable_path(item.path), exc) + else: + del mf.art + mf.save() From 77d46bb2df577cc4d76f3c3c593dc41fcb909c2d Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 25 Jan 2015 18:33:20 +0100 Subject: [PATCH 05/10] Embedart logging: higher level for img comparisons --- beetsplug/embedart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/embedart.py b/beetsplug/embedart.py index 6f6daa96f..efb32e0fa 100644 --- a/beetsplug/embedart.py +++ b/beetsplug/embedart.py @@ -123,10 +123,10 @@ class EmbedCoverArtPlugin(BeetsPlugin): if compare_threshold: if not self.check_art_similarity(item, imagepath, compare_threshold): - self._log.debug(u'Image not similar; skipping.') + self._log.info(u'Image not similar; skipping.') return if ifempty and self.get_art(item): - self._log.debug(u'media file already contained art') + self._log.info(u'media file already contained art') return if maxwidth and not as_album: imagepath = self.resize_image(imagepath, maxwidth) From bd29ab21e18b164073239e81c82e8f73e43945e7 Mon Sep 17 00:00:00 2001 From: Bruno Cauet Date: Sun, 25 Jan 2015 21:38:29 +0100 Subject: [PATCH 06/10] Delete outdated disabled test in test_library.py TemplateTest.album_fields_override_item_values() never ran because of its name (missing 'test_' prefix). When run it now fails for it targets outdated functionality. --- test/test_library.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/test_library.py b/test/test_library.py index 717cc1aa1..58e24fd75 100644 --- a/test/test_library.py +++ b/test/test_library.py @@ -1067,15 +1067,6 @@ class ImportTimeTest(_common.TestCase): class TemplateTest(_common.LibTestCase): - def album_fields_override_item_values(self): - self.album = self.lib.add_album([self.i]) - self.album.albumartist = 'album-level' - self.album.store() - self.i.albumartist = 'track-level' - self.i.store() - self.assertEqual(self.i.evaluate_template('$albumartist'), - 'album-level') - def test_year_formatted_in_template(self): self.i.year = 123 self.i.store() From b5c4edaaf5689a7ad701faf8c5be344d123b18f7 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 25 Jan 2015 13:00:58 -0800 Subject: [PATCH 07/10] Changelog for Wikipedia fetchart backend (#1194) --- docs/changelog.rst | 2 ++ docs/plugins/fetchart.rst | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 96264ef6e..828ef3b0a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ Features: by default. :bug:`1246` * :doc:`/plugins/fetchart`: Names of extracted image art is taken from the ``art_filename`` configuration option. :bug:`1258` +* :doc:`/plugins/fetchart`: There's a new Wikipedia image source that uses + DBpedia to find albums. Thanks to Tom Jaspers. :bug:`1194` Fixes: diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 74057ce35..36266871b 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -47,7 +47,8 @@ file. The available options are: found in the filesystem. - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. - Default: ``coverart itunes albumart amazon google wikipedia``, i.e., all sources + Default: ``coverart itunes albumart amazon google wikipedia``, i.e., + all sources. Here's an example that makes plugin select only images that contain *front* or *back* keywords in their filenames and prioritizes the iTunes source over From e0cc68cf078358d47ffd5561d5e64fcb01b8e745 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 25 Jan 2015 13:03:28 -0800 Subject: [PATCH 08/10] Tiny renaming for #1233 --- beets/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beets/library.py b/beets/library.py index e7a46e403..d1c62b7e7 100644 --- a/beets/library.py +++ b/beets/library.py @@ -795,7 +795,7 @@ class Album(LibModel): # the album's directory as `path`. getters = plugins.album_field_getters() getters['path'] = Album.item_dir - getters['albumtotal'] = Album.tracktotal + getters['albumtotal'] = Album._tracktotal return getters def items(self): @@ -883,7 +883,7 @@ class Album(LibModel): raise ValueError('empty album') return os.path.dirname(item.path) - def tracktotal(self): + def _albumtotal(self): """Return the total number of tracks on all discs on the album """ if self.disctotal == 1 or not beets.config['per_disc_numbering']: From 2e083f0a8c7b477f47539fa10c18797b4799726d Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 25 Jan 2015 13:12:21 -0800 Subject: [PATCH 09/10] Changelog wording --- docs/changelog.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cb1abe442..c10a64981 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,9 @@ Features: * A new :doc:`/plugins/filefilter` lets you write regular expressions to automatically avoid importing certain files. Thanks to :user:`mried`. :bug:`1186` -* Stop on invalid queries instead of ignoring the invalid part. +* When there's a parse error in a query (for example, when you type a + malformed date in a :ref:`date query `), beets now stops with an + error instead of silently ignoring the query component. * A new :ref:`searchlimit` configuration option allows you to specify how many search results you wish to see when looking up releases at MusicBrainz during import. :bug:`1245` @@ -27,20 +29,19 @@ Features: * :doc:`/plugins/fetchart`: There's a new Wikipedia image source that uses DBpedia to find albums. Thanks to Tom Jaspers. :bug:`1194` -Core improvements: +Core changes: * The ``tracktotal`` attribute is now a *track-level field* instead of an album-level one. This field stores the total number of tracks on the - album, or if the ``per_disc_numbering`` config option is set, the total - number of tracks on a particular medium. With the latter option the - album-level incarnation of this field could not represent releases where - the total number of tracks differs per medium ---for example 20 tracks - on medium 1 and 21 tracks on medium 2. Now, tracktotal is correctly - handled also when ``per_disc_numbering`` is set. -* Complimentary to the change for ``tracktotal`` there is now an album-level - ``albumtotal`` attribute. This field always provides the total numbers of - tracks on the album. The ``per_disc_numbering`` config option has no - influence on this field. + album, or if the :ref:`per_disc_numbering` config option is set, the total + number of tracks on a particular medium (i.e., disc). The field was causing + problems with that :ref:`per_disc_numbering` mode: different discs on the + same album needed different track totals. The field can now work correctly + in either mode. +* To replace ``tracktotal`` as an album-level field, there is a new + ``albumtotal`` computed attribute that provides the total number of tracks + on the album. (The :ref:`per_disc_numbering` option has no influence on this + field.) Fixes: From f2ed7b23737e78229bb6400dba6a50afb0f2f25e Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 25 Jan 2015 13:18:26 -0800 Subject: [PATCH 10/10] Fix dumb naming mistake in e0cc68c --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index d1c62b7e7..2890ee901 100644 --- a/beets/library.py +++ b/beets/library.py @@ -795,7 +795,7 @@ class Album(LibModel): # the album's directory as `path`. getters = plugins.album_field_getters() getters['path'] = Album.item_dir - getters['albumtotal'] = Album._tracktotal + getters['albumtotal'] = Album._albumtotal return getters def items(self):