Merge branch 'master' into libmodels-formatting

Conflicts:
	beetsplug/embedart.py

beets/library.py

@@ -748,7 +748,6 @@ class Album(LibModel):
         'year':               types.PaddedInt(4),
         'month':              types.PaddedInt(2),
         'day':                types.PaddedInt(2),
-        'tracktotal':         types.PaddedInt(2),
         'disctotal':          types.PaddedInt(2),
         'comp':               types.BOOLEAN,
         'mb_albumid':         types.STRING,
@@ -787,7 +786,6 @@ class Album(LibModel):
         'year',
         'month',
         'day',
-        'tracktotal',
         'disctotal',
         'comp',
         'mb_albumid',
@@ -819,6 +817,7 @@ class Album(LibModel):
         # the album's directory as `path`.
         getters = plugins.album_field_getters()
         getters['path'] = Album.item_dir
+        getters['albumtotal'] = Album._albumtotal
         return getters
 
     def items(self):
@@ -906,6 +905,27 @@ class Album(LibModel):
             raise ValueError('empty album')
         return os.path.dirname(item.path)
 
+    def _albumtotal(self):
+        """Return the total number of tracks on all discs on the album
+        """
+        if self.disctotal == 1 or not beets.config['per_disc_numbering']:
+            return self.items()[0].tracktotal
+
+        counted = []
+        total = 0
+
+        for item in self.items():
+            if item.disc in counted:
+                continue
+
+            total += item.tracktotal
+            counted.append(item.disc)
+
+            if len(counted) == self.disctotal:
+                break
+
+        return total
+
     def art_destination(self, image, item_dir=None):
         """Returns a path to the destination for the album art image
         for the album. `image` is the path of the image that will be

beetsplug/embedart.py

@@ -19,7 +19,6 @@ import subprocess
 import platform
 from tempfile import NamedTemporaryFile
 
-from beets import logging
 from beets.plugins import BeetsPlugin
 from beets import mediafile
 from beets import ui
@@ -43,12 +42,12 @@ class EmbedCoverArtPlugin(BeetsPlugin):
 
         if self.config['maxwidth'].get(int) and not ArtResizer.shared.local:
             self.config['maxwidth'] = 0
-            self._log.warn(u"ImageMagick or PIL not found; "
+            self._log.warning(u"ImageMagick or PIL not found; "
                               u"'maxwidth' option ignored")
         if self.config['compare_threshold'].get(int) and not \
                 ArtResizer.shared.can_compare:
             self.config['compare_threshold'] = 0
-            self._log.warn(u"ImageMagick 6.8.7 or higher not installed; "
+            self._log.warning(u"ImageMagick 6.8.7 or higher not installed; "
                               u"'compare_threshold' option ignored")
 
         self.register_listener('album_imported', self.album_imported)
@@ -118,15 +117,10 @@ class EmbedCoverArtPlugin(BeetsPlugin):
         if compare_threshold:
             if not self.check_art_similarity(item, imagepath,
                                              compare_threshold):
-                self._log.warn(u'Image not similar; skipping.')
+                self._log.info(u'Image not similar; skipping.')
                 return
-        if ifempty:
+        if ifempty and self.get_art(item):
-            art = self.get_art(item)
+                self._log.info(u'media file already contained art')
-            if not art:
-                pass
-            else:
-                self._log.debug(u'media file contained art already {0}',
-                                displayable_path(imagepath))
                 return
         if maxwidth and not as_album:
             imagepath = self.resize_image(imagepath, maxwidth)
@@ -135,7 +129,7 @@ class EmbedCoverArtPlugin(BeetsPlugin):
             self._log.debug(u'embedding {0}', displayable_path(imagepath))
             item['images'] = [self._mediafile_image(imagepath, maxwidth)]
         except IOError as exc:
-            self._log.error(u'could not read image file: {0}', exc)
+            self._log.warning(u'could not read image file: {0}', exc)
         else:
             # We don't want to store the image in the database.
             item.try_write(itempath)
@@ -146,19 +140,16 @@ class EmbedCoverArtPlugin(BeetsPlugin):
         """
         imagepath = album.artpath
         if not imagepath:
-            self._log.info(u'No album art present: {0}', album)
+            self._log.info(u'No album art present for {0}', album)
             return
         if not os.path.isfile(syspath(imagepath)):
-            self._log.error(u'Album art not found at {0}',
+            self._log.info(u'Album art not found at {0} for {1}',
-                            displayable_path(imagepath))
+                           displayable_path(imagepath), album)
             return
         if maxwidth:
             imagepath = self.resize_image(imagepath, maxwidth)
 
-        self._log.log(
+        self._log.info(u'Embedding album art into {0}', album)
-            logging.DEBUG if quiet else logging.INFO,
-            u'Embedding album art into {0}.', album
-        )
 
         for item in album.items():
             thresh = self.config['compare_threshold'].get(int)
@@ -169,7 +160,7 @@ class EmbedCoverArtPlugin(BeetsPlugin):
     def resize_image(self, imagepath, maxwidth):
         """Returns path to an image resized to maxwidth.
         """
-        self._log.info(u'Resizing album art to {0} pixels wide', maxwidth)
+        self._log.debug(u'Resizing album art to {0} pixels wide', maxwidth)
         imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath))
         return imagepath
 
@@ -217,9 +208,8 @@ class EmbedCoverArtPlugin(BeetsPlugin):
                                     out_str)
                     return
 
-                self._log.info(u'compare PHASH score is {0}', phash_diff)
+                self._log.debug(u'compare PHASH score is {0}', phash_diff)
-                if phash_diff > compare_threshold:
+                return phash_diff <= compare_threshold
-                    return False
 
         return True
 
@@ -236,7 +226,7 @@ class EmbedCoverArtPlugin(BeetsPlugin):
         try:
             mf = mediafile.MediaFile(syspath(item.path))
         except mediafile.UnreadableFileError as exc:
-            self._log.error(u'Could not extract art from {0}: {1}',
+            self._log.warning(u'Could not extract art from {0}: {1}',
                               displayable_path(item.path), exc)
             return
 
@@ -245,20 +235,17 @@ class EmbedCoverArtPlugin(BeetsPlugin):
     # 'extractart' command.
 
     def extract(self, outpath, item):
-        if not item:
-            self._log.error(u'No item matches query.')
-            return
-
         art = self.get_art(item)
 
         if not art:
-            self._log.error(u'No album art present in {0}.', item)
+            self._log.info(u'No album art present in {0}, skipping.', item)
             return
 
         # Add an extension to the filename.
         ext = imghdr.what(None, h=art)
         if not ext:
-            self._log.error(u'Unknown image type.')
+            self._log.warning(u'Unknown image type in {0}.',
+                              displayable_path(item.path))
             return
         outpath += '.' + ext
 
@@ -270,15 +257,17 @@ class EmbedCoverArtPlugin(BeetsPlugin):
 
     # 'clearart' command.
     def clear(self, lib, query):
-        self._log.info(u'Clearing album art from items:')
+        id3v23 = config['id3v23'].get(bool)
-        for item in lib.items(query):
+
-            self._log.info(u'{0}', item)
+        items = lib.items(query)
+        self._log.info(u'Clearing album art from {0} items', len(items))
+        for item in items:
+            self._log.debug(u'Clearing art for {0}', item)
             try:
-                mf = mediafile.MediaFile(syspath(item.path),
+                mf = mediafile.MediaFile(syspath(item.path), id3v23)
-                                         config['id3v23'].get(bool))
             except mediafile.UnreadableFileError as exc:
-                self._log.error(u'Could not clear art from {0}: {1}',
+                self._log.warning(u'Could not read file {0}: {1}',
                                   displayable_path(item.path), exc)
-                continue
+            else:
                 del mf.art
                 mf.save()

beetsplug/fetchart.py

@@ -158,6 +158,77 @@ class ITunesStore(ArtSource):
             self._log.debug(u'album not found in iTunes Store')
 
 
+class Wikipedia(ArtSource):
+    # Art from Wikipedia (queried through DBpedia)
+    DBPEDIA_URL = 'http://dbpedia.org/sparql'
+    WIKIPEDIA_URL = 'http://en.wikipedia.org/w/api.php'
+    SPARQL_QUERY = '''PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+                 PREFIX dbpprop: <http://dbpedia.org/property/>
+                 PREFIX owl: <http://dbpedia.org/ontology/>
+                 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+                 SELECT DISTINCT ?coverFilename WHERE {{
+                   ?subject dbpprop:name ?name .
+                   ?subject rdfs:label ?label .
+                   {{ ?subject dbpprop:artist ?artist }}
+                     UNION
+                   {{ ?subject owl:artist ?artist }}
+                   {{ ?artist rdfs:label "{artist}"@en }}
+                     UNION
+                   {{ ?artist dbpprop:name "{artist}"@en }}
+                   ?subject rdf:type <http://dbpedia.org/ontology/Album> .
+                   ?subject dbpprop:cover ?coverFilename .
+                   FILTER ( regex(?name, "{album}", "i") )
+                  }}
+                 Limit 1'''
+
+    def get(self, album):
+        if not (album.albumartist and album.album):
+            return
+
+        # Find the name of the cover art filename on DBpedia
+        cover_filename = None
+        dbpedia_response = requests.get(
+            self.DBPEDIA_URL,
+            params={
+                'format': 'application/sparql-results+json',
+                'timeout': 2500,
+                'query': self.SPARQL_QUERY.format(artist=album.albumartist,
+                                                  album=album.album)
+            }, headers={'content-type': 'application/json'})
+        try:
+            data = dbpedia_response.json()
+            results = data['results']['bindings']
+            if results:
+                cover_filename = results[0]['coverFilename']['value']
+            else:
+                self._log.debug(u'album not found on dbpedia')
+        except:
+            self._log.debug(u'error scraping dbpedia album page')
+
+        # Ensure we have a filename before attempting to query wikipedia
+        if not cover_filename:
+            return
+
+        # Find the absolute url of the cover art on Wikipedia
+        wikipedia_response = requests.get(self.WIKIPEDIA_URL, params={
+            'format': 'json',
+            'action': 'query',
+            'continue': '',
+            'prop': 'imageinfo',
+            'iiprop': 'url',
+            'titles': ('File:' + cover_filename).encode('utf-8')},
+            headers={'content-type': 'application/json'})
+        try:
+            data = wikipedia_response.json()
+            results = data['query']['pages']
+            for _, result in results.iteritems():
+                image_url = result['imageinfo'][0]['url']
+                yield image_url
+        except:
+            self._log.debug(u'error scraping wikipedia imageinfo')
+            return
+
+
 class FileSystem(ArtSource):
     """Art from the filesystem"""
     @staticmethod
@@ -203,7 +274,8 @@ class FileSystem(ArtSource):
 
 # Try each source in turn.
 
-SOURCES_ALL = [u'coverart', u'itunes', u'amazon', u'albumart', u'google']
+SOURCES_ALL = [u'coverart', u'itunes', u'amazon', u'albumart', u'google',
+               u'wikipedia']
 
 ART_FUNCS = {
     u'coverart': CoverArtArchive,
@@ -211,6 +283,7 @@ ART_FUNCS = {
     u'albumart': AlbumArtOrg,
     u'amazon': Amazon,
     u'google': GoogleImages,
+    u'wikipedia': Wikipedia,
 }
 
 # PLUGIN LOGIC ###############################################################

beetsplug/missing.py

@@ -23,7 +23,7 @@ from beets.ui import decargs, print_obj, Subcommand
 def _missing_count(album):
     """Return number of missing items in `album`.
     """
-    return (album.tracktotal or 0) - len(album.items())
+    return (album.albumtotal or 0) - len(album.items())
 
 
 def _item(track_info, album_info, album_id):
@@ -139,7 +139,7 @@ class MissingPlugin(BeetsPlugin):
         """
         item_mbids = map(lambda x: x.mb_trackid, album.items())
 
-        if len([i for i in album.items()]) < album.tracktotal:
+        if len([i for i in album.items()]) < album.albumtotal:
             # fetch missing items
             # TODO: Implement caching that without breaking other stuff
             album_info = hooks.album_for_mbid(album.mb_albumid)

docs/changelog.rst

@@ -9,7 +9,9 @@ Features:
 * A new :doc:`/plugins/filefilter` lets you write regular expressions to
   automatically avoid importing certain files. Thanks to :user:`mried`.
   :bug:`1186`
-* Stop on invalid queries instead of ignoring the invalid part.
+* When there's a parse error in a query (for example, when you type a
+  malformed date in a :ref:`date query <datequery>`), beets now stops with an
+  error instead of silently ignoring the query component.
 * A new :ref:`searchlimit` configuration option allows you to specify how many
   search results you wish to see when looking up releases at MusicBrainz
   during import. :bug:`1245`
@@ -24,6 +26,22 @@ Features:
   by default. :bug:`1246`
 * :doc:`/plugins/fetchart`: Names of extracted image art is taken from the
   ``art_filename`` configuration option. :bug:`1258`
+* :doc:`/plugins/fetchart`: There's a new Wikipedia image source that uses
+  DBpedia to find albums. Thanks to Tom Jaspers. :bug:`1194`
+
+Core changes:
+
+* The ``tracktotal`` attribute is now a *track-level field* instead of an
+  album-level one. This field stores the total number of tracks on the
+  album, or if the :ref:`per_disc_numbering` config option is set, the total
+  number of tracks on a particular medium (i.e., disc). The field was causing
+  problems with that :ref:`per_disc_numbering` mode: different discs on the
+  same album needed different track totals. The field can now work correctly
+  in either mode.
+* To replace ``tracktotal`` as an album-level field, there is a new
+  ``albumtotal`` computed attribute that provides the total number of tracks
+  on the album. (The :ref:`per_disc_numbering` option has no influence on this
+  field.)
 
 Fixes:
 

docs/plugins/fetchart.rst

@@ -47,7 +47,8 @@ file. The available options are:
   found in the filesystem.
 - **sources**: List of sources to search for images. An asterisk `*` expands
   to all available sources.
-  Default: ``coverart itunes albumart amazon google``, i.e., all sources
+  Default: ``coverart itunes albumart amazon google wikipedia``, i.e.,
+  all sources.
 
 Here's an example that makes plugin select only images that contain *front* or
 *back* keywords in their filenames and prioritizes the iTunes source over
@@ -97,7 +98,7 @@ Album Art Sources
 
 By default, this plugin searches for art in the local filesystem as well as on
 the Cover Art Archive, the iTunes Store, Amazon, AlbumArt.org,
-and Google Image Search, in that order. You can reorder the sources or remove
+and Google Image Search, and Wikipedia, in that order. You can reorder the sources or remove
 some to speed up the process using the ``sources`` configuration option.
 
 When looking for local album art, beets checks for image files located in the

test/test_library.py

@@ -1068,15 +1068,6 @@ class ImportTimeTest(_common.TestCase):
 
 
 class TemplateTest(_common.LibTestCase):
-    def album_fields_override_item_values(self):
-        self.album = self.lib.add_album([self.i])
-        self.album.albumartist = 'album-level'
-        self.album.store()
-        self.i.albumartist = 'track-level'
-        self.i.store()
-        self.assertEqual(self.i.evaluate_template('$albumartist'),
-                         'album-level')
-
     def test_year_formatted_in_template(self):
         self.i.year = 123
         self.i.store()