From c14abe56a5e451cac23eb9a95fdf41b20a76633f Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Mon, 28 Dec 2015 23:16:12 +1030 Subject: [PATCH 01/12] fetchart: Add new google backend using google custom search Requires an API key from google similar to lyrics plugin. Limited to 100 queries/day. --- beetsplug/fetchart.py | 50 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index a56f9f95a..89e6a04e4 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -96,8 +96,9 @@ class RequestMixin(object): # ART SOURCES ################################################################ class ArtSource(RequestMixin): - def __init__(self, log): + def __init__(self, log, config): self._log = log + self._config = config def get(self, album): raise NotImplementedError() @@ -157,6 +158,43 @@ class AlbumArtOrg(ArtSource): self._log.debug(u'no image found on page') +class GoogleImages(ArtSource): + URL = u'https://www.googleapis.com/customsearch/v1' + + def get(self, album): + """Return art URL from google custom search engine + given an album title and interpreter. + """ + self._log.debug('fetching art from google...') + api_key = self._config['google_api_key'].get() + engine_id = self._config['google_engine_ID'].get() + self._log.debug('API Key: ' + api_key) + self._log.debug('Engine ID: ' + engine_id) + if not api_key: + self._log.debug(u'google api key not set') + return + if not (album.albumartist and album.album): + return + search_string = (album.albumartist + ',' + album.album).encode('utf-8') + response = self.request(self.URL, params={ + 'key': api_key, + 'cx': engine_id, + 'q': search_string, + 'searchType': 'image' + }) + + # Get results using JSON. + data = response.json() + if 'error' in data: + reason = data['error']['errors'][0]['reason'] + self._log.debug(u'google fetchart error: {0}', reason) + return + + if 'items' in data.keys(): + for item in data['items']: + yield item['link'] + + class ITunesStore(ArtSource): # Art from the iTunes Store. def get(self, album): @@ -361,7 +399,7 @@ class FileSystem(ArtSource): # Try each source in turn. SOURCES_ALL = [u'coverart', u'itunes', u'amazon', u'albumart', - u'wikipedia'] + u'wikipedia', u'google'] ART_SOURCES = { u'coverart': CoverArtArchive, @@ -369,6 +407,7 @@ ART_SOURCES = { u'albumart': AlbumArtOrg, u'amazon': Amazon, u'wikipedia': Wikipedia, + u'google': GoogleImages, } # PLUGIN LOGIC ############################################################### @@ -387,7 +426,10 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'cautious': False, 'cover_names': ['cover', 'front', 'art', 'album', 'folder'], 'sources': ['coverart', 'itunes', 'amazon', 'albumart'], + 'google_API_key': None, + 'google_engine_ID': u'001442825323518660753:hrh5ch1gjzm', }) + self.config['google_API_key'].redact = True # Holds paths to downloaded images between fetching them and # placing them in the filesystem. @@ -407,8 +449,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): available_sources.remove(u'itunes') sources_name = plugins.sanitize_choices( self.config['sources'].as_str_seq(), available_sources) - self.sources = [ART_SOURCES[s](self._log) for s in sources_name] - self.fs_source = FileSystem(self._log) + self.sources = [ART_SOURCES[s](self._log, self.config) for s in sources_name] + self.fs_source = FileSystem(self._log, self.config) # Asynchronous; after music is added to the library. def fetch_art(self, session, task): From ff15f4af9b5e8ac5ad3d17c036e3568d70c942c4 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Mon, 28 Dec 2015 23:42:11 +1030 Subject: [PATCH 02/12] fetchart: Use config singleton rather than passing object to each backend --- beetsplug/fetchart.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 89e6a04e4..9d05b55c4 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -96,9 +96,8 @@ class RequestMixin(object): # ART SOURCES ################################################################ class ArtSource(RequestMixin): - def __init__(self, log, config): + def __init__(self, log): self._log = log - self._config = config def get(self, album): raise NotImplementedError() @@ -166,8 +165,8 @@ class GoogleImages(ArtSource): given an album title and interpreter. """ self._log.debug('fetching art from google...') - api_key = self._config['google_api_key'].get() - engine_id = self._config['google_engine_ID'].get() + api_key = config['fetchart']['google_api_key'].get() + engine_id = config['fetchart']['google_engine_ID'].get() self._log.debug('API Key: ' + api_key) self._log.debug('Engine ID: ' + engine_id) if not api_key: @@ -449,8 +448,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): available_sources.remove(u'itunes') sources_name = plugins.sanitize_choices( self.config['sources'].as_str_seq(), available_sources) - self.sources = [ART_SOURCES[s](self._log, self.config) for s in sources_name] - self.fs_source = FileSystem(self._log, self.config) + self.sources = [ART_SOURCES[s](self._log) for s in sources_name] + self.fs_source = FileSystem(self._log) # Asynchronous; after music is added to the library. def fetch_art(self, session, task): From b73cfb9f8d6e302dba65343a1923617351118661 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Mon, 28 Dec 2015 23:53:56 +1030 Subject: [PATCH 03/12] fetchart: Fix typo in google API config key --- beetsplug/fetchart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 9d05b55c4..66273a3df 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -165,7 +165,7 @@ class GoogleImages(ArtSource): given an album title and interpreter. """ self._log.debug('fetching art from google...') - api_key = config['fetchart']['google_api_key'].get() + api_key = config['fetchart']['google_API_key'].get() engine_id = config['fetchart']['google_engine_ID'].get() self._log.debug('API Key: ' + api_key) self._log.debug('Engine ID: ' + engine_id) From deadc4d04964d1b1d71fff132a3aafcfe4ff1bfe Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Mon, 28 Dec 2015 23:57:03 +1030 Subject: [PATCH 04/12] fetchart: Automatically disable google backend if no API key is set --- beetsplug/fetchart.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 66273a3df..297db5d0d 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -169,9 +169,6 @@ class GoogleImages(ArtSource): engine_id = config['fetchart']['google_engine_ID'].get() self._log.debug('API Key: ' + api_key) self._log.debug('Engine ID: ' + engine_id) - if not api_key: - self._log.debug(u'google api key not set') - return if not (album.albumartist and album.album): return search_string = (album.albumartist + ',' + album.album).encode('utf-8') @@ -446,6 +443,9 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): available_sources = list(SOURCES_ALL) if not HAVE_ITUNES and u'itunes' in available_sources: available_sources.remove(u'itunes') + if not self.config['google_API_key'].get() and \ + u'google' in available_sources: + available_sources.remove(u'google') sources_name = plugins.sanitize_choices( self.config['sources'].as_str_seq(), available_sources) self.sources = [ART_SOURCES[s](self._log) for s in sources_name] From 99fb656f52b128dccd300e1683b97012a879686f Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Mon, 28 Dec 2015 23:57:38 +1030 Subject: [PATCH 05/12] fetchart: Remove some debug logging from google backend --- beetsplug/fetchart.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 297db5d0d..7aab88567 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -164,11 +164,8 @@ class GoogleImages(ArtSource): """Return art URL from google custom search engine given an album title and interpreter. """ - self._log.debug('fetching art from google...') api_key = config['fetchart']['google_API_key'].get() engine_id = config['fetchart']['google_engine_ID'].get() - self._log.debug('API Key: ' + api_key) - self._log.debug('Engine ID: ' + engine_id) if not (album.albumartist and album.album): return search_string = (album.albumartist + ',' + album.album).encode('utf-8') From 03a41e82da06e1d4abeec5705dee573a580d2817 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Tue, 29 Dec 2015 00:02:41 +1030 Subject: [PATCH 06/12] fetchart: minor google backend refactor --- beetsplug/fetchart.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 7aab88567..9d971a3fc 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -164,14 +164,12 @@ class GoogleImages(ArtSource): """Return art URL from google custom search engine given an album title and interpreter. """ - api_key = config['fetchart']['google_API_key'].get() - engine_id = config['fetchart']['google_engine_ID'].get() if not (album.albumartist and album.album): return search_string = (album.albumartist + ',' + album.album).encode('utf-8') response = self.request(self.URL, params={ - 'key': api_key, - 'cx': engine_id, + 'key': config['fetchart']['google_API_key'].get(), + 'cx': config['fetchart']['google_engine_ID'].get(), 'q': search_string, 'searchType': 'image' }) From 2e10b8c284c7577cff7542633d1d5db2dda05e02 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Tue, 29 Dec 2015 01:37:53 +1030 Subject: [PATCH 07/12] fetchart: Pass config object to backends when initialized --- beetsplug/fetchart.py | 11 ++++++----- test/test_art.py | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 9d971a3fc..392db9e47 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -96,8 +96,9 @@ class RequestMixin(object): # ART SOURCES ################################################################ class ArtSource(RequestMixin): - def __init__(self, log): + def __init__(self, log, config): self._log = log + self._config = config def get(self, album): raise NotImplementedError() @@ -168,8 +169,8 @@ class GoogleImages(ArtSource): return search_string = (album.albumartist + ',' + album.album).encode('utf-8') response = self.request(self.URL, params={ - 'key': config['fetchart']['google_API_key'].get(), - 'cx': config['fetchart']['google_engine_ID'].get(), + 'key': self._config['google_API_key'].get(), + 'cx': self._config['google_engine_ID'].get(), 'q': search_string, 'searchType': 'image' }) @@ -443,8 +444,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): available_sources.remove(u'google') sources_name = plugins.sanitize_choices( self.config['sources'].as_str_seq(), available_sources) - self.sources = [ART_SOURCES[s](self._log) for s in sources_name] - self.fs_source = FileSystem(self._log) + self.sources = [ART_SOURCES[s](self._log, self.config) for s in sources_name] + self.fs_source = FileSystem(self._log, self.config) # Asynchronous; after music is added to the library. def fetch_art(self, session, task): diff --git a/test/test_art.py b/test/test_art.py index cb29f3769..324c3fbb0 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -65,13 +65,13 @@ class FetchImageTest(UseThePlugin): self.assertNotEqual(artpath, None) -class FSArtTest(_common.TestCase): +class FSArtTest(UseThePlugin): def setUp(self): super(FSArtTest, self).setUp() self.dpath = os.path.join(self.temp_dir, 'arttest') os.mkdir(self.dpath) - self.source = fetchart.FileSystem(logger) + self.source = fetchart.FileSystem(logger, self.plugin.config) def test_finds_jpg_in_directory(self): _common.touch(os.path.join(self.dpath, 'a.jpg')) @@ -190,13 +190,13 @@ class CombinedTest(UseThePlugin): self.assertEqual(len(responses.calls), 0) -class AAOTest(_common.TestCase): +class AAOTest(UseThePlugin): ASIN = 'xxxx' AAO_URL = 'http://www.albumart.org/index_detail.php?asin={0}'.format(ASIN) def setUp(self): super(AAOTest, self).setUp() - self.source = fetchart.AlbumArtOrg(logger) + self.source = fetchart.AlbumArtOrg(logger, self.plugin.config) @responses.activate def run(self, *args, **kwargs): From be64df640911da41678b968eceedd6bc755b8f4a Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Tue, 29 Dec 2015 01:48:33 +1030 Subject: [PATCH 08/12] fetchart: Handle errors when parsing malformed JSON from google --- beetsplug/fetchart.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 392db9e47..7267f2e49 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -176,7 +176,12 @@ class GoogleImages(ArtSource): }) # Get results using JSON. - data = response.json() + try: + data = response.json() + except ValueError: + self._log.debug(u'google: error loading response: {}'.format(response.text)) + return + if 'error' in data: reason = data['error']['errors'][0]['reason'] self._log.debug(u'google fetchart error: {0}', reason) From f7b05729a3bcaeff2b14858a2338a9f6811226f5 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Tue, 29 Dec 2015 01:49:07 +1030 Subject: [PATCH 09/12] fetchart: Add tests for google backend --- test/test_art.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/test_art.py b/test/test_art.py index 324c3fbb0..e2ba9b89b 100644 --- a/test/test_art.py +++ b/test/test_art.py @@ -226,6 +226,41 @@ class AAOTest(UseThePlugin): self.assertEqual(list(res), []) +class GoogleImageTest(UseThePlugin): + def setUp(self): + super(GoogleImageTest, self).setUp() + self.source = fetchart.GoogleImages(logger, self.plugin.config) + + @responses.activate + def run(self, *args, **kwargs): + super(GoogleImageTest, self).run(*args, **kwargs) + + def mock_response(self, url, json): + responses.add(responses.GET, url, body=json, + content_type='application/json') + + def test_google_art_finds_image(self): + album = _common.Bag(albumartist="some artist", album="some album") + json = b'{"items": [{"link": "url_to_the_image"}]}' + self.mock_response(fetchart.GoogleImages.URL, json) + result_url = self.source.get(album) + self.assertEqual(list(result_url)[0], 'url_to_the_image') + + def test_google_art_returns_no_result_when_error_received(self): + album = _common.Bag(albumartist="some artist", album="some album") + json = b'{"error": {"errors": [{"reason": "some reason"}]}}' + self.mock_response(fetchart.GoogleImages.URL, json) + result_url = self.source.get(album) + self.assertEqual(list(result_url), []) + + def test_google_art_returns_no_result_with_malformed_response(self): + album = _common.Bag(albumartist="some artist", album="some album") + json = b"""bla blup""" + self.mock_response(fetchart.GoogleImages.URL, json) + result_url = self.source.get(album) + self.assertEqual(list(result_url), []) + + class ArtImporterTest(UseThePlugin): def setUp(self): super(ArtImporterTest, self).setUp() From 7b273b431005f113d42202cb88277d94569a9307 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Tue, 29 Dec 2015 17:29:17 +1030 Subject: [PATCH 10/12] fetchart: PEP8 fixes --- beetsplug/fetchart.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 7267f2e49..34690f275 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -179,7 +179,8 @@ class GoogleImages(ArtSource): try: data = response.json() except ValueError: - self._log.debug(u'google: error loading response: {}'.format(response.text)) + self._log.debug(u'google: error loading response: {}' + .format(response.text)) return if 'error' in data: @@ -449,7 +450,8 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): available_sources.remove(u'google') sources_name = plugins.sanitize_choices( self.config['sources'].as_str_seq(), available_sources) - self.sources = [ART_SOURCES[s](self._log, self.config) for s in sources_name] + self.sources = [ART_SOURCES[s](self._log, self.config) + for s in sources_name] self.fs_source = FileSystem(self._log, self.config) # Asynchronous; after music is added to the library. From e771f34dfd177503cc3ec099ea96ef735317d6a1 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Tue, 29 Dec 2015 17:31:40 +1030 Subject: [PATCH 11/12] fetchart: Add documentation for new google backend --- docs/plugins/fetchart.rst | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 80149d478..61cd259d0 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -50,12 +50,18 @@ file. The available options are: - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. Default: ``coverart itunes amazon albumart``, i.e., everything but - ``wikipedia``. Enable those two sources for more matches at + ``wikipedia`` and ``google``. Enable those two sources for more matches at the cost of some speed. +- **google_API_key**: Your Google API key (to enable the Google Custom Search + backend). + Default: None. +- **google_engine_ID**: The custom search engine to use. + Default: The `beets custom search engine`_, which searches the entire web. Note: ``minwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ or `Pillow`_. +.. _beets custom search engine: https://cse.google.com.au:443/cse/publicurl?cx=001442825323518660753:hrh5ch1gjzm .. _Pillow: https://github.com/python-pillow/Pillow .. _ImageMagick: http://www.imagemagick.org/ @@ -137,6 +143,24 @@ Once the library is installed, the plugin will use it to search automatically. .. _python-itunes: https://github.com/ocelma/python-itunes .. _pip: http://pip.openplans.org/ +Google custom search +'''''''''''''''''''' + +To use the google image search backend you need to +`register for a Google API key`_. Set the ``google_API_key`` configuration +option to your key, then add ``google`` to the list of sources in your +configuration. + +.. _register for a Google API key: https://code.google.com/apis/console. + +Optionally, you can `define a custom search engine`_. Get your search engine's +token and use it for your ``google_engine_ID`` configuration option. The +default engine searches the entire web for cover art. + +.. _define a custom search engine: http://www.google.com/cse/all + +Note that the Google custom search API is limited to 100 queries per day. +After that, the fetchart plugin will fall back on other declared data sources. Embedding Album Art ------------------- From 0d16f764f2e9fc32f11c02f10d7eddab12fb0919 Mon Sep 17 00:00:00 2001 From: Lachlan Charlick Date: Wed, 30 Dec 2015 13:34:58 +1030 Subject: [PATCH 12/12] fetchart: Tidier google backend configuration keys --- beetsplug/fetchart.py | 12 ++++++------ docs/plugins/fetchart.rst | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/beetsplug/fetchart.py b/beetsplug/fetchart.py index 34690f275..b1ce11409 100644 --- a/beetsplug/fetchart.py +++ b/beetsplug/fetchart.py @@ -169,8 +169,8 @@ class GoogleImages(ArtSource): return search_string = (album.albumartist + ',' + album.album).encode('utf-8') response = self.request(self.URL, params={ - 'key': self._config['google_API_key'].get(), - 'cx': self._config['google_engine_ID'].get(), + 'key': self._config['google_key'].get(), + 'cx': self._config['google_engine'].get(), 'q': search_string, 'searchType': 'image' }) @@ -424,10 +424,10 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): 'cautious': False, 'cover_names': ['cover', 'front', 'art', 'album', 'folder'], 'sources': ['coverart', 'itunes', 'amazon', 'albumart'], - 'google_API_key': None, - 'google_engine_ID': u'001442825323518660753:hrh5ch1gjzm', + 'google_key': None, + 'google_engine': u'001442825323518660753:hrh5ch1gjzm', }) - self.config['google_API_key'].redact = True + self.config['google_key'].redact = True # Holds paths to downloaded images between fetching them and # placing them in the filesystem. @@ -445,7 +445,7 @@ class FetchArtPlugin(plugins.BeetsPlugin, RequestMixin): available_sources = list(SOURCES_ALL) if not HAVE_ITUNES and u'itunes' in available_sources: available_sources.remove(u'itunes') - if not self.config['google_API_key'].get() and \ + if not self.config['google_key'].get() and \ u'google' in available_sources: available_sources.remove(u'google') sources_name = plugins.sanitize_choices( diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index 61cd259d0..e53dbc219 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -52,10 +52,10 @@ file. The available options are: Default: ``coverart itunes amazon albumart``, i.e., everything but ``wikipedia`` and ``google``. Enable those two sources for more matches at the cost of some speed. -- **google_API_key**: Your Google API key (to enable the Google Custom Search +- **google_key**: Your Google API key (to enable the Google Custom Search backend). Default: None. -- **google_engine_ID**: The custom search engine to use. +- **google_engine**: The custom search engine to use. Default: The `beets custom search engine`_, which searches the entire web. Note: ``minwidth`` and ``enforce_ratio`` options require either `ImageMagick`_ @@ -147,14 +147,14 @@ Google custom search '''''''''''''''''''' To use the google image search backend you need to -`register for a Google API key`_. Set the ``google_API_key`` configuration +`register for a Google API key`_. Set the ``google_key`` configuration option to your key, then add ``google`` to the list of sources in your configuration. .. _register for a Google API key: https://code.google.com/apis/console. Optionally, you can `define a custom search engine`_. Get your search engine's -token and use it for your ``google_engine_ID`` configuration option. The +token and use it for your ``google_engine`` configuration option. The default engine searches the entire web for cover art. .. _define a custom search engine: http://www.google.com/cse/all