diff --git a/beets/autotag/__init__.py b/beets/autotag/__init__.py index 822bb60ef..4c5f09eb4 100644 --- a/beets/autotag/__init__.py +++ b/beets/autotag/__init__.py @@ -49,6 +49,8 @@ def apply_item_metadata(item, track_info): item.lyricist = track_info.lyricist if track_info.composer is not None: item.composer = track_info.composer + if track_info.composer_sort is not None: + item.composer_sort = track_info.composer_sort if track_info.arranger is not None: item.arranger = track_info.arranger @@ -155,6 +157,8 @@ def apply_metadata(album_info, mapping): item.lyricist = track_info.lyricist if track_info.composer is not None: item.composer = track_info.composer + if track_info.composer_sort is not None: + item.composer_sort = track_info.composer_sort if track_info.arranger is not None: item.arranger = track_info.arranger diff --git a/beets/autotag/hooks.py b/beets/autotag/hooks.py index 3c403fcf4..053d050c6 100644 --- a/beets/autotag/hooks.py +++ b/beets/autotag/hooks.py @@ -144,6 +144,7 @@ class TrackInfo(object): - ``data_url``: The data source release URL. - ``lyricist``: individual track lyricist name - ``composer``: individual track composer name + - ``composer_sort``: individual track composer sort name - ``arranger`: individual track arranger name - ``track_alt``: alternative track number (tape, vinyl, etc.) @@ -155,8 +156,8 @@ class TrackInfo(object): length=None, index=None, medium=None, medium_index=None, medium_total=None, artist_sort=None, disctitle=None, artist_credit=None, data_source=None, data_url=None, - media=None, lyricist=None, composer=None, arranger=None, - track_alt=None): + media=None, lyricist=None, composer=None, composer_sort=None, + arranger=None, track_alt=None): self.title = title self.track_id = track_id self.artist = artist @@ -174,6 +175,7 @@ class TrackInfo(object): self.data_url = data_url self.lyricist = lyricist self.composer = composer + self.composer_sort = composer_sort self.arranger = arranger self.track_alt = track_alt diff --git a/beets/autotag/mb.py b/beets/autotag/mb.py index 21dd8a715..9da709160 100644 --- a/beets/autotag/mb.py +++ b/beets/autotag/mb.py @@ -207,6 +207,7 @@ def track_info(recording, index=None, medium=None, medium_index=None, lyricist = [] composer = [] + composer_sort = [] for work_relation in recording.get('work-relation-list', ()): if work_relation['type'] != 'performance': continue @@ -218,10 +219,13 @@ def track_info(recording, index=None, medium=None, medium_index=None, lyricist.append(artist_relation['artist']['name']) elif type == 'composer': composer.append(artist_relation['artist']['name']) + composer_sort.append( + artist_relation['artist']['sort-name']) if lyricist: info.lyricist = u', '.join(lyricist) if composer: info.composer = u', '.join(composer) + info.composer_sort = u', '.join(composer_sort) arranger = [] for artist_relation in recording.get('artist-relation-list', ()): diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 89ee7aef7..51790b9fa 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -47,7 +47,7 @@ class InvalidQueryError(ParsingError): super(InvalidQueryError, self).__init__(message) -class InvalidQueryArgumentTypeError(ParsingError): +class InvalidQueryArgumentValueError(ParsingError): """Represent a query argument that could not be converted as expected. It exists to be caught in upper stack levels so a meaningful (i.e. with the @@ -57,7 +57,7 @@ class InvalidQueryArgumentTypeError(ParsingError): message = u"'{0}' is not {1}".format(what, expected) if detail: message = u"{0}: {1}".format(message, detail) - super(InvalidQueryArgumentTypeError, self).__init__(message) + super(InvalidQueryArgumentValueError, self).__init__(message) class Query(object): @@ -211,9 +211,9 @@ class RegexpQuery(StringFieldQuery): self.pattern = re.compile(self.pattern) except re.error as exc: # Invalid regular expression. - raise InvalidQueryArgumentTypeError(pattern, - u"a regular expression", - format(exc)) + raise InvalidQueryArgumentValueError(pattern, + u"a regular expression", + format(exc)) @staticmethod def _normalize(s): @@ -285,7 +285,7 @@ class NumericQuery(FieldQuery): try: return float(s) except ValueError: - raise InvalidQueryArgumentTypeError(s, u"an int or a float") + raise InvalidQueryArgumentValueError(s, u"an int or a float") def __init__(self, field, pattern, fast=True): super(NumericQuery, self).__init__(field, pattern, fast) @@ -556,7 +556,7 @@ class Period(object): @classmethod def parse(cls, string): """Parse a date and return a `Period` object, or `None` if the - string is empty, or raise an InvalidQueryArgumentTypeError if + string is empty, or raise an InvalidQueryArgumentValueError if the string could not be parsed to a date. """ @@ -575,8 +575,8 @@ class Period(object): return None date, ordinal = find_date_and_format(string) if date is None: - raise InvalidQueryArgumentTypeError(string, - 'a valid datetime string') + raise InvalidQueryArgumentValueError(string, + 'a valid datetime string') precision = cls.precisions[ordinal] return cls(date, precision) @@ -704,7 +704,7 @@ class DurationQuery(NumericQuery): try: return float(s) except ValueError: - raise InvalidQueryArgumentTypeError( + raise InvalidQueryArgumentValueError( s, u"a M:SS string or a float") diff --git a/beets/library.py b/beets/library.py index b263ecd64..4b2b194fc 100644 --- a/beets/library.py +++ b/beets/library.py @@ -417,6 +417,7 @@ class Item(LibModel): 'genre': types.STRING, 'lyricist': types.STRING, 'composer': types.STRING, + 'composer_sort': types.STRING, 'arranger': types.STRING, 'grouping': types.STRING, 'year': types.PaddedInt(4), @@ -1306,7 +1307,7 @@ class Library(dbcore.Database): query, parsed_sort = parse_query_string(query, model_cls) elif isinstance(query, (list, tuple)): query, parsed_sort = parse_query_parts(query, model_cls) - except dbcore.query.InvalidQueryArgumentTypeError as exc: + except dbcore.query.InvalidQueryArgumentValueError as exc: raise dbcore.InvalidQueryError(query, exc) # Any non-null sort specified by the parsed query overrides the diff --git a/beets/mediafile.py b/beets/mediafile.py index 13f1b2dfb..9242ab1f1 100644 --- a/beets/mediafile.py +++ b/beets/mediafile.py @@ -1317,7 +1317,7 @@ class DateField(MediaField): for item in items: try: items_.append(int(item)) - except: + except (TypeError, ValueError): items_.append(None) return items_ @@ -1638,6 +1638,12 @@ class MediaFile(object): StorageStyle('COMPOSER'), ASFStorageStyle('WM/Composer'), ) + composer_sort = MediaField( + MP3StorageStyle('TSOC'), + MP4StorageStyle('soco'), + StorageStyle('COMPOSERSORT'), + ASFStorageStyle('WM/Composersortorder'), + ) arranger = MediaField( MP3PeopleStorageStyle('TIPL', involvement='arranger'), MP4StorageStyle('----:com.apple.iTunes:Arranger'), diff --git a/beets/plugins.py b/beets/plugins.py index 2ecdb8472..d62f3c011 100644 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -264,7 +264,7 @@ def load_plugins(names=()): and obj != BeetsPlugin and obj not in _classes: _classes.add(obj) - except: + except Exception: log.warning( u'** error loading plugin {}:\n{}', name, diff --git a/beets/ui/commands.py b/beets/ui/commands.py old mode 100755 new mode 100644 index 06ab6f0a5..995ff87e9 --- a/beets/ui/commands.py +++ b/beets/ui/commands.py @@ -84,7 +84,7 @@ def _do_query(lib, query, album, also_items=True): def _print_keys(query): """Given a SQLite query result, print the `key` field of each - returned row, with identation of 2 spaces. + returned row, with indentation of 2 spaces. """ for row in query: print_(u' ' * 2 + row['key']) @@ -615,7 +615,7 @@ def choose_candidate(candidates, singleton, rec, cur_artist=None, require = True # Bell ring when user interaction is needed. if config['import']['bell']: - ui.print_('\a', end='') + ui.print_(u'\a', end=u'') sel = ui.input_options((u'Apply', u'More candidates') + choice_opts, require=require, default=default) if sel == u'a': diff --git a/beets/util/bluelet.py b/beets/util/bluelet.py index 48dd7bd94..0da17559b 100644 --- a/beets/util/bluelet.py +++ b/beets/util/bluelet.py @@ -269,7 +269,7 @@ def run(root_coro): except StopIteration: # Thread is done. complete_thread(coro, None) - except: + except BaseException: # Thread raised some other exception. del threads[coro] raise ThreadException(coro, sys.exc_info()) @@ -366,7 +366,7 @@ def run(root_coro): exit_te = te break - except: + except BaseException: # For instance, KeyboardInterrupt during select(). Raise # into root thread and terminate others. threads = {root_coro: ExceptionEvent(sys.exc_info())} diff --git a/beets/util/confit.py b/beets/util/confit.py index 373e05ffc..73ae97abc 100644 --- a/beets/util/confit.py +++ b/beets/util/confit.py @@ -668,7 +668,7 @@ def load_yaml(filename): parsed, a ConfigReadError is raised. """ try: - with open(filename, 'r') as f: + with open(filename, 'rb') as f: return yaml.load(f, Loader=Loader) except (IOError, yaml.error.YAMLError) as exc: raise ConfigReadError(filename, exc) @@ -908,9 +908,10 @@ class Configuration(RootView): default_source = source break if default_source and default_source.filename: - with open(default_source.filename, 'r') as fp: + with open(default_source.filename, 'rb') as fp: default_data = fp.read() - yaml_out = restore_yaml_comments(yaml_out, default_data) + yaml_out = restore_yaml_comments(yaml_out, + default_data.decode('utf8')) return yaml_out diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 51716552c..0e13db4a0 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -325,7 +325,7 @@ class Parser(object): # Common parsing resources. special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE, ESCAPE_CHAR) - special_char_re = re.compile(r'[%s]|$' % + special_char_re = re.compile(r'[%s]|\Z' % u''.join(re.escape(c) for c in special_chars)) escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP) terminator_chars = (GROUP_CLOSE,) @@ -343,8 +343,11 @@ class Parser(object): if self.in_argument: extra_special_chars = (ARG_SEP,) special_char_re = re.compile( - r'[%s]|$' % u''.join(re.escape(c) for c in - self.special_chars + extra_special_chars)) + r'[%s]|\Z' % u''.join( + re.escape(c) for c in + self.special_chars + extra_special_chars + ) + ) text_parts = [] @@ -570,7 +573,7 @@ class Template(object): """ try: res = self.compiled(values, functions) - except: # Handle any exceptions thrown by compiled version. + except Exception: # Handle any exceptions thrown by compiled version. res = self.interpret(values, functions) return res diff --git a/beets/util/pipeline.py b/beets/util/pipeline.py index 367e5d980..39bc7152e 100644 --- a/beets/util/pipeline.py +++ b/beets/util/pipeline.py @@ -270,7 +270,7 @@ class FirstPipelineThread(PipelineThread): return self.out_queue.put(msg) - except: + except BaseException: self.abort_all(sys.exc_info()) return @@ -318,7 +318,7 @@ class MiddlePipelineThread(PipelineThread): return self.out_queue.put(msg) - except: + except BaseException: self.abort_all(sys.exc_info()) return @@ -357,7 +357,7 @@ class LastPipelineThread(PipelineThread): # Send to consumer. self.coro.send(msg) - except: + except BaseException: self.abort_all(sys.exc_info()) return @@ -425,7 +425,7 @@ class Pipeline(object): while threads[-1].is_alive(): threads[-1].join(1) - except: + except BaseException: # Stop all the threads immediately. for thread in threads: thread.abort() diff --git a/beetsplug/duplicates.py b/beetsplug/duplicates.py index 93d53c58a..2f6bba3e6 100644 --- a/beetsplug/duplicates.py +++ b/beetsplug/duplicates.py @@ -185,7 +185,7 @@ class DuplicatesPlugin(BeetsPlugin): if tag: try: k, v = tag.split('=') - except: + except Exception: raise UserError( u"{}: can't parse k=v tag: {}".format(PLUGIN, tag) ) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 6714b2fee..113bed104 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, division, print_function import difflib import itertools import json +import struct import re import requests import unicodedata @@ -53,7 +54,6 @@ from beets import plugins from beets import ui import beets - DIV_RE = re.compile(r'<(/?)div>?', re.I) COMMENT_RE = re.compile(r'', re.S) TAG_RE = re.compile(r'<[^>]*>') @@ -77,6 +77,12 @@ USER_AGENT = 'beets/{}'.format(beets.__version__) # Utilities. +def unichar(i): + try: + return six.unichr(i) + except ValueError: + return struct.pack('i', i).decode('utf-32') + def unescape(text): """Resolve &#xxx; HTML entities (and some others).""" @@ -86,7 +92,7 @@ def unescape(text): def replchar(m): num = m.group(1) - return six.unichr(int(num)) + return unichar(int(num)) out = re.sub(u"&#(\d+);", replchar, out) return out @@ -104,7 +110,6 @@ def extract_text_in(html, starttag): """Extract the text from a
tag in the HTML starting with ``starttag``. Returns None if parsing fails. """ - # Strip off the leading text before opening tag. try: _, html = html.split(starttag, 1) @@ -145,10 +150,10 @@ def search_pairs(item): and featured artists from the strings and add them as candidates. The method also tries to split multiple titles separated with `/`. """ - def generate_alternatives(string, patterns): """Generate string alternatives by extracting first matching group for - each given pattern.""" + each given pattern. + """ alternatives = [string] for pattern in patterns: match = re.search(pattern, string, re.IGNORECASE) @@ -254,16 +259,18 @@ class MusiXmatch(SymbolsReplaced): def fetch(self, artist, title): url = self.build_url(artist, title) + html = self.fetch_url(url) if not html: return - lyrics = extract_text_between(html, - '"body":', '"language":') + html_part = html.split('

', '

') - if not lyrics: - return - for not_found_str in self.NOT_FOUND: - if not_found_str in lyrics: - return - - parts = lyrics.split('\n---\nLyrics powered by', 1) - if parts: - return parts[0] - - def remove_credits(text): """Remove first/last line of text if it contains the word 'lyrics' eg 'Lyrics by songsdatabase.com' @@ -478,6 +454,7 @@ def scrape_lyrics_from_html(html): class Google(Backend): """Fetch lyrics from Google search results.""" + def __init__(self, config, log): super(Google, self).__init__(config, log) self.api_key = config['google_API_key'].as_str() @@ -595,11 +572,10 @@ class Google(Backend): class LyricsPlugin(plugins.BeetsPlugin): - SOURCES = ['google', 'lyricwiki', 'lyrics.com', 'musixmatch'] + SOURCES = ['google', 'lyricwiki', 'musixmatch'] SOURCE_BACKENDS = { 'google': Google, 'lyricwiki': LyricsWiki, - 'lyrics.com': LyricsCom, 'musixmatch': MusiXmatch, 'genius': Genius, } @@ -713,7 +689,8 @@ class LyricsPlugin(plugins.BeetsPlugin): def fetch_item_lyrics(self, lib, item, write, force): """Fetch and store lyrics for a single item. If ``write``, then the - lyrics will also be written to the file itself.""" + lyrics will also be written to the file itself. + """ # Skip if the item already has lyrics. if not force and item.lyrics: self._log.info(u'lyrics already present: {0}', item) diff --git a/beetsplug/thumbnails.py b/beetsplug/thumbnails.py index 838206156..04845e880 100644 --- a/beetsplug/thumbnails.py +++ b/beetsplug/thumbnails.py @@ -274,8 +274,6 @@ class GioURI(URIGetter): try: uri_ptr = self.libgio.g_file_get_uri(g_file_ptr) - except: - raise finally: self.libgio.g_object_unref(g_file_ptr) if not uri_ptr: @@ -285,8 +283,6 @@ class GioURI(URIGetter): try: uri = copy_c_string(uri_ptr) - except: - raise finally: self.libgio.g_free(uri_ptr) diff --git a/beetsplug/web/__init__.py b/beetsplug/web/__init__.py index 3c18ebd5d..290f25b48 100644 --- a/beetsplug/web/__init__.py +++ b/beetsplug/web/__init__.py @@ -25,6 +25,7 @@ from flask import g from werkzeug.routing import BaseConverter, PathConverter import os import json +import base64 # Utilities. @@ -42,6 +43,11 @@ def _rep(obj, expand=False): else: del out['path'] + # Filter all bytes attributes and convert them to strings. + for key, value in out.items(): + if isinstance(out[key], bytes): + out[key] = base64.b64encode(value).decode('ascii') + # Get the size (in bytes) of the backing file. This is useful # for the Tomahawk resolver API. try: diff --git a/docs/changelog.rst b/docs/changelog.rst index 5c0d15e0f..caea9c6a4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -57,6 +57,10 @@ New features: Thanks to :user:`jansol`. :bug:`2488` :bug:`2524` +* A new field, ``composer_sort``, is now supported and fetched from + MusicBrainz. + Thanks to :user:`dosoe`. + :bug:`2519` :bug:`2529` Fixes: @@ -100,6 +104,22 @@ Fixes: ignored. Thanks to :user:`discopatrick`. :bug:`2513` :bug:`2517` * When the SQLite database stops being accessible, we now print a friendly error message. Thanks to :user:`Mary011196`. :bug:`1676` :bug:`2508` +* :doc:`/plugins/web`: Avoid a crash when sending binary data, such as + Chromaprint fingerprints, in music attributes. :bug:`2542` :bug:`2532` +* Fix a hang when parsing templates that end in newlines. :bug:`2562` +* Fix a crash when reading non-ASCII characters in configuration files on + Windows under Python 3. :bug:`2456` :bug:`2565` :bug:`2566` + +Two plugins had backends removed due to bitrot: + +* :doc:`/plugins/lyrics`: The Lyrics.com backend has been removed. (It stopped + working because of changes to the site's URL structure.) + :bug:`2548` :bug:`2549` +* :doc:`/plugins/fetchart`: The documentation no longer recommends iTunes + Store artwork lookup because the unmaintained `python-itunes`_ is broken. + Want to adopt it? :bug:`2371` :bug:`1610` + +.. _python-itunes: https://github.com/ocelma/python-itunes 1.4.3 (January 9, 2017) diff --git a/docs/plugins/beetsweb.png b/docs/plugins/beetsweb.png index c335104eb..dcb5aae17 100644 Binary files a/docs/plugins/beetsweb.png and b/docs/plugins/beetsweb.png differ diff --git a/docs/plugins/fetchart.rst b/docs/plugins/fetchart.rst index ee0014072..97e88c03c 100644 --- a/docs/plugins/fetchart.rst +++ b/docs/plugins/fetchart.rst @@ -49,7 +49,7 @@ file. The available options are: (``enforce_ratio: 0.5%``). Default: ``no``. - **sources**: List of sources to search for images. An asterisk `*` expands to all available sources. - Default: ``filesystem coverart itunes amazon albumart``, i.e., everything but + Default: ``filesystem coverart amazon albumart``, i.e., everything but ``wikipedia``, ``google`` and ``fanarttv``. Enable those sources for more matches at the cost of some speed. They are searched in the given order, thus in the default config, no remote (Web) art source are queried if @@ -82,13 +82,13 @@ or `Pillow`_. .. _ImageMagick: http://www.imagemagick.org/ Here's an example that makes plugin select only images that contain *front* or -*back* keywords in their filenames and prioritizes the iTunes source over +*back* keywords in their filenames and prioritizes the Amazon source over others:: fetchart: cautious: true cover_names: front back - sources: itunes * + sources: amazon * Manually Fetching Album Art @@ -128,7 +128,7 @@ Album Art Sources ----------------- By default, this plugin searches for art in the local filesystem as well as on -the Cover Art Archive, the iTunes Store, Amazon, and AlbumArt.org, in that +the Cover Art Archive, Amazon, and AlbumArt.org, in that order. You can reorder the sources or remove some to speed up the process using the ``sources`` configuration option. @@ -143,22 +143,6 @@ When you choose to apply changes during an import, beets will search for art as described above. For "as-is" imports (and non-autotagged imports using the ``-A`` flag), beets only looks for art on the local filesystem. -iTunes Store -'''''''''''' - -To use the iTunes Store as an art source, install the `python-itunes`_ -library. You can do this using `pip`_, like so:: - - $ pip install https://github.com/ocelma/python-itunes/archive/master.zip - -(There's currently `a problem`_ that prevents a plain ``pip install -python-itunes`` from working.) -Once the library is installed, the plugin will use it to search automatically. - -.. _a problem: https://github.com/ocelma/python-itunes/issues/9 -.. _python-itunes: https://github.com/ocelma/python-itunes -.. _pip: http://www.pip-installer.org/ - Google custom search '''''''''''''''''''' diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 7263304f2..d7c268c7e 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -2,11 +2,10 @@ Lyrics Plugin ============= The ``lyrics`` plugin fetches and stores song lyrics from databases on the Web. -Namely, the current version of the plugin uses `Lyric Wiki`_, `Lyrics.com`_, +Namely, the current version of the plugin uses `Lyric Wiki`_, `Musixmatch`_, `Genius.com`_, and, optionally, the Google custom search API. .. _Lyric Wiki: http://lyrics.wikia.com/ -.. _Lyrics.com: http://www.lyrics.com/ .. _Musixmatch: https://www.musixmatch.com/ .. _Genius.com: http://genius.com/ @@ -60,7 +59,7 @@ configuration file. The available options are: sources known to be scrapeable. - **sources**: List of sources to search for lyrics. An asterisk ``*`` expands to all available sources. - Default: ``google lyricwiki lyrics.com musixmatch``, i.e., all the + Default: ``google lyricwiki musixmatch``, i.e., all the sources except for `genius`. The `google` source will be automatically deactivated if no ``google_API_key`` is setup. diff --git a/test/rsrc/lyricstext.yaml b/test/rsrc/lyricstext.yaml index 814c207df..7ae1a70e7 100644 --- a/test/rsrc/lyricstext.yaml +++ b/test/rsrc/lyricstext.yaml @@ -1,45 +1,56 @@ -Beets_song: - - geeks - - bouquet - - panacea +# Song used by LyricsGooglePluginMachineryTest -Amsterdam: - - oriflammes - - fortune - - batave - - pissent - -Lady_Madonna: - - heaven - - tuesday - - thursday - -Jazz_n_blues: - - parkway - - balance - - impatient - - shoes - -Hey_it_s_ok: - - swear - - forgive - - drink - - found - -City_of_dreams: - - groves - - landmarks - - twilight - - freeways - -Black_magic_woman: - - devil - - magic - - spell - - heart +Beets_song: | + beets is the media library management system for obsessive-compulsive music geeks the purpose of + beets is to get your music collection right once and for all it catalogs your collection + automatically improving its metadata as it goes it then provides a bouquet of tools for + manipulating and accessing your music here's an example of beets' brainy tag corrector doing its + because beets is designed as a library it can do almost anything you can imagine for your + music collection via plugins beets becomes a panacea missing_texts: | Lyricsmania staff is working hard for you to add $TITLE lyrics as soon as they'll be released by $ARTIST, check back soon! In case you have the lyrics to $TITLE and want to send them to us, fill out the following form. + +# Songs lyrics used to test the different sources present in the google custom search engine. +# Text is randomized for copyright infringement reason. + +Amsterdam: | + coup corps coeur invitent mains comme trop morue le hantent mais la dames joli revenir aux + mangent croquer pleine plantent rire de sortent pleins fortune d'amsterdam bruit ruisselants + large poissons braguette leur putains blanches jusque pissent dans soleils dansent et port + bien vertu nez sur chaleur femmes rotant dorment marins boivent bu les que d'un qui je + une cou hambourg plus ils dents ou tournent or berges d'ailleurs tout ciel haubans ce son lueurs + en lune ont mouchent leurs long frottant jusqu'en vous regard montrent langueurs chantent + tordent pleure donnent drames mornes des panse pour un sent encore referment nappes au meurent + geste quand puis alors frites grosses batave expire naissent reboivent oriflammes grave riant a + enfin rance fier y bouffer s'entendre se mieux + +Lady_Madonna: | + feed his money tuesday manage didn't head feet see arrives at in madonna rest morning children + wonder how make thursday your to sunday music papers come tie you has was is listen suitcase + ends friday run that needed breast they child baby mending on lady learned a nun like did wednesday + bed think without afternoon night meet the playing lying + +Jazz_n_blues: | + all shoes money through follow blow til father to his hit jazz kiss now cool bar cause 50 night + heading i'll says yeah cash forgot blues out what for ways away fingers waiting got ever bold + screen sixty throw wait on about last compton days o pick love wall had within jeans jd next + miss standing from it's two long fight extravagant tell today more buy shopping that didn't + what's but russian up can parkway balance my and gone am it as at in check if bags when cross + machine take you drinks coke june wrong coming fancy's i n' impatient so the main's spend + that's + +Hey_it_s_ok: | + and forget be when please it against fighting mama cause ! again what said + things papa hey to much lovers way wet was too do drink and i who forgive + hey fourteen please know not wanted had myself ok friends bed times looked + swear act found the my mean + +Black_magic_woman: | + blind heart sticks just don't into back alone see need yes your out devil make that to black got + you might me woman turning spell stop baby with 'round a on stone messin' magic i of + tricks up leave turn bad so pick she's my can't + diff --git a/test/test_datequery.py b/test/test_datequery.py index b8bb10925..6cd97fd30 100644 --- a/test/test_datequery.py +++ b/test/test_datequery.py @@ -22,7 +22,7 @@ from datetime import datetime import unittest import time from beets.dbcore.query import _parse_periods, DateInterval, DateQuery,\ - InvalidQueryArgumentTypeError + InvalidQueryArgumentValueError def _date(string): @@ -157,11 +157,11 @@ class DateQueryTest(_common.LibTestCase): class DateQueryConstructTest(unittest.TestCase): def test_long_numbers(self): - with self.assertRaises(InvalidQueryArgumentTypeError): + with self.assertRaises(InvalidQueryArgumentValueError): DateQuery('added', '1409830085..1412422089') def test_too_many_components(self): - with self.assertRaises(InvalidQueryArgumentTypeError): + with self.assertRaises(InvalidQueryArgumentValueError): DateQuery('added', '12-34-56-78') def test_invalid_date_query(self): @@ -176,7 +176,7 @@ class DateQueryConstructTest(unittest.TestCase): '..2aa' ] for q in q_list: - with self.assertRaises(InvalidQueryArgumentTypeError): + with self.assertRaises(InvalidQueryArgumentValueError): DateQuery('added', q) def test_datetime_uppercase_t_separator(self): diff --git a/test/test_logging.py b/test/test_logging.py index a6b02a572..826b2447b 100644 --- a/test/test_logging.py +++ b/test/test_logging.py @@ -257,7 +257,7 @@ class ConcurrentEventsTest(TestCase, helper.TestHelper): t2.join(.1) self.assertFalse(t2.is_alive()) - except: + except Exception: print(u"Alive threads:", threading.enumerate()) if dp.lock1.locked(): print(u"Releasing lock1 after exception in test") diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 13ba07fdf..e811da8d7 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -15,21 +15,25 @@ """Tests for the 'lyrics' plugin.""" -from __future__ import division, absolute_import, print_function +from __future__ import absolute_import, division, print_function import os -import sys import re +import six +import sys import unittest +from mock import patch from test import _common -from mock import MagicMock + +from beets import logging +from beets.library import Item +from beets.util import bytestring_path, confit from beetsplug import lyrics -from beets.library import Item -from beets.util import confit, bytestring_path -from beets import logging -import six + +from mock import MagicMock + log = logging.getLogger('beets.test_lyrics') raw_backend = lyrics.Backend({}, log) @@ -37,8 +41,9 @@ google = lyrics.Google(MagicMock(), log) class LyricsPluginTest(unittest.TestCase): + def setUp(self): - """Set up configuration""" + """Set up configuration.""" lyrics.LyricsPlugin() def test_search_artist(self): @@ -194,16 +199,8 @@ def url_to_filename(url): return fn -def check_lyrics_fetched(): - """Return True if lyrics_download_samples.py has been runned and lyrics - pages are present in resources directory""" - lyrics_dirs = len([d for d in os.listdir(LYRICS_ROOT_DIR) if - os.path.isdir(os.path.join(LYRICS_ROOT_DIR, d))]) - # example.com is the only lyrics dir added to repo - return lyrics_dirs > 1 - - class MockFetchUrl(object): + def __init__(self, pathval='fetched_path'): self.pathval = pathval self.fetched = None @@ -217,174 +214,172 @@ class MockFetchUrl(object): def is_lyrics_content_ok(title, text): - """Compare lyrics text to expected lyrics for given title""" - - keywords = LYRICS_TEXTS[google.slugify(title)] - return all(x in text.lower() for x in keywords) + """Compare lyrics text to expected lyrics for given title.""" + if not text: + return + keywords = set(LYRICS_TEXTS[google.slugify(title)].split()) + words = set(x.strip(".?, ") for x in text.lower().split()) + return keywords <= words LYRICS_ROOT_DIR = os.path.join(_common.RSRC, b'lyrics') LYRICS_TEXTS = confit.load_yaml(os.path.join(_common.RSRC, b'lyricstext.yaml')) -DEFAULT_SONG = dict(artist=u'The Beatles', title=u'Lady Madonna') - -DEFAULT_SOURCES = [ - dict(DEFAULT_SONG, url=u'http://lyrics.wikia.com/', - path=u'The_Beatles:Lady_Madonna'), - dict(artist=u'Santana', title=u'Black magic woman', - url='http://www.lyrics.com/', - path=u'black-magic-woman-lyrics-santana.html'), - dict(DEFAULT_SONG, url='https://www.musixmatch.com/', - path=u'lyrics/The-Beatles/Lady-Madonna'), -] - -# Every source entered in default beets google custom search engine -# must be listed below. -# Use default query when possible, or override artist and title fields -# if website don't have lyrics for default query. -GOOGLE_SOURCES = [ - dict(DEFAULT_SONG, - url=u'http://www.absolutelyrics.com', - path=u'/lyrics/view/the_beatles/lady_madonna'), - dict(DEFAULT_SONG, - url=u'http://www.azlyrics.com', - path=u'/lyrics/beatles/ladymadonna.html'), - dict(DEFAULT_SONG, - url=u'http://www.chartlyrics.com', - path=u'/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx'), - dict(DEFAULT_SONG, - url=u'http://www.elyricsworld.com', - path=u'/lady_madonna_lyrics_beatles.html'), - dict(url=u'http://www.lacoccinelle.net', - artist=u'Jacques Brel', title=u"Amsterdam", - path=u'/paroles-officielles/275679.html'), - dict(DEFAULT_SONG, - url=u'http://letras.mus.br/', path=u'the-beatles/275/'), - dict(DEFAULT_SONG, - url='http://www.lyricsmania.com/', - path='lady_madonna_lyrics_the_beatles.html'), - dict(artist=u'Santana', title=u'Black magic woman', - url='http://www.lyrics.com/', - path=u'black-magic-woman-lyrics-santana.html'), - dict(DEFAULT_SONG, url=u'http://lyrics.wikia.com/', - path=u'The_Beatles:Lady_Madonna'), - dict(DEFAULT_SONG, - url=u'http://www.lyrics.net', path=u'/lyric/19110224'), - dict(DEFAULT_SONG, - url=u'http://www.lyricsmode.com', - path=u'/lyrics/b/beatles/lady_madonna.html'), - dict(url=u'http://www.lyricsontop.com', - artist=u'Amy Winehouse', title=u"Jazz'n'blues", - path=u'/amy-winehouse-songs/jazz-n-blues-lyrics.html'), - dict(DEFAULT_SONG, - url='http://www.metrolyrics.com/', - path='lady-madonna-lyrics-beatles.html'), - dict(url='http://www.musica.com/', path='letras.asp?letra=2738', - artist=u'Santana', title=u'Black magic woman'), - dict(DEFAULT_SONG, - url=u'http://www.onelyrics.net/', - artist=u'Ben & Ellen Harper', title=u'City of dreams', - path='ben-ellen-harper-city-of-dreams-lyrics'), - dict(url=u'http://www.paroles.net/', - artist=u'Lilly Wood & the prick', title=u"Hey it's ok", - path=u'lilly-wood-the-prick/paroles-hey-it-s-ok'), - dict(DEFAULT_SONG, - url='http://www.releaselyrics.com', - path=u'/346e/the-beatles-lady-madonna-(love-version)/'), - dict(DEFAULT_SONG, - url=u'http://www.smartlyrics.com', - path=u'/Song18148-The-Beatles-Lady-Madonna-lyrics.aspx'), - dict(DEFAULT_SONG, - url='http://www.songlyrics.com', - path=u'/the-beatles/lady-madonna-lyrics'), - dict(DEFAULT_SONG, - url=u'http://www.stlyrics.com', - path=u'/songs/r/richiehavens48961/ladymadonna2069109.html'), - dict(DEFAULT_SONG, - url=u'http://www.sweetslyrics.com', - path=u'/761696.The%20Beatles%20-%20Lady%20Madonna.html') -] -class LyricsGooglePluginTest(unittest.TestCase): - """Test scraping heuristics on a fake html page. - Or run lyrics_download_samples.py first to check that beets google - custom search engine sources are correctly scraped. - """ - source = dict(url=u'http://www.example.com', artist=u'John Doe', - title=u'Beets song', path=u'/lyrics/beetssong') +class LyricsGoogleBaseTest(unittest.TestCase): def setUp(self): - """Set up configuration""" + """Set up configuration.""" try: __import__('bs4') except ImportError: self.skipTest('Beautiful Soup 4 not available') if sys.version_info[:3] < (2, 7, 3): self.skipTest("Python's built-in HTML parser is not good enough") - lyrics.LyricsPlugin() - raw_backend.fetch_url = MockFetchUrl() + +class LyricsPluginSourcesTest(LyricsGoogleBaseTest): + """Check that beets google custom search engine sources are correctly + scraped. + """ + + DEFAULT_SONG = dict(artist=u'The Beatles', title=u'Lady Madonna') + + DEFAULT_SOURCES = [ + dict(DEFAULT_SONG, backend=lyrics.LyricsWiki), + dict(artist=u'Santana', title=u'Black magic woman', + backend=lyrics.MusiXmatch), + dict(DEFAULT_SONG, backend=lyrics.Genius), + ] + + GOOGLE_SOURCES = [ + dict(DEFAULT_SONG, + url=u'http://www.absolutelyrics.com', + path=u'/lyrics/view/the_beatles/lady_madonna'), + dict(DEFAULT_SONG, + url=u'http://www.azlyrics.com', + path=u'/lyrics/beatles/ladymadonna.html'), + dict(DEFAULT_SONG, + url=u'http://www.chartlyrics.com', + path=u'/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx'), + dict(DEFAULT_SONG, + url=u'http://www.elyricsworld.com', + path=u'/lady_madonna_lyrics_beatles.html'), + dict(url=u'http://www.lacoccinelle.net', + artist=u'Jacques Brel', title=u"Amsterdam", + path=u'/paroles-officielles/275679.html'), + dict(DEFAULT_SONG, + url=u'http://letras.mus.br/', path=u'the-beatles/275/'), + dict(DEFAULT_SONG, + url='http://www.lyricsmania.com/', + path='lady_madonna_lyrics_the_beatles.html'), + dict(DEFAULT_SONG, url=u'http://lyrics.wikia.com/', + path=u'The_Beatles:Lady_Madonna'), + dict(DEFAULT_SONG, + url=u'http://www.lyricsmode.com', + path=u'/lyrics/b/beatles/lady_madonna.html'), + dict(url=u'http://www.lyricsontop.com', + artist=u'Amy Winehouse', title=u"Jazz'n'blues", + path=u'/amy-winehouse-songs/jazz-n-blues-lyrics.html'), + dict(DEFAULT_SONG, + url='http://www.metrolyrics.com/', + path='lady-madonna-lyrics-beatles.html'), + dict(url='http://www.musica.com/', path='letras.asp?letra=2738', + artist=u'Santana', title=u'Black magic woman'), + dict(url=u'http://www.paroles.net/', + artist=u'Lilly Wood & the prick', title=u"Hey it's ok", + path=u'lilly-wood-the-prick/paroles-hey-it-s-ok'), + dict(DEFAULT_SONG, + url='http://www.songlyrics.com', + path=u'/the-beatles/lady-madonna-lyrics'), + dict(DEFAULT_SONG, + url=u'http://www.sweetslyrics.com', + path=u'/761696.The%20Beatles%20-%20Lady%20Madonna.html') + ] + + def setUp(self): + LyricsGoogleBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @unittest.skipUnless(os.environ.get( + 'BEETS_TEST_LYRICS_SOURCES', '0') == '1', + 'lyrics sources testing not enabled') + def test_backend_sources_ok(self): + """Test default backends with songs known to exist in respective databases. + """ + errors = [] + for s in self.DEFAULT_SOURCES: + res = s['backend'](self.plugin.config, self.plugin._log).fetch( + s['artist'], s['title']) + if not is_lyrics_content_ok(s['title'], res): + errors.append(s['backend'].__name__) + self.assertFalse(errors) + + @unittest.skipUnless(os.environ.get( + 'BEETS_TEST_LYRICS_SOURCES', '0') == '1', + 'lyrics sources testing not enabled') + def test_google_sources_ok(self): + """Test if lyrics present on websites registered in beets google custom + search engine are correctly scraped. + """ + for s in self.GOOGLE_SOURCES: + url = s['url'] + s['path'] + res = lyrics.scrape_lyrics_from_html( + raw_backend.fetch_url(url)) + self.assertTrue(google.is_lyrics(res), url) + self.assertTrue(is_lyrics_content_ok(s['title'], res), url) + + +class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): + """Test scraping heuristics on a fake html page. + """ + + source = dict(url=u'http://www.example.com', artist=u'John Doe', + title=u'Beets song', path=u'/lyrics/beetssong') + + def setUp(self): + """Set up configuration""" + LyricsGoogleBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @patch.object(lyrics.Backend, 'fetch_url', MockFetchUrl()) def test_mocked_source_ok(self): """Test that lyrics of the mocked page are correctly scraped""" url = self.source['url'] + self.source['path'] - if os.path.isfile(url_to_filename(url)): - res = lyrics.scrape_lyrics_from_html(raw_backend.fetch_url(url)) - self.assertTrue(google.is_lyrics(res), url) - self.assertTrue(is_lyrics_content_ok(self.source['title'], res), - url) - - def test_google_sources_ok(self): - """Test if lyrics present on websites registered in beets google custom - search engine are correctly scraped.""" - if not check_lyrics_fetched(): - self.skipTest("Run lyrics_download_samples.py script first.") - for s in GOOGLE_SOURCES: - url = s['url'] + s['path'] - if os.path.isfile(url_to_filename(url)): - res = lyrics.scrape_lyrics_from_html( - raw_backend.fetch_url(url)) - self.assertTrue(google.is_lyrics(res), url) - self.assertTrue(is_lyrics_content_ok(s['title'], res), url) - - def test_default_ok(self): - """Test default engines with the default query""" - if not check_lyrics_fetched(): - self.skipTest("Run lyrics_download_samples.py script first.") - for (source, s) in zip([lyrics.LyricsWiki, - lyrics.LyricsCom, - lyrics.MusiXmatch], DEFAULT_SOURCES): - url = s['url'] + s['path'] - if os.path.isfile(url_to_filename(url)): - res = source({}, log).fetch(s['artist'], s['title']) - self.assertTrue(google.is_lyrics(res), url) - self.assertTrue(is_lyrics_content_ok(s['title'], res), url) + res = lyrics.scrape_lyrics_from_html(raw_backend.fetch_url(url)) + self.assertTrue(google.is_lyrics(res), url) + self.assertTrue(is_lyrics_content_ok(self.source['title'], res), + url) + @patch.object(lyrics.Backend, 'fetch_url', MockFetchUrl()) def test_is_page_candidate_exact_match(self): """Test matching html page title with song infos -- when song infos are - present in the title.""" + present in the title. + """ from bs4 import SoupStrainer, BeautifulSoup s = self.source url = six.text_type(s['url'] + s['path']) html = raw_backend.fetch_url(url) soup = BeautifulSoup(html, "html.parser", parse_only=SoupStrainer('title')) - self.assertEqual(google.is_page_candidate(url, soup.title.string, - s['title'], s['artist']), - True, url) + self.assertEqual( + google.is_page_candidate(url, soup.title.string, + s['title'], s['artist']), True, url) def test_is_page_candidate_fuzzy_match(self): """Test matching html page title with song infos -- when song infos are - not present in the title.""" + not present in the title. + """ s = self.source url = s['url'] + s['path'] url_title = u'example.com | Beats song by John doe' # very small diffs (typo) are ok eg 'beats' vs 'beets' with same artist self.assertEqual(google.is_page_candidate(url, url_title, s['title'], - s['artist']), True, url) + s['artist']), True, url) # reject different title url_title = u'example.com | seets bong lyrics by John doe' self.assertEqual(google.is_page_candidate(url, url_title, s['title'], - s['artist']), False, url) + s['artist']), False, url) def test_is_page_candidate_special_chars(self): """Ensure that `is_page_candidate` doesn't crash when the artist diff --git a/test/test_mediafile.py b/test/test_mediafile.py index 63df38b8e..18dcc11a3 100644 --- a/test/test_mediafile.py +++ b/test/test_mediafile.py @@ -350,6 +350,7 @@ class ReadWriteTestBase(ArtTestMixin, GenreListTestMixin, 'genre', 'lyricist', 'composer', + 'composer_sort', 'arranger', 'grouping', 'year', @@ -913,7 +914,7 @@ class AIFFTest(ReadWriteTestBase, unittest.TestCase): # remove this once we require a version that includes the feature. try: import mutagen.dsf # noqa -except: +except ImportError: HAVE_DSF = False else: HAVE_DSF = True diff --git a/test/test_query.py b/test/test_query.py index 3538c15a8..61df3ca10 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -30,7 +30,7 @@ import beets.library from beets import dbcore from beets.dbcore import types from beets.dbcore.query import (NoneQuery, ParsingError, - InvalidQueryArgumentTypeError) + InvalidQueryArgumentValueError) from beets.library import Library, Item from beets import util import platform @@ -301,11 +301,11 @@ class GetTest(DummyDataTestCase): self.assertFalse(results) def test_invalid_query(self): - with self.assertRaises(InvalidQueryArgumentTypeError) as raised: + with self.assertRaises(InvalidQueryArgumentValueError) as raised: dbcore.query.NumericQuery('year', u'199a') self.assertIn(u'not an int', six.text_type(raised.exception)) - with self.assertRaises(InvalidQueryArgumentTypeError) as raised: + with self.assertRaises(InvalidQueryArgumentValueError) as raised: dbcore.query.RegexpQuery('year', u'199(') exception_text = six.text_type(raised.exception) self.assertIn(u'not a regular expression', exception_text) diff --git a/test/test_replaygain.py b/test/test_replaygain.py index 6ea21ecb6..6ddee54da 100644 --- a/test/test_replaygain.py +++ b/test/test_replaygain.py @@ -52,14 +52,14 @@ class ReplayGainCliTestBase(TestHelper): try: self.load_plugins('replaygain') - except: + except Exception: import sys # store exception info so an error in teardown does not swallow it exc_info = sys.exc_info() try: self.teardown_beets() self.unload_plugins() - except: + except Exception: # if load_plugins() failed then setup is incomplete and # teardown operations may fail. In particular # {Item,Album} # may not have the _original_types attribute in unload_plugins diff --git a/test/test_template.py b/test/test_template.py index 1cbe9be0c..288bc2314 100644 --- a/test/test_template.py +++ b/test/test_template.py @@ -227,6 +227,11 @@ class ParseTest(unittest.TestCase): self.assertEqual(parts[2], u',') self._assert_symbol(parts[3], u"bar") + def test_newline_at_end(self): + parts = list(_normparse(u'foo\n')) + self.assertEqual(len(parts), 1) + self.assertEqual(parts[0], u'foo\n') + class EvalTest(unittest.TestCase): def _eval(self, template): diff --git a/tox.ini b/tox.ini index 43bff8014..8c3731f5d 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,7 @@ deps = flake8 flake8-coding flake8-future-import + flake8-blind-except pep8-naming files = beets beetsplug beet test setup.py docs