two
three" self.assertEqual(lyrics._scrape_merge_paragraphs(text), "one\ntwo\nthree") def test_missing_lyrics(self): self.assertFalse(google.is_lyrics(LYRICS_TEXTS['missing_texts'])) def url_to_filename(url): url = re.sub(r'https?://|www.', '', url) fn = "".join(x for x in url if (x.isalnum() or x == '/')) fn = fn.split('/') fn = os.path.join(LYRICS_ROOT_DIR, bytestring_path(fn[0]), bytestring_path(fn[-1] + '.txt')) return fn class MockFetchUrl: def __init__(self, pathval='fetched_path'): self.pathval = pathval self.fetched = None def __call__(self, url, filename=None): self.fetched = url fn = url_to_filename(url) with open(fn, encoding="utf8") as f: content = f.read() return content def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" if not text: return keywords = set(LYRICS_TEXTS[google.slugify(title)].split()) words = {x.strip(".?, ") for x in text.lower().split()} return keywords <= words LYRICS_ROOT_DIR = os.path.join(_common.RSRC, b'lyrics') yaml_path = os.path.join(_common.RSRC, b'lyricstext.yaml') LYRICS_TEXTS = confuse.load_yaml(yaml_path) class LyricsGoogleBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: __import__('bs4') except ImportError: self.skipTest('Beautiful Soup 4 not available') class LyricsPluginSourcesTest(LyricsGoogleBaseTest): """Check that beets google custom search engine sources are correctly scraped. """ DEFAULT_SONG = dict(artist='The Beatles', title='Lady Madonna') DEFAULT_SOURCES = [ # dict(artist=u'Santana', title=u'Black magic woman', # backend=lyrics.MusiXmatch), dict(DEFAULT_SONG, backend=lyrics.Genius, # GitHub actions is on some form of Cloudflare blacklist. skip=os.environ.get('GITHUB_ACTIONS') == 'true'), dict(artist='Boy In Space', title='u n eye', backend=lyrics.Tekstowo), ] GOOGLE_SOURCES = [ dict(DEFAULT_SONG, url='http://www.absolutelyrics.com', path='/lyrics/view/the_beatles/lady_madonna'), dict(DEFAULT_SONG, url='http://www.azlyrics.com', path='/lyrics/beatles/ladymadonna.html', # AZLyrics returns a 403 on GitHub actions. skip=os.environ.get('GITHUB_ACTIONS') == 'true'), dict(DEFAULT_SONG, url='http://www.chartlyrics.com', path='/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx'), # dict(DEFAULT_SONG, # url=u'http://www.elyricsworld.com', # path=u'/lady_madonna_lyrics_beatles.html'), dict(url='http://www.lacoccinelle.net', artist='Jacques Brel', title="Amsterdam", path='/paroles-officielles/275679.html'), dict(DEFAULT_SONG, url='http://letras.mus.br/', path='the-beatles/275/'), dict(DEFAULT_SONG, url='http://www.lyricsmania.com/', path='lady_madonna_lyrics_the_beatles.html'), dict(DEFAULT_SONG, url='http://www.lyricsmode.com', path='/lyrics/b/beatles/lady_madonna.html'), dict(url='http://www.lyricsontop.com', artist='Amy Winehouse', title="Jazz'n'blues", path='/amy-winehouse-songs/jazz-n-blues-lyrics.html'), # dict(DEFAULT_SONG, # url='http://www.metrolyrics.com/', # path='lady-madonna-lyrics-beatles.html'), # dict(url='http://www.musica.com/', path='letras.asp?letra=2738', # artist=u'Santana', title=u'Black magic woman'), dict(url='http://www.paroles.net/', artist='Lilly Wood & the prick', title="Hey it's ok", path='lilly-wood-the-prick/paroles-hey-it-s-ok'), dict(DEFAULT_SONG, url='http://www.songlyrics.com', path='/the-beatles/lady-madonna-lyrics'), dict(DEFAULT_SONG, url='http://www.sweetslyrics.com', path='/761696.The%20Beatles%20-%20Lady%20Madonna.html') ] def setUp(self): LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() @unittest.skipUnless( os.environ.get('INTEGRATION_TEST', '0') == '1', 'integration testing not enabled') def test_backend_sources_ok(self): """Test default backends with songs known to exist in respective databases. """ errors = [] # Don't test any sources marked as skipped. sources = [s for s in self.DEFAULT_SOURCES if not s.get("skip", False)] for s in sources: res = s['backend'](self.plugin.config, self.plugin._log).fetch( s['artist'], s['title']) if not is_lyrics_content_ok(s['title'], res): errors.append(s['backend'].__name__) self.assertFalse(errors) @unittest.skipUnless( os.environ.get('INTEGRATION_TEST', '0') == '1', 'integration testing not enabled') def test_google_sources_ok(self): """Test if lyrics present on websites registered in beets google custom search engine are correctly scraped. """ # Don't test any sources marked as skipped. sources = [s for s in self.GOOGLE_SOURCES if not s.get("skip", False)] for s in sources: url = s['url'] + s['path'] res = lyrics.scrape_lyrics_from_html( raw_backend.fetch_url(url)) self.assertTrue(google.is_lyrics(res), url) self.assertTrue(is_lyrics_content_ok(s['title'], res), url) class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): """Test scraping heuristics on a fake html page. """ source = dict(url='http://www.example.com', artist='John Doe', title='Beets song', path='/lyrics/beetssong') def setUp(self): """Set up configuration""" LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() @patch.object(lyrics.Backend, 'fetch_url', MockFetchUrl()) def test_mocked_source_ok(self): """Test that lyrics of the mocked page are correctly scraped""" url = self.source['url'] + self.source['path'] res = lyrics.scrape_lyrics_from_html(raw_backend.fetch_url(url)) self.assertTrue(google.is_lyrics(res), url) self.assertTrue(is_lyrics_content_ok(self.source['title'], res), url) @patch.object(lyrics.Backend, 'fetch_url', MockFetchUrl()) def test_is_page_candidate_exact_match(self): """Test matching html page title with song infos -- when song infos are present in the title. """ from bs4 import SoupStrainer, BeautifulSoup s = self.source url = str(s['url'] + s['path']) html = raw_backend.fetch_url(url) soup = BeautifulSoup(html, "html.parser", parse_only=SoupStrainer('title')) self.assertEqual( google.is_page_candidate(url, soup.title.string, s['title'], s['artist']), True, url) def test_is_page_candidate_fuzzy_match(self): """Test matching html page title with song infos -- when song infos are not present in the title. """ s = self.source url = s['url'] + s['path'] url_title = 'example.com | Beats song by John doe' # very small diffs (typo) are ok eg 'beats' vs 'beets' with same artist self.assertEqual(google.is_page_candidate(url, url_title, s['title'], s['artist']), True, url) # reject different title url_title = 'example.com | seets bong lyrics by John doe' self.assertEqual(google.is_page_candidate(url, url_title, s['title'], s['artist']), False, url) def test_is_page_candidate_special_chars(self): """Ensure that `is_page_candidate` doesn't crash when the artist and such contain special regular expression characters. """ # https://github.com/beetbox/beets/issues/1673 s = self.source url = s['url'] + s['path'] url_title = 'foo' google.is_page_candidate(url, url_title, s['title'], 'Sunn O)))') # test Genius backend class GeniusBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: __import__('bs4') except ImportError: self.skipTest('Beautiful Soup 4 not available') class GeniusScrapeLyricsFromHtmlTest(GeniusBaseTest): """tests Genius._scrape_lyrics_from_html()""" def setUp(self): """Set up configuration""" GeniusBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() def test_no_lyrics_div(self): """Ensure we don't crash when the scraping the html for a genius page doesn't contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None url = 'https://genius.com/sample' mock = MockFetchUrl() self.assertEqual(genius._scrape_lyrics_from_html(mock(url)), None) def test_good_lyrics(self): """Ensure we are able to scrape a page with lyrics""" url = 'https://genius.com/Wu-tang-clan-cream-lyrics' mock = MockFetchUrl() self.assertIsNotNone(genius._scrape_lyrics_from_html(mock(url))) # TODO: find an example of a lyrics page with multiple divs and test it class GeniusFetchTest(GeniusBaseTest): """tests Genius.fetch()""" def setUp(self): """Set up configuration""" GeniusBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() @patch.object(lyrics.Genius, '_scrape_lyrics_from_html') @patch.object(lyrics.Backend, 'fetch_url', return_value=True) def test_json(self, mock_fetch_url, mock_scrape): """Ensure we're finding artist matches""" with patch.object( lyrics.Genius, '_search', return_value={ "response": { "hits": [ { "result": { "primary_artist": { "name": "\u200Bblackbear", }, "url": "blackbear_url" } }, { "result": { "primary_artist": { "name": "El\u002Dp" }, "url": "El-p_url" } } ] } } ) as mock_json: # genius uses zero-width-spaces (\u200B) for lowercase # artists so we make sure we can match those self.assertIsNotNone(genius.fetch('blackbear', 'Idfc')) mock_fetch_url.assert_called_once_with("blackbear_url") mock_scrape.assert_called_once_with(True) # genius uses the hypen minus (\u002D) as their dash self.assertIsNotNone(genius.fetch('El-p', 'Idfc')) mock_fetch_url.assert_called_with('El-p_url') mock_scrape.assert_called_with(True) # test no matching artist self.assertIsNone(genius.fetch('doesntexist', 'none')) # test invalid json mock_json.return_value = None self.assertIsNone(genius.fetch('blackbear', 'Idfc')) # TODO: add integration test hitting real api # test utilties class SlugTests(unittest.TestCase): def test_slug(self): # plain ascii passthrough text = "test" self.assertEqual(lyrics.slug(text), 'test') # german unicode and capitals text = "Mørdag" self.assertEqual(lyrics.slug(text), 'mordag') # more accents and quotes text = "l'été c'est fait pour jouer" self.assertEqual(lyrics.slug(text), 'l-ete-c-est-fait-pour-jouer') # accents, parens and spaces text = "\xe7afe au lait (boisson)" self.assertEqual(lyrics.slug(text), 'cafe-au-lait-boisson') text = "Multiple spaces -- and symbols! -- merged" self.assertEqual(lyrics.slug(text), 'multiple-spaces-and-symbols-merged') text = "\u200Bno-width-space" self.assertEqual(lyrics.slug(text), 'no-width-space') # variations of dashes should get standardized dashes = ['\u200D', '\u2010'] for dash1, dash2 in itertools.combinations(dashes, 2): self.assertEqual(lyrics.slug(dash1), lyrics.slug(dash2)) def suite(): return unittest.TestLoader().loadTestsFromName(__name__) if __name__ == '__main__': unittest.main(defaultTest='suite')