two
three" self.assertEqual( lyrics._scrape_merge_paragraphs(text), "one\ntwo\nthree" ) def test_missing_lyrics(self): self.assertFalse(google.is_lyrics(LYRICS_TEXTS["missing_texts"])) def url_to_filename(url): url = re.sub(r"https?://|www.", "", url) url = re.sub(r".html", "", url) fn = "".join(x for x in url if (x.isalnum() or x == "/")) fn = fn.split("/") fn = os.path.join( LYRICS_ROOT_DIR, bytestring_path(fn[0]), bytestring_path(fn[-1] + ".txt"), ) return fn class MockFetchUrl: def __init__(self, pathval="fetched_path"): self.pathval = pathval self.fetched = None def __call__(self, url, filename=None): self.fetched = url fn = url_to_filename(url) with open(fn, encoding="utf8") as f: content = f.read() return content class LyricsAssertions: """A mixin with lyrics-specific assertions.""" def assertLyricsContentOk(self, title, text, msg=""): # noqa: N802 """Compare lyrics text to expected lyrics for given title.""" if not text: return keywords = set(LYRICS_TEXTS[google.slugify(title)].split()) words = {x.strip(".?, ()") for x in text.lower().split()} if not keywords <= words: details = ( f"{keywords!r} is not a subset of {words!r}." f" Words only in expected set {keywords - words!r}," f" Words only in result set {words - keywords!r}." ) self.fail(f"{details} : {msg}") LYRICS_ROOT_DIR = os.path.join(_common.RSRC, b"lyrics") yaml_path = os.path.join(_common.RSRC, b"lyricstext.yaml") LYRICS_TEXTS = confuse.load_yaml(yaml_path) class LyricsGoogleBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: __import__("bs4") except ImportError: self.skipTest("Beautiful Soup 4 not available") class LyricsPluginSourcesTest(LyricsGoogleBaseTest, LyricsAssertions): """Check that beets google custom search engine sources are correctly scraped. """ DEFAULT_SONG = dict(artist="The Beatles", title="Lady Madonna") DEFAULT_SOURCES = [ # dict(artist=u'Santana', title=u'Black magic woman', # backend=lyrics.MusiXmatch), dict( DEFAULT_SONG, backend=lyrics.Genius, # GitHub actions is on some form of Cloudflare blacklist. skip=os.environ.get("GITHUB_ACTIONS") == "true", ), dict(artist="Boy In Space", title="u n eye", backend=lyrics.Tekstowo), ] GOOGLE_SOURCES = [ dict( DEFAULT_SONG, url="http://www.absolutelyrics.com", path="/lyrics/view/the_beatles/lady_madonna", ), dict( DEFAULT_SONG, url="http://www.azlyrics.com", path="/lyrics/beatles/ladymadonna.html", # AZLyrics returns a 403 on GitHub actions. skip=os.environ.get("GITHUB_ACTIONS") == "true", ), dict( DEFAULT_SONG, url="http://www.chartlyrics.com", path="/_LsLsZ7P4EK-F-LD4dJgDQ/Lady+Madonna.aspx", ), # dict(DEFAULT_SONG, # url=u'http://www.elyricsworld.com', # path=u'/lady_madonna_lyrics_beatles.html'), dict( url="http://www.lacoccinelle.net", artist="Jacques Brel", title="Amsterdam", path="/paroles-officielles/275679.html", ), dict( DEFAULT_SONG, url="http://letras.mus.br/", path="the-beatles/275/" ), dict( DEFAULT_SONG, url="http://www.lyricsmania.com/", path="lady_madonna_lyrics_the_beatles.html", ), dict( DEFAULT_SONG, url="http://www.lyricsmode.com", path="/lyrics/b/beatles/lady_madonna.html", ), dict( url="http://www.lyricsontop.com", artist="Amy Winehouse", title="Jazz'n'blues", path="/amy-winehouse-songs/jazz-n-blues-lyrics.html", ), # dict(DEFAULT_SONG, # url='http://www.metrolyrics.com/', # path='lady-madonna-lyrics-beatles.html'), # dict(url='http://www.musica.com/', path='letras.asp?letra=2738', # artist=u'Santana', title=u'Black magic woman'), dict( url="http://www.paroles.net/", artist="Lilly Wood & the prick", title="Hey it's ok", path="lilly-wood-the-prick/paroles-hey-it-s-ok", ), dict( DEFAULT_SONG, url="http://www.songlyrics.com", path="/the-beatles/lady-madonna-lyrics", ), dict( DEFAULT_SONG, url="http://www.sweetslyrics.com", path="/761696.The%20Beatles%20-%20Lady%20Madonna.html", ), ] def setUp(self): LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_backend_sources_ok(self): """Test default backends with songs known to exist in respective databases. """ # Don't test any sources marked as skipped. sources = [s for s in self.DEFAULT_SOURCES if not s.get("skip", False)] for s in sources: with self.subTest(s["backend"].__name__): backend = s["backend"](self.plugin.config, self.plugin._log) res = backend.fetch(s["artist"], s["title"]) self.assertLyricsContentOk(s["title"], res) @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_google_sources_ok(self): """Test if lyrics present on websites registered in beets google custom search engine are correctly scraped. """ # Don't test any sources marked as skipped. sources = [s for s in self.GOOGLE_SOURCES if not s.get("skip", False)] for s in sources: url = s["url"] + s["path"] res = lyrics.scrape_lyrics_from_html(raw_backend.fetch_url(url)) self.assertTrue(google.is_lyrics(res), url) self.assertLyricsContentOk(s["title"], res, url) class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest, LyricsAssertions): """Test scraping heuristics on a fake html page.""" source = dict( url="http://www.example.com", artist="John Doe", title="Beets song", path="/lyrics/beetssong", ) def setUp(self): """Set up configuration""" LyricsGoogleBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() @patch.object(lyrics.Backend, "fetch_url", MockFetchUrl()) def test_mocked_source_ok(self): """Test that lyrics of the mocked page are correctly scraped""" url = self.source["url"] + self.source["path"] res = lyrics.scrape_lyrics_from_html(raw_backend.fetch_url(url)) self.assertTrue(google.is_lyrics(res), url) self.assertLyricsContentOk(self.source["title"], res, url) @patch.object(lyrics.Backend, "fetch_url", MockFetchUrl()) def test_is_page_candidate_exact_match(self): """Test matching html page title with song infos -- when song infos are present in the title. """ from bs4 import BeautifulSoup, SoupStrainer s = self.source url = str(s["url"] + s["path"]) html = raw_backend.fetch_url(url) soup = BeautifulSoup( html, "html.parser", parse_only=SoupStrainer("title") ) self.assertEqual( google.is_page_candidate( url, soup.title.string, s["title"], s["artist"] ), True, url, ) def test_is_page_candidate_fuzzy_match(self): """Test matching html page title with song infos -- when song infos are not present in the title. """ s = self.source url = s["url"] + s["path"] url_title = "example.com | Beats song by John doe" # very small diffs (typo) are ok eg 'beats' vs 'beets' with same artist self.assertEqual( google.is_page_candidate(url, url_title, s["title"], s["artist"]), True, url, ) # reject different title url_title = "example.com | seets bong lyrics by John doe" self.assertEqual( google.is_page_candidate(url, url_title, s["title"], s["artist"]), False, url, ) def test_is_page_candidate_special_chars(self): """Ensure that `is_page_candidate` doesn't crash when the artist and such contain special regular expression characters. """ # https://github.com/beetbox/beets/issues/1673 s = self.source url = s["url"] + s["path"] url_title = "foo" google.is_page_candidate(url, url_title, s["title"], "Sunn O)))") # test Genius backend class GeniusBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: __import__("bs4") except ImportError: self.skipTest("Beautiful Soup 4 not available") class GeniusScrapeLyricsFromHtmlTest(GeniusBaseTest): """tests Genius._scrape_lyrics_from_html()""" def setUp(self): """Set up configuration""" GeniusBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() def test_no_lyrics_div(self): """Ensure we don't crash when the scraping the html for a genius page doesn't contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None url = "https://genius.com/sample" mock = MockFetchUrl() self.assertEqual(genius._scrape_lyrics_from_html(mock(url)), None) def test_good_lyrics(self): """Ensure we are able to scrape a page with lyrics""" url = "https://genius.com/Ttng-chinchilla-lyrics" mock = MockFetchUrl() self.assertIsNotNone(genius._scrape_lyrics_from_html(mock(url))) # TODO: find an example of a lyrics page with multiple divs and test it class GeniusFetchTest(GeniusBaseTest): """tests Genius.fetch()""" def setUp(self): """Set up configuration""" GeniusBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() @patch.object(lyrics.Genius, "_scrape_lyrics_from_html") @patch.object(lyrics.Backend, "fetch_url", return_value=True) def test_json(self, mock_fetch_url, mock_scrape): """Ensure we're finding artist matches""" with patch.object( lyrics.Genius, "_search", return_value={ "response": { "hits": [ { "result": { "primary_artist": { "name": "\u200Bblackbear", }, "url": "blackbear_url", } }, { "result": { "primary_artist": {"name": "El\u002Dp"}, "url": "El-p_url", } }, ] } }, ) as mock_json: # genius uses zero-width-spaces (\u200B) for lowercase # artists so we make sure we can match those self.assertIsNotNone(genius.fetch("blackbear", "Idfc")) mock_fetch_url.assert_called_once_with("blackbear_url") mock_scrape.assert_called_once_with(True) # genius uses the hyphen minus (\u002D) as their dash self.assertIsNotNone(genius.fetch("El-p", "Idfc")) mock_fetch_url.assert_called_with("El-p_url") mock_scrape.assert_called_with(True) # test no matching artist self.assertIsNone(genius.fetch("doesntexist", "none")) # test invalid json mock_json.return_value = None self.assertIsNone(genius.fetch("blackbear", "Idfc")) # TODO: add integration test hitting real api # test Tekstowo class TekstowoBaseTest(unittest.TestCase): def setUp(self): """Set up configuration.""" try: __import__("bs4") except ImportError: self.skipTest("Beautiful Soup 4 not available") class TekstowoExtractLyricsTest(TekstowoBaseTest): """tests Tekstowo.extract_lyrics()""" def setUp(self): """Set up configuration""" TekstowoBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() tekstowo.config = self.plugin.config def test_good_lyrics(self): """Ensure we are able to scrape a page with lyrics""" url = "https://www.tekstowo.pl/piosenka,24kgoldn,city_of_angels_1.html" mock = MockFetchUrl() self.assertIsNotNone( tekstowo.extract_lyrics(mock(url), "24kGoldn", "City of Angels") ) def test_no_lyrics(self): """Ensure we don't crash when the scraping the html for a Tekstowo page doesn't contain lyrics """ url = ( "https://www.tekstowo.pl/piosenka,beethoven," "beethoven_piano_sonata_17_tempest_the_3rd_movement.html" ) mock = MockFetchUrl() self.assertEqual( tekstowo.extract_lyrics( mock(url), "Beethoven", "Beethoven Piano Sonata 17" "Tempest The 3rd Movement", ), None, ) def test_song_no_match(self): """Ensure we return None when a song does not match the search query""" # https://github.com/beetbox/beets/issues/4406 # expected return value None url = ( "https://www.tekstowo.pl/piosenka,bailey_bigger" ",black_eyed_susan.html" ) mock = MockFetchUrl() self.assertEqual( tekstowo.extract_lyrics( mock(url), "Kelly Bailey", "Black Mesa Inbound" ), None, ) class TekstowoParseSearchResultsTest(TekstowoBaseTest): """tests Tekstowo.parse_search_results()""" def setUp(self): """Set up configuration""" TekstowoBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() def test_multiple_results(self): """Ensure we are able to scrape a page with multiple search results""" url = ( "https://www.tekstowo.pl/szukaj,wykonawca,juice+wrld" ",tytul,lucid+dreams.html" ) mock = MockFetchUrl() self.assertEqual( tekstowo.parse_search_results(mock(url)), "http://www.tekstowo.pl/piosenka,juice_wrld," "lucid_dreams__remix__ft__lil_uzi_vert.html", ) def test_no_results(self): """Ensure we are able to scrape a page with no search results""" url = ( "https://www.tekstowo.pl/szukaj,wykonawca," "agfdgja,tytul,agfdgafg.html" ) mock = MockFetchUrl() self.assertEqual(tekstowo.parse_search_results(mock(url)), None) class TekstowoIntegrationTest(TekstowoBaseTest, LyricsAssertions): """Tests Tekstowo lyric source with real requests""" def setUp(self): """Set up configuration""" TekstowoBaseTest.setUp(self) self.plugin = lyrics.LyricsPlugin() tekstowo.config = self.plugin.config @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_normal(self): """Ensure we can fetch a song's lyrics in the ordinary case""" lyrics = tekstowo.fetch("Boy in Space", "u n eye") self.assertLyricsContentOk("u n eye", lyrics) @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_no_matching_results(self): """Ensure we fetch nothing if there are search results returned but no matches""" # https://github.com/beetbox/beets/issues/4406 # expected return value None lyrics = tekstowo.fetch("Kelly Bailey", "Black Mesa Inbound") self.assertEqual(lyrics, None) # test LRCLib backend class LRCLibLyricsTest(unittest.TestCase): def setUp(self): self.plugin = lyrics.LyricsPlugin() lrclib.config = self.plugin.config @patch("beetsplug.lyrics.requests.get") def test_fetch_synced_lyrics(self, mock_get): mock_response = { "syncedLyrics": "[00:00.00] la la la", "plainLyrics": "la la la", } mock_get.return_value.json.return_value = mock_response mock_get.return_value.status_code = 200 lyrics = lrclib.fetch("la", "la", "la", 999) self.assertEqual(lyrics, mock_response["plainLyrics"]) self.plugin.config["synced"] = True lyrics = lrclib.fetch("la", "la", "la", 999) self.assertEqual(lyrics, mock_response["syncedLyrics"]) @patch("beetsplug.lyrics.requests.get") def test_fetch_plain_lyrics(self, mock_get): mock_response = { "syncedLyrics": "", "plainLyrics": "la la la", } mock_get.return_value.json.return_value = mock_response mock_get.return_value.status_code = 200 lyrics = lrclib.fetch("la", "la", "la", 999) self.assertEqual(lyrics, mock_response["plainLyrics"]) @patch("beetsplug.lyrics.requests.get") def test_fetch_not_found(self, mock_get): mock_response = { "statusCode": 404, "error": "Not Found", "message": "Failed to find specified track", } mock_get.return_value.json.return_value = mock_response mock_get.return_value.status_code = 404 lyrics = lrclib.fetch("la", "la", "la", 999) self.assertIsNone(lyrics) @patch("beetsplug.lyrics.requests.get") def test_fetch_exception(self, mock_get): mock_get.side_effect = requests.RequestException lyrics = lrclib.fetch("la", "la", "la", 999) self.assertIsNone(lyrics) class LRCLibIntegrationTest(LyricsAssertions): def setUp(self): self.plugin = lyrics.LyricsPlugin() lrclib.config = self.plugin.config @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_track_with_lyrics(self): lyrics = lrclib.fetch("Boy in Space", "u n eye", "Live EP", 160) self.assertLyricsContentOk("u n eye", lyrics) @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_instrumental_track(self): lyrics = lrclib.fetch( "Kelly Bailey", "Black Mesa Inbound", "Half Life 2 Soundtrack", 134, ) self.assertIsNone(lyrics) @unittest.skipUnless( os.environ.get("INTEGRATION_TEST", "0") == "1", "integration testing not enabled", ) def test_nonexistent_track(self): lyrics = lrclib.fetch("blah", "blah", "blah", 999) self.assertIsNone(lyrics) # test utilities class SlugTests(unittest.TestCase): def test_slug(self): # plain ascii passthrough text = "test" self.assertEqual(lyrics.slug(text), "test") # german unicode and capitals text = "Mørdag" self.assertEqual(lyrics.slug(text), "mordag") # more accents and quotes text = "l'été c'est fait pour jouer" self.assertEqual(lyrics.slug(text), "l-ete-c-est-fait-pour-jouer") # accents, parens and spaces text = "\xe7afe au lait (boisson)" self.assertEqual(lyrics.slug(text), "cafe-au-lait-boisson") text = "Multiple spaces -- and symbols! -- merged" self.assertEqual( lyrics.slug(text), "multiple-spaces-and-symbols-merged" ) text = "\u200Bno-width-space" self.assertEqual(lyrics.slug(text), "no-width-space") # variations of dashes should get standardized dashes = ["\u200D", "\u2010"] for dash1, dash2 in itertools.combinations(dashes, 2): self.assertEqual(lyrics.slug(dash1), lyrics.slug(dash2)) def suite(): return unittest.TestLoader().loadTestsFromName(__name__) if __name__ == "__main__": unittest.main(defaultTest="suite")