From 8e28f0b694bc04fa0735c8d3f6feb8232647c8d7 Mon Sep 17 00:00:00 2001 From: lijacky Date: Sun, 12 Apr 2020 00:05:17 -0400 Subject: [PATCH 1/6] added null check for genius lyrics scrape --- beetsplug/lyrics.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 0e797d5a3..bb225007d 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,7 +373,13 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. - lyrics = html.find("div", class_="lyrics").get_text() + try: + lyrics = html.find("div", class_="lyrics").get_text() + except AttributeError as exc: + # html is a NoneType cannot retrieve lyrics + self._log.debug(u'Genius lyrics for {0} not found: {1}', + page_url, exc) + return None return lyrics From 29d7b80847fd4f1d5c4f47d50d758160d6866c39 Mon Sep 17 00:00:00 2001 From: lijacky Date: Wed, 15 Apr 2020 22:23:27 -0400 Subject: [PATCH 2/6] Added unit test for null check --- test/test_lyrics.py | 61 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index f7ea538e2..e72b2d4f8 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -24,7 +24,7 @@ import sys import unittest from mock import patch -from test import _common +import _common from beets import logging from beets.library import Item @@ -39,6 +39,7 @@ from mock import MagicMock log = logging.getLogger('beets.test_lyrics') raw_backend = lyrics.Backend({}, log) google = lyrics.Google(MagicMock(), log) +genius = lyrics.Genius(MagicMock(), log) class LyricsPluginTest(unittest.TestCase): @@ -213,6 +214,29 @@ class MockFetchUrl(object): content = f.read() return content +class GeniusMockGet(object): + def __init__(self, pathval='fetched_path'): + self.pathval = pathval + self.fetched = None + + def __call__(self, url, headers=False): + from requests.models import Response + # for the first requests.get() return a path + if headers: + response = Response() + response.status_code = 200 + response._content = b'{"meta":{"status":200},"response":{"song":{"path":"/lyrics/sample"}}}' + return response + # for the second requests.get() return the genius page + else: + from mock import PropertyMock + self.fetched = url + fn = url_to_filename(url) + with open(fn, 'r') as f: + content = f.read() + response = Response() + type(response).text = PropertyMock(return_value=content) + return response def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" @@ -395,6 +419,41 @@ class LyricsGooglePluginMachineryTest(LyricsGoogleBaseTest): google.is_page_candidate(url, url_title, s['title'], u'Sunn O)))') +class LyricsGeniusBaseTest(unittest.TestCase): + + def setUp(self): + """Set up configuration.""" + try: + __import__('bs4') + except ImportError: + self.skipTest('Beautiful Soup 4 not available') + if sys.version_info[:3] < (2, 7, 3): + self.skipTest("Python's built-in HTML parser is not good enough") + + +class LyricsGeniusScrapTest(LyricsGeniusBaseTest): + """Checks that Genius backend works as intended. + """ + import requests + def setUp(self): + """Set up configuration""" + LyricsGeniusBaseTest.setUp(self) + self.plugin = lyrics.LyricsPlugin() + + @patch.object(requests, 'get', GeniusMockGet()) + def test_no_lyrics_div(self): + """Ensure that `lyrics_from_song_api_path` doesn't crash when the html + for a Genius page contain
...
+ """ + # https://github.com/beetbox/beets/issues/3535 + # expected return value None + try: + self.assertEqual(genius.lyrics_from_song_api_path('/no_lyric_page'), None) + except AttributeError: + # if AttributeError we aren't doing a null check + self.assertTrue(False) + + class SlugTests(unittest.TestCase): def test_slug(self): From 525202e52919e971781a867f20d49b1548161d43 Mon Sep 17 00:00:00 2001 From: lijacky Date: Wed, 15 Apr 2020 22:30:31 -0400 Subject: [PATCH 3/6] adding genius sample html --- test/rsrc/lyrics/geniuscom/sample.txt | 1119 +++++++++++++++++++++++++ 1 file changed, 1119 insertions(+) create mode 100644 test/rsrc/lyrics/geniuscom/sample.txt diff --git a/test/rsrc/lyrics/geniuscom/sample.txt b/test/rsrc/lyrics/geniuscom/sample.txt new file mode 100644 index 000000000..da8a4d0b6 --- /dev/null +++ b/test/rsrc/lyrics/geniuscom/sample.txt @@ -0,0 +1,1119 @@ + + + + + + + +SAMPLE – SONG Lyrics | Genius Lyrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + {{:: 'cloud_flare_always_on_short_message' | i18n }} +
Check @genius for updates. We'll have things fixed soon. + +
+
+ + +
+ GENIUS +
+ + + +
+ + + + + + + + + + + +
+ + +
+ + + +
+ + + + + + + + + + + + + + + + +
+ +
+ +
+
+
+
+
+ Https%3a%2f%2fimages +
+
+ +
+
+ +

SONG

+

+ SAMPLE +

+ +

+ + +

+

+ +
+ Produced by + + + Carter Lang & The Antydote + +
+ + +

+

+ +
+ Album + ALBUM +
+ + +

+ +
+
+
+
+
+
+
+ + +
+
+
+ +

SONG Lyrics

+ +
+
+ + +

[Verse 1]
+Fearin' not growin' up
+Keepin' me up at night
+Am I doin' enough?
+Feel like I'm wastin' time
+
+[Pre-Chorus]
+Promise to get a little
+Better as I get older
+And you're so patient
+And sick of waitin'
+Promise to do better
+Shoulda coulda
+Prolly wanna let me go
+But you can't, oh
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+
+[Chorus]
+Please don't take it, don't take it personal
+Like I know you usually do
+Please don't take it, don't take it personal
+Like I know you usually do
+Please, please
+Don't take it personal
+Don't take it personal
+Darling, like I know you will, ooh
+
+[Verse 2]
+Forget to call your mama on the weekend
+You should put yourself in time out
+(Shame, shame on you)
+But lately you've been feelin' so good
+I forget my future, never pull out
+(Shame, shame on me)
+Baby the money'll make it easier for me
+To run and hide out somewhere
+(So far away)
+Hoppin' through poppy fields
+Dodgin' evil witches
+These houses keep droppin' everywhere
+
+[Pre-Chorus]
+Promise to get a little
+Better as I get older
+And you're so patient
+And sick of waitin'
+Promise to do better
+Shoulda coulda
+Prolly wanna let me go
+But you can't, oh
+Right now I feel it pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+Right now it's really pourin'
+I need a little bit
+Just a little bit
+Just a little bit
+
+[Chorus]
+Please don't take it, don't take it personal
+Like I know you usually do
+Please don't take it, take it personal
+
+[Outro]
+Like winters fall on us, heavy
+Take it off me, all it off
+Winter, I can't stand this
+Snow is falling all on me

+ + + + +
+
+ +
+ +
+
More on Genius
+ +
+ +
+
+
+ +
+
+ +
+
+
+ + + +
+ +

+ About “SONG” +

+ + +
+
+

The fifth track on SAMPLE’s debut album, ALBUM is the most reminiscent of her previous ep’s with a pop-disco sound.

+ +

She reflects about her insecurities on not maturing as quickly as her partner is. SONG season is usually the last big event before graduation for a high schooler, and SAMPLE is too busy living in the moment that she forgets she has her future to look forward to.

+ +

+ +

This is also around the age when friends are growing apart and have decided where and what they’re doing with their lives, and a lot are trying to hold on to the people they’ve known, even though they’re growing up differently. The person SAMPLE is talking to in the song seems to take everything, even change, to heart.

+
+
+
+ +
+
+
    + +
  • +
    +

    What have the artists said about this song

    +
    + +
    +

    SAMPLE tweeted:

    + +

    Iss called SONG cus I ain’t have no friends in high school so I went to Miami w my mama for SONG instead . Lol

    +
    + + +
    • + + + +
+
+ + +
+
+

"SONG" Track Info

+ + +
+ Written By + + + The Antydote, Carter Lang & SAMPLE + +
+ + + + + +
+ Released by + + + RCA Records & Top Dawg Entertainment + +
+ + + + +
+ Mixing + + + Mixed by Ali + +
+ + + + + +
+ Recorded At + TDE Red Room, Carson, California +
+ + + +
+ Release Date + June 9, 2017 +
+ + + + + + + + + + + + + + + + + + +
+ Interpolated By + + +
+ Wicked Games (StarFox Remix) by The Weeknd +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+
+
+ + Https%3a%2f%2fimages + +
+ + ALBUM + + + SAMPLE +
+
+ +
+ +
+
+ + 1.   + + + + # + + + + +
+
+ +
+
+ + 2.   + + + + # + + + + +
+
+ +
+
+ + 3.   + + + + # + + + + +
+
+ +
+
+ + 4.   + + + + # + + + + +
+
+ +
+
+ + 5.   + + + + SONG + + + + +
+
+ +
+
+ + 6.   + + + + # + + + + +
+
+ +
+
+ + 7.   + + + + # + + + + +
+
+ +
+
+ + 8.   + + + + # + + + + +
+
+ +
+
+ + 9.   + + + + # + + + + +
+
+ +
+
+ + 10.   + + + + # + + + + +
+
+ +
+
+ + 11.   + + + + # + + + + +
+
+ +
+
+ + 12.   + + + + # + + + + +
+
+ +
+
+ + 13.   + + + + # + + + + +
+
+ +
+
+ + 14.   + + + + # + + + + +
+
+ + + +
+ +
+
+ +
+
+
+ + + + + + + + + +
+
+ + + + +
+ © 2020 Genius Media Group Inc. +
+
+ +
+ Verified Artists + + All Artists: +
    + +
  • + A +
    • + +
  • + B +
    • + +
  • + C +
    • + +
  • + D +
    • + +
  • + E +
    • + +
  • + F +
    • + +
  • + G +
    • + +
  • + H +
    • + +
  • + I +
    • + +
  • + J +
    • + +
  • + K +
    • + +
  • + L +
    • + +
  • + M +
    • + +
  • + N +
    • + +
  • + O +
    • + +
  • + P +
    • + +
  • + Q +
    • + +
  • + R +
    • + +
  • + S +
    • + +
  • + T +
    • + +
  • + U +
    • + +
  • + V +
    • + +
  • + W +
    • + +
  • + X +
    • + +
  • + Y +
    • + +
  • + Z +
    • + +
  • + # +
    • + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + From b7ecf32f2876848611dcced77c0a2624198d35a9 Mon Sep 17 00:00:00 2001 From: lijacky Date: Thu, 16 Apr 2020 17:20:08 -0400 Subject: [PATCH 4/6] style changes --- test/test_lyrics.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/test_lyrics.py b/test/test_lyrics.py index e72b2d4f8..6c23c627a 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -24,7 +24,7 @@ import sys import unittest from mock import patch -import _common +from test import _common from beets import logging from beets.library import Item @@ -214,7 +214,9 @@ class MockFetchUrl(object): content = f.read() return content + class GeniusMockGet(object): + def __init__(self, pathval='fetched_path'): self.pathval = pathval self.fetched = None @@ -225,7 +227,8 @@ class GeniusMockGet(object): if headers: response = Response() response.status_code = 200 - response._content = b'{"meta":{"status":200},"response":{"song":{"path":"/lyrics/sample"}}}' + response._content = b'{"meta":{"status":200},\ + "response":{"song":{"path":"/lyrics/sample"}}}' return response # for the second requests.get() return the genius page else: @@ -238,6 +241,7 @@ class GeniusMockGet(object): type(response).text = PropertyMock(return_value=content) return response + def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title.""" if not text: @@ -432,9 +436,11 @@ class LyricsGeniusBaseTest(unittest.TestCase): class LyricsGeniusScrapTest(LyricsGeniusBaseTest): + """Checks that Genius backend works as intended. """ import requests + def setUp(self): """Set up configuration""" LyricsGeniusBaseTest.setUp(self) @@ -443,12 +449,13 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): @patch.object(requests, 'get', GeniusMockGet()) def test_no_lyrics_div(self): """Ensure that `lyrics_from_song_api_path` doesn't crash when the html - for a Genius page contain
...
+ for a Genius page contain
""" # https://github.com/beetbox/beets/issues/3535 # expected return value None try: - self.assertEqual(genius.lyrics_from_song_api_path('/no_lyric_page'), None) + self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + None) except AttributeError: # if AttributeError we aren't doing a null check self.assertTrue(False) From 9ec0d725e52a06863ac733e2f560dbc2b08fc014 Mon Sep 17 00:00:00 2001 From: lijacky Date: Fri, 17 Apr 2020 17:14:21 -0400 Subject: [PATCH 5/6] Changes given feedback on https://github.com/beetbox/beets/pull/3554 and trimmed sample html --- beetsplug/lyrics.py | 13 +- test/rsrc/lyrics/geniuscom/sample.txt | 1289 +++++-------------------- test/test_lyrics.py | 8 +- 3 files changed, 229 insertions(+), 1081 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index bb225007d..00b8820f4 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -373,13 +373,14 @@ class Genius(Backend): # At least Genius is nice and has a tag called 'lyrics'! # Updated css where the lyrics are based in HTML. - try: - lyrics = html.find("div", class_="lyrics").get_text() - except AttributeError as exc: - # html is a NoneType cannot retrieve lyrics - self._log.debug(u'Genius lyrics for {0} not found: {1}', - page_url, exc) + lyrics_div = html.find("div", class_="lyrics") + + # nullcheck + if lyrics_div is None: + self._log.debug(u'Genius lyrics for {0} not found', + page_url) return None + lyrics = lyrics_div.get_text() return lyrics diff --git a/test/rsrc/lyrics/geniuscom/sample.txt b/test/rsrc/lyrics/geniuscom/sample.txt index da8a4d0b6..1648d070a 100644 --- a/test/rsrc/lyrics/geniuscom/sample.txt +++ b/test/rsrc/lyrics/geniuscom/sample.txt @@ -1,1119 +1,270 @@ - + + //]]> + -SAMPLE – SONG Lyrics | Genius Lyrics + SAMPLE – SONG Lyrics | g-example Lyrics - - + + - + - + + + - + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ga('create', "UA-10346621-1", 'auto', {'useAmpClientId': true}); + ga('set', 'dimension1', "false"); + ga('set', 'dimension2', "songs#show"); + ga('set', 'dimension3', "r-b"); + ga('set', 'dimension4', "true"); + ga('set', 'dimension5', 'false'); + ga('set', 'dimension6', "none"); + ga('send', 'pageview'); + + - - + - - +
+
+
- - - - -
- - {{:: 'cloud_flare_always_on_short_message' | i18n }} -
Check @genius for updates. We'll have things fixed soon. - -
-
- - -
- GENIUS -
- - - -
- - - - - - - - - - - -
- - -
- - - -
- - - - - - - - - - - - - - - - -
- -
- -
-
-
-
-
- Https%3a%2f%2fimages -
-
- -
-
- -

SONG

-

- SAMPLE -

- -

- - -

-

- -
- Produced by - - - Carter Lang & The Antydote - -
- - -

-

- -
- Album - ALBUM -
- - -

- -
-
-
-
-
-
-
- - -
-
-
- -

SONG Lyrics

- -
-
- - -

[Verse 1]
-Fearin' not growin' up
-Keepin' me up at night
-Am I doin' enough?
-Feel like I'm wastin' time
-
-[Pre-Chorus]
-Promise to get a little
-Better as I get older
-And you're so patient
-And sick of waitin'
-Promise to do better
-Shoulda coulda
-Prolly wanna let me go
-But you can't, oh
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-
-[Chorus]
-Please don't take it, don't take it personal
-Like I know you usually do
-Please don't take it, don't take it personal
-Like I know you usually do
-Please, please
-Don't take it personal
-Don't take it personal
-Darling, like I know you will, ooh
-
-[Verse 2]
-Forget to call your mama on the weekend
-You should put yourself in time out
-(Shame, shame on you)
-But lately you've been feelin' so good
-I forget my future, never pull out
-(Shame, shame on me)
-Baby the money'll make it easier for me
-To run and hide out somewhere
-(So far away)
-Hoppin' through poppy fields
-Dodgin' evil witches
-These houses keep droppin' everywhere
-
-[Pre-Chorus]
-Promise to get a little
-Better as I get older
-And you're so patient
-And sick of waitin'
-Promise to do better
-Shoulda coulda
-Prolly wanna let me go
-But you can't, oh
-Right now I feel it pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-Right now it's really pourin'
-I need a little bit
-Just a little bit
-Just a little bit
-
-[Chorus]
-Please don't take it, don't take it personal
-Like I know you usually do
-Please don't take it, take it personal
-
-[Outro]
-Like winters fall on us, heavy
-Take it off me, all it off
-Winter, I can't stand this
-Snow is falling all on me

- - - - -
-
- -
-
-
More on Genius
- +
+ g-example
+ + + +
-
-
-
- -
-
-
+ + - +
+
+
+
+
+ # +
+
-
+
+
+

SONG

+

+ + SAMPLE + +

+

+
+ Produced by + + Person 1 & Person 2 + +
+

+

+
+ Album + ALBUM +
+

+
+
+ +
+
+
+ +
+
+
+

SONG Lyrics

+
+
+ !!!! MISSING LYRICS HERE !!! +
+
+
+
+
More on g-example
+
+
+
+
+
-

- About “SONG” -

- - -
-
-

The fifth track on SAMPLE’s debut album, ALBUM is the most reminiscent of her previous ep’s with a pop-disco sound.

+
+ Released by -

She reflects about her insecurities on not maturing as quickly as her partner is. SONG season is usually the last big event before graduation for a high schooler, and SAMPLE is too busy living in the moment that she forgets she has her future to look forward to.

- -

- -

This is also around the age when friends are growing apart and have decided where and what they’re doing with their lives, and a lot are trying to hold on to the people they’ve known, even though they’re growing up differently. The person SAMPLE is talking to in the song seems to take everything, even change, to heart.

-
-
-
- -
-
-
    - -
  • -
    -

    What have the artists said about this song

    -
    - -
    -

    SAMPLE tweeted:

    - -

    Iss called SONG cus I ain’t have no friends in high school so I went to Miami w my mama for SONG instead . Lol

    -
    - - -
    • - - - -
-
- - -
-
-

"SONG" Track Info

- - -
- Written By - - - The Antydote, Carter Lang & SAMPLE - -
- - - - - -
- Released by - - - RCA Records & Top Dawg Entertainment - -
- - - - -
- Mixing - - - Mixed by Ali - -
- - - - - -
- Recorded At - TDE Red Room, Carson, California -
- - - -
- Release Date - June 9, 2017 -
- - - - - - - - - - - - - - - - - - -
- Interpolated By - - -
- Wicked Games (StarFox Remix) by The Weeknd + + Records & Top +
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - +
+ Mixing + + Mixed by Person + +
+ +
+ Recorded At + City, Place +
+ +
+ Release Date + Feb 30, 1290 +
+ +
+ Interpolated By + + +
+ # +
+ +
-
-
- - Https%3a%2f%2fimages - -
- - ALBUM - - - SAMPLE -
-
- -
- -
-
- - 1.   - - - - # - - - - -
-
- -
-
- - 2.   - - - - # - - - - -
-
- -
-
- - 3.   - - - - # - - - - -
-
- -
-
- - 4.   - - - - # - - - - -
-
- -
-
- - 5.   - - - - SONG - - - - -
-
- -
-
- - 6.   - - - - # - - - - -
-
- -
-
- - 7.   - - - - # - - - - -
-
- -
-
- - 8.   - - - - # - - - - -
-
- -
-
- - 9.   - - - - # - - - - -
-
- -
-
- - 10.   - - - - # - - - - -
-
- -
-
- - 11.   - - - - # - - - - -
-
- -
-
- - 12.   - - - - # - - - - -
-
- -
-
- - 13.   - - - - # - - - - -
-
- -
-
- - 14.   - - - - # - - - - -
-
- - - -
- + + # + +
+ + ALBUM + + SAMPLE +
-
- -
-
-
- - - - - - - + +
-
- -
- Privacy Policy - - - Licensing - Jobs - Developers - Terms of Use - Copyright Policy - Contact us - - Do Not Sell My Personal Information - - - Sign in + + + - All Artists: -
    - -
  • - A -
    • - -
  • - B -
    • - -
  • - C -
    • - -
  • - D -
    • - -
  • - E -
    • - -
  • - F -
    • - -
  • - G -
    • - -
  • - H -
    • - -
  • - I -
    • - -
  • - J -
    • - -
  • - K -
    • - -
  • - L -
    • - -
  • - M -
    • - -
  • - N -
    • - -
  • - O -
    • - -
  • - P -
    • - -
  • - Q -
    • - -
  • - R -
    • - -
  • - S -
    • - -
  • - T -
    • - -
  • - U -
    • - -
  • - V -
    • - -
  • - W -
    • - -
  • - X -
    • - -
  • - Y -
    • - -
  • - Z -
    • - -
  • - # -
    • - -
- -
- -
- - - - - - - - - - - - - - - - - + + + + + - diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 6c23c627a..83a229f62 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -453,12 +453,8 @@ class LyricsGeniusScrapTest(LyricsGeniusBaseTest): """ # https://github.com/beetbox/beets/issues/3535 # expected return value None - try: - self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), - None) - except AttributeError: - # if AttributeError we aren't doing a null check - self.assertTrue(False) + self.assertEqual(genius.lyrics_from_song_api_path('/nolyric'), + None) class SlugTests(unittest.TestCase): From 175c392ef33928f7decffed038cce90cdf00a7b6 Mon Sep 17 00:00:00 2001 From: lijacky Date: Fri, 17 Apr 2020 17:25:36 -0400 Subject: [PATCH 6/6] add fix to changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 33ad386ad..fd0c225f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -178,6 +178,8 @@ Fixes: * Removed ``@classmethod`` decorator from dbcore.query.NoneQuery.match method failing with AttributeError when called. It is now an instance method. :bug:`3516` :bug:`3517` +* :doc:`/plugins/lyrics`: Tolerate missing lyrics div in Genius scraper + :bug:`3535` For plugin developers: