From a0579d66cfebe16d85cc96c5f7e34147a20ae310 Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Tue, 11 Nov 2014 09:06:37 +0100 Subject: [PATCH] lyrics: remove full texts from lyricstext.yaml lyrics are considered valid if they contain a set of selected keywords picked along the original full text. --- test/rsrc/lyricstext.yaml | 277 ++++---------------------------------- test/test_lyrics.py | 16 +-- 2 files changed, 33 insertions(+), 260 deletions(-) diff --git a/test/rsrc/lyricstext.yaml b/test/rsrc/lyricstext.yaml index da4b9a56c..6ff9af7be 100644 --- a/test/rsrc/lyricstext.yaml +++ b/test/rsrc/lyricstext.yaml @@ -1,256 +1,37 @@ -Beets_song: | - Beets is the media library management system for obsessive-compulsive music - geeks. - The purpose of beets is to get your music collection right once and for all - It catalogs your collection, automatically improving its metadata as it - goes. - It then provides a bouquet of tools for manipulating and accessing your - music. Here's an example of beets' brainy tag corrector doing its thing: +Amsterdam: + - oriflammes + - fortune + - batave + - pissent -Amsterdam: | - Dans le port d'Amsterdam - Y a des marins qui chantent - Les rêves qui les hantent - Au large d'Amsterdam - Dans le port d'Amsterdam - Y a des marins qui dorment - Comme des oriflammes - Le long des berges mornes - Dans le port d'Amsterdam - Y a des marins qui meurent - Pleins de bière et de drames - Aux premières lueurs - Mais dans le port d'Amsterdam - Y a des marins qui naissent - Dans la chaleur épaisse - Des langueurs océanes - Dans le port d'Amsterdam - Y a des marins qui mangent - Sur des nappes trop blanches - Des poissons ruisselants - Ils vous montrent des dents - A croquer la fortune - A décroisser la lune - A bouffer des haubans - Et ça sent la morue - Jusque dans le coeur des frites - Que leurs grosses mains invitent - A revenir en plus - Puis se lèvent en riant - Dans un bruit de tempête - Referment leur braguette - Et sortent en rotant - Dans le port d'Amsterdam - Y a des marins qui dansent - En se frottant la panse - Sur la panse des femmes - Et ils tournent et ils dansent - Comme des soleils crachés - Dans le son déchiré - D'un accordéon rance - Ils se tordent le cou - Pour mieux s'entendre rire - Jusqu'à ce que tout à coup - L'accordéon expire - Alors le geste grave - Alors le regard fier - Ils ramènent leur batave - Jusqu'en pleine lumière - Dans le port d'Amsterdam - Y a des marins qui boivent - Et qui boivent et reboivent - Et qui reboivent encore - Ils boivent à la santé - Des putains d'Amsterdam - De Hambourg ou d'ailleurs - Enfin ils boivent aux dames - Qui leur donnent leur joli corps - Qui leur donnent leur vertu - Pour une pièce en or - Et quand ils ont bien bu - Se plantent le nez au ciel - Se mouchent dans les étoiles - Et ils pissent comme je pleure - Sur les femmes infidèles - Dans le port d'Amsterdam +Lady_Madonna: + - heaven + - tuesday + - thursday -Lady_Madonna: | - Lady Madonna, children at your feet - Wonder how you manage to make ends meet - Who finds the money? When you pay the rent? - Did you think that money was Heaven sent? - Friday night arrives without a suitcase - Sunday morning creep in like a nun - Monday's child has learned to tie his bootlace - See how they run +Jazz_n_blues: + - parkway + - balance + - impatient + - shoes - Lady Madonna, baby at your breast - Wonder how you manage to feed the rest +I_could_beat_myself: + - funny + - reputation + - destroy + - shiver - See how they run +Hey_it_s_ok: + - swear + - forgive + - drink + - found - Lady Madonna, lying on the bed - Listen to the music playing in your head - - Tuesday afternoon is never ending - Wednesday morning papers didn't come - Thursday night you stockings needed mending - See how they run - - Lady Madonna, children at your feet - Wonder how you manage to make ends meet - -Jazz_n_blues: | - It's always gone within two days. - Follow my father, his extravagant ways. - So if I got it I will spend it all, - Camden and Parkway 'til I hit a wall. - - I cross my fingers at the cash machine, - As I check my balance I kiss the screen. - I love it when it says I got the means, - To go to Miss Sixty and pick up my new jeans. - - Chorus: (x2) - Never lasts me long, - Handle finance wrong, - Blow it all on bags and shoes, - Jazz 'n blues. - - Standin' too deep at the bar today, - Wait with impatience to throw my cash away. - Four white Russians a JD and Coke, - Buy their drinks all night and now I am broke. - But that's cool 'cause I can borrow more from you, - And I didn't forget about that fifty pound, too. - Tell you what, my advance is comin' through, - I'll take you out shoppin', - Can you wait 'til next June? Yeah. - - Chorus: - Never lasts me long, - Handle finance wrong (handle it wrong), - Blow it all on bags and shoes, - Jazz 'n blues (jazz and blues). - Never lasts me long (long), - Handle finance wrong (wrong), - Blow it all on bags and shoes (blow it all), - Jazz 'n blues. - -I_could_beat_myself: | - oooh ahh, I'm hurting, bad - oooh ahh I'm hurting bad! - - I did not see what I was supposed to see - taking it easy when a friend told me I was in - danger, so much danger. - Underestimating my woman, not taking her - out, working too hard and now she's gone off - with a stranger, someone I don't even know. - - I should have taken her out, every once and a - while, taken her to dinner on the finer side, shown - her a life that was all worth while, now I guese I gotta - walk an extra mile! I could beat myself! Ahh yeah. - - Now I'm gonna feel funny out there in the crowd - when my friends all ask me, where is your woman? - long time I don't see. Now I've got to think fast gotta - use my head, give a good story and make sure they - buy my version of the situation. - - All the while I wouldn't lie I'm gonna do this once - see my reputation sinking in the distance, if they - knew the truth that really existed...then my little sanity - would be wasted! I could beat myself ahh yes..I could - beat myself!!! Ohh lord lord. - - oooh ooh I'm hurting, hurting inside - oooh ooh I'm hurting - - Now I really want to hear a little news now and - then, this is not what I expect to hear from my - friend I'm dissapointed. He should realize that its - gonna destroy my position (remember I'm a name - brand) now I've gotta see....I should have held her tight - every once and a while, gotten it together on the finer style - shown her a life that was all worth while, now it seems I gotta - walk an extra mile....I could beat myself..ahh yes, I could beat myself - ooh lord - - ooh I'm hurting - ooooh I'm hurting,hurting inside - - I don't know what I wanna tell you - but I wanna tell you something real, real good yes - someting to make you - wanna shiver - -Hey_it_s_ok: | - Mama, Papa, please forget the times - I wet my bed - I swear not to do it again - Please forgive the way I looked - when I was fourteen - I didn't know who I wanted to be - - Hey It's OK, It's OK - Cause I've found what i wanted - Hey It's OK, I'ts Ok - Cause I've found what i wanted - - Friends and lovers please forgive - the mean things I've said - I swear not to do again - Please forgive the way I act when - I've had too much to drink - I'm fighting against myself - - Hey It's OK, It's OK - Cause I've found what i wanted - Hey It's OK, I'ts Ok - - Cause I've found what i wanted - - And I swear not to do anything funny anymore - yes I swear not to do anything funny anymore - - Hey It's OK, It's OK - Cause I've found what i wanted - Hey It's OK, I'ts Ok - Cause I've found what i wanted - Hey It's OK, It's OK - Cause I've found what i wanted - Hey It's OK, I'ts Ok - Cause I've found what i wanted - -City_of_dreams: | - Day breaks over the city of my childhood - Daybreak over the city I called home - Where the sage met the sea and the groves were sweet and green - It's a city that lives only in my dreams - - The groves where we played when we were children - The groves where we fooled around as teens - Those green groves are paved from la to santa fe - That city lives only in my dreams - - Landmarks lost to parking lots in the city I called home - Looking back I see what used to be - Now freeways crawl though the suburban sprawl - As far as the eye can see - And the city lives only in my dreams - - Twilight shades the valley of my memory - When citrus groves still perfumed the sky - But I guess those orange blossoms weren't so special after all - Now it's a city of days gone by - - Landmarks lost to parking lots in the city I called home - Looking back I see what used to be - Now freeways crawl through the suburban sprawl - As far as the eye can see - And the city lives only in my dreams +City_of_dreams: + - groves + - landmarks + - twilight + - freeways missing_texts: | Lyricsmania staff is working hard for you to add $TITLE lyrics as soon diff --git a/test/test_lyrics.py b/test/test_lyrics.py index 1c3b6f6a1..59c0dcdba 100644 --- a/test/test_lyrics.py +++ b/test/test_lyrics.py @@ -195,14 +195,8 @@ class MockFetchUrl(object): def is_lyrics_content_ok(title, text): """Compare lyrics text to expected lyrics for given title""" - setexpected = set(LYRICS_TEXTS[lyrics.slugify(title)].split()) - settext = set(text.split()) - setinter = setexpected.intersection(settext) - # consider lyrics ok if they share 50% or more with the reference - if len(setinter): - ratio = 1.0 * max(len(setexpected), len(settext)) / len(setinter) - return (ratio > .5 and ratio < 2.5) - return False + keywords = LYRICS_TEXTS[lyrics.slugify(title)] + return all(x in text.lower() for x in keywords) LYRICS_ROOT_DIR = os.path.join(_common.RSRC, 'lyrics') LYRICS_TEXTS = confit.load_yaml(os.path.join(_common.RSRC, 'lyricstext.yaml')) @@ -215,6 +209,7 @@ DEFAULT_SOURCES = [ path=u'lady-madonna-lyrics-the-beatles.html') ] + # Every source entered in default beets google custom search engine # must be listed below. # Use default query when possible, or override artist and title fields @@ -235,9 +230,6 @@ GOOGLE_SOURCES = [ dict(url=u'http://www.lacoccinelle.net', artist=u'Jacques Brel', title=u"Amsterdam", path=u'/paroles-officielles/275679.html'), - dict(DEFAULT_SONG, - url=u'http://www.lyrics007.com', - path=u'/The%20Beatles%20Lyrics/Lady%20Madonna%20Lyrics.html'), dict(DEFAULT_SONG, url='http://www.lyricsmania.com/', path='lady_madonna_lyrics_the_beatles.html'), @@ -276,7 +268,7 @@ GOOGLE_SOURCES = [ class LyricsGooglePluginTest(unittest.TestCase): """Test scraping heuristics on a fake html page. - Use `nosetests -s test_lyrics.py -a '!slow'` to check that beets google + Or run lyrics_download_samples.py first to check that beets google custom search engine sources are correctly scraped. """ source = dict(url=u'http://www.example.com', artist=u'John Doe',