Move script to download pages out of tests_lyrics.py

By default (as runned by CI tools), only *fake* example.com page is present in
rsr/lyrics and tests that check content of pages coming from *real* sources are
thus skipped.
Execute lyrics_download_samples.py to download pages from *real* sources. When
done and *real* pages are present on disk, no tests are skipped.
This commit is contained in:
Fabrice Laporte 2014-11-08 07:25:26 +01:00
parent 356c1f44b5
commit f5e7bd5d05
2 changed files with 54 additions and 24 deletions

View file

@ -0,0 +1,54 @@
# This file is part of beets.
# Copyright 2014, Fabrice Laporte
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
import os
import sys
import requests
import test_lyrics
def mkdir_p(path):
try:
os.makedirs(path)
except OSError:
if os.path.isdir(path):
pass
else:
raise
def safe_open_w(path):
''' Open "path" for writing, creating any parent directories as needed.
'''
mkdir_p(os.path.dirname(path))
return open(path, 'w')
def main(argv=None):
"""download"""
if argv is None:
argv = sys.argv
for s in test_lyrics.SOURCES:
url = s['url'] + s['path']
fn = test_lyrics.url_to_filename(url)
if not os.path.isfile(fn):
html = requests.get(url).text
with safe_open_w(fn) as f:
f.write(html.encode('utf8'))
if __name__ == "__main__":
sys.exit(main())

View file

@ -172,22 +172,6 @@ def url_to_filename(url):
return fn
def mkdir_p(path):
try:
os.makedirs(path)
except OSError:
if os.path.isdir(path):
pass
else:
raise
def safe_open_w(path):
''' Open "path" for writing, creating any parent directories as needed.
'''
mkdir_p(os.path.dirname(path))
return open(path, 'w')
class MockFetchUrl(object):
def __init__(self, pathval='fetched_path'):
@ -282,14 +266,6 @@ SOURCES = [
]
def download_source_sample(url):
fn = url_to_filename(url)
if not os.path.isfile(fn):
html = requests.get(url).text
with safe_open_w(fn) as f:
f.write(html.encode('utf8'))
class LyricsGooglePluginTest(unittest.TestCase):
"""Test scraping heuristics on a fake html page.
Use `nosetests -s test_lyrics.py -a '!slow'` to check that beets google