From 9f3e5b28b4d3cbacc5d9f08647ef20d3bbca5084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 14 Jul 2017 15:31:17 -0400 Subject: [PATCH 01/22] output lyrics in HTML, allow skipping the idea here is to format the lyrics output a little better so that it can (for example) be shown as a web page or an ebook. the new skip option allows for faster generation of the output in the (most common) case where not all lyrics are available. --- beetsplug/lyrics.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 113bed104..f90159708 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -595,6 +595,7 @@ class LyricsPlugin(plugins.BeetsPlugin): "76V-uFL5jks5dNvcGCdarqFjDhP9c", 'fallback': None, 'force': False, + 'skip': False, 'sources': self.SOURCES, }) self.config['bing_client_secret'].redact = True @@ -663,18 +664,33 @@ class LyricsPlugin(plugins.BeetsPlugin): action='store_true', default=False, help=u'always re-download lyrics', ) + cmd.parser.add_option( + u'-s', u'--skip', dest='skip_fetched', + action='store_true', default=False, + help=u'skip already fetched lyrics', + ) def func(lib, opts, args): # The "write to files" option corresponds to the # import_write config value. write = ui.should_write() + artist = '' + album = '' for item in lib.items(ui.decargs(args)): - self.fetch_item_lyrics( - lib, item, write, - opts.force_refetch or self.config['force'], - ) + if not opts.skip_fetched and not self.config['skip']: + self.fetch_item_lyrics( + lib, item, write, + opts.force_refetch or self.config['force'], + ) if opts.printlyr and item.lyrics: - ui.print_(item.lyrics) + if artist != item.artist: + artist = item.artist + ui.print_('

' + artist + '

') + if album != item.album: + album = item.album + ui.print_('

' + artist + '

') + ui.print_('

' + item.title + '

') + ui.print_('
' + item.lyrics + '
') cmd.func = func return [cmd] From 0fbfa1feae1397affc388743d56c49879970f502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 14 Jul 2017 17:34:51 -0400 Subject: [PATCH 02/22] render RST instead of HTML ReStructuredText has the advantage over HTML that it can be rendered easily to multiple formats (HTML, ePUB, PDF) and it supports indexes. the output needs to be fed into a file and integrated into an existing Sphinx document, of course. --- beetsplug/lyrics.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index f90159708..3c0afab46 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -659,6 +659,11 @@ class LyricsPlugin(plugins.BeetsPlugin): action='store_true', default=False, help=u'print lyrics to console', ) + cmd.parser.add_option( + u'-r', u'--print-rst', dest='printrst', + action='store_true', default=False, + help=u'print lyrics to console as RST text', + ) cmd.parser.add_option( u'-f', u'--force', dest='force_refetch', action='store_true', default=False, @@ -682,15 +687,27 @@ class LyricsPlugin(plugins.BeetsPlugin): lib, item, write, opts.force_refetch or self.config['force'], ) - if opts.printlyr and item.lyrics: - if artist != item.artist: - artist = item.artist - ui.print_('

' + artist + '

') - if album != item.album: - album = item.album - ui.print_('

' + artist + '

') - ui.print_('

' + item.title + '

') - ui.print_('
' + item.lyrics + '
') + if item.lyrics: + if opts.printlyr: + ui.print_(item.lyrics) + if opts.printrst: + if artist != item.artist: + artist = item.artist + ui.print_(artist) + ui.print_(u'=' * len(artist)) + ui.print_() + if album != item.album: + album = item.album + ui.print_(album) + ui.print_(u'-' * len(album)) + ui.print_() + title_str = u':index:`' + item.title + u'`' + ui.print_(title_str) + ui.print_(u'~' * len(title_str)) + ui.print_() + # turn lyrics into a line block + ui.print_(u'| ' + item.lyrics.replace(u'\n', u'\n| ')) + ui.print_() cmd.func = func return [cmd] From 2fa46a750735f82350037ce21b48665f7b63fe71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 14 Jul 2017 17:43:37 -0400 Subject: [PATCH 03/22] add documentation for new lyrics flags --- docs/plugins/lyrics.rst | 49 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index d7c268c7e..454f16d57 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -84,10 +84,55 @@ lyrics will be added to the beets database and, if ``import.write`` is on, embedded into files' metadata. The ``-p`` option to the ``lyrics`` command makes it print lyrics out to the -console so you can view the fetched (or previously-stored) lyrics. +console so you can view the fetched (or previously-stored) lyrics. The +``-r`` option similarly shows all lyrics as an RST (ReStructuredText) +document. That document, in turn, can be parsed by tools like Sphinx +to generate HTML, ePUB or PDF formatted documents. Use, for example, +the following ``conf.py``:: + + # -*- coding: utf-8 -*- + master_doc = 'index' + project = u'Lyrics' + copyright = u'none' + author = u'Various Authors' + latex_documents = [ + (master_doc, 'Lyrics.tex', project, + author, 'manual'), + ] + epub_title = project + epub_author = author + epub_publisher = author + epub_copyright = copyright + epub_exclude_files = ['search.html'] + epub_tocdepth = 1 + epub_tocdup = False + +Then the output can be written to ``index.rst``. An alternative is to +use the following ``index.rst`` file, which will also generate an +index of song titles:: + + Lyrics + ====== + + * :ref:`Song index ` + * :ref:`search` + + Artist index: + + .. toctree:: + :maxdepth: 1 + + artists + +Then the correct format can be generated with one of:: + + sphinx-build -b epub3 . _build/epub + sphinx-build -b latex . _build/latex + sphinx-build -b html . _build/html The ``-f`` option forces the command to fetch lyrics, even for tracks that -already have lyrics. +already have lyrics. Inversely, the ``-s`` option skips lyrics that +are not locally available, to dump lyrics faster. .. _activate-google-custom-search: From 63aa3b316514062a53dec185f06c1c4201eb16cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 00:50:52 -0400 Subject: [PATCH 04/22] write to separate rst files this makes the ePUB easier to parse by e-readers, because they do not need to load one giant HTML file, but one per author. it also makes sphinx rendering more efficient and interactive --- beetsplug/lyrics.py | 58 +++++++++++++++++++++++++++++------------ docs/plugins/lyrics.rst | 5 ++-- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 3c0afab46..42ddef025 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -22,12 +22,15 @@ import difflib import itertools import json import struct +import os.path import re import requests import unicodedata +from unidecode import unidecode import warnings import six from six.moves import urllib +import sys try: from bs4 import SoupStrainer, BeautifulSoup @@ -660,9 +663,9 @@ class LyricsPlugin(plugins.BeetsPlugin): help=u'print lyrics to console', ) cmd.parser.add_option( - u'-r', u'--print-rst', dest='printrst', - action='store_true', default=False, - help=u'print lyrics to console as RST text', + u'-r', u'--write-rst', dest='writerst', + action='store', default='.', + help=u'write lyrics to given directory as RST files', ) cmd.parser.add_option( u'-f', u'--force', dest='force_refetch', @@ -679,8 +682,9 @@ class LyricsPlugin(plugins.BeetsPlugin): # The "write to files" option corresponds to the # import_write config value. write = ui.should_write() - artist = '' - album = '' + artist = u'Unknown artist' + album = False + output = sys.stdout for item in lib.items(ui.decargs(args)): if not opts.skip_fetched and not self.config['skip']: self.fetch_item_lyrics( @@ -690,24 +694,44 @@ class LyricsPlugin(plugins.BeetsPlugin): if item.lyrics: if opts.printlyr: ui.print_(item.lyrics) - if opts.printrst: + if opts.writerst: if artist != item.artist: artist = item.artist - ui.print_(artist) - ui.print_(u'=' * len(artist)) - ui.print_() + if output != sys.stdout: + output.close() + slug = re.sub(r'\W+', '-', + unidecode(artist).lower()) + path = os.path.join(opts.writerst, slug + u'.rst') + output = open(path, 'w') + output.write(artist.encode('utf-8')) + output.write(u'\n') + output.write(u'=' * len(artist)) + output.write(u'\n') + output.write(u''' +.. contents:: + :local: +''') + output.write(u'\n') if album != item.album: album = item.album - ui.print_(album) - ui.print_(u'-' * len(album)) - ui.print_() + output.write(album.encode('utf-8')) + output.write(u'\n') + output.write(u'-' * len(album)) + output.write(u'\n') + output.write(u'\n') title_str = u':index:`' + item.title + u'`' - ui.print_(title_str) - ui.print_(u'~' * len(title_str)) - ui.print_() + output.write(title_str.encode('utf-8')) + output.write(u'\n') + output.write(u'~' * len(title_str)) + output.write(u'\n') + output.write(u'\n') # turn lyrics into a line block - ui.print_(u'| ' + item.lyrics.replace(u'\n', u'\n| ')) - ui.print_() + block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') + output.write(block.encode('utf-8')) + output.write(u'\n') + output.write(u'\n') + if opts.writerst: + output.close() cmd.func = func return [cmd] diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 454f16d57..2fe7422f9 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -85,8 +85,9 @@ embedded into files' metadata. The ``-p`` option to the ``lyrics`` command makes it print lyrics out to the console so you can view the fetched (or previously-stored) lyrics. The -``-r`` option similarly shows all lyrics as an RST (ReStructuredText) -document. That document, in turn, can be parsed by tools like Sphinx +``-r directory`` option similarly shows all lyrics as an RST (ReStructuredText) +document structure located in ``directory`` (which defaults to the +current directory). That document, in turn, can be parsed by tools like Sphinx to generate HTML, ePUB or PDF formatted documents. Use, for example, the following ``conf.py``:: From 469c03a7bfc43957d5e4f768fbc363f06ec04240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 01:13:18 -0400 Subject: [PATCH 05/22] deal properly with empty album titles --- beetsplug/lyrics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 42ddef025..17c226589 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -713,10 +713,12 @@ class LyricsPlugin(plugins.BeetsPlugin): ''') output.write(u'\n') if album != item.album: - album = item.album - output.write(album.encode('utf-8')) + tmpalbum = album = item.album + if album == '': + tmpalbum = 'Unknown album' + output.write(tmpalbum.encode('utf-8')) output.write(u'\n') - output.write(u'-' * len(album)) + output.write(u'-' * len(tmpalbum)) output.write(u'\n') output.write(u'\n') title_str = u':index:`' + item.title + u'`' From ac32ae574cb789aa17f2abf243946d7165b489b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 09:21:54 -0400 Subject: [PATCH 06/22] optimize: write only 3 times per file this makes the code more readable and reduces the number of syscalls to write files --- beetsplug/lyrics.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 17c226589..959506e99 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -703,35 +703,27 @@ class LyricsPlugin(plugins.BeetsPlugin): unidecode(artist).lower()) path = os.path.join(opts.writerst, slug + u'.rst') output = open(path, 'w') - output.write(artist.encode('utf-8')) - output.write(u'\n') - output.write(u'=' * len(artist)) - output.write(u'\n') - output.write(u''' + rst = u'''%s +%s + .. contents:: :local: -''') - output.write(u'\n') + +''' % (artist, u'=' * len(artist)) + output.write(rst.encode('utf-8')) if album != item.album: tmpalbum = album = item.album if album == '': - tmpalbum = 'Unknown album' - output.write(tmpalbum.encode('utf-8')) - output.write(u'\n') - output.write(u'-' * len(tmpalbum)) - output.write(u'\n') - output.write(u'\n') - title_str = u':index:`' + item.title + u'`' - output.write(title_str.encode('utf-8')) - output.write(u'\n') - output.write(u'~' * len(title_str)) - output.write(u'\n') - output.write(u'\n') - # turn lyrics into a line block + tmpalbum = u'Unknown album' + rst = u"%s\n%s\n\n" % (tmpalbum, + u'-' * len(tmpalbum)) + output.write(rst.encode('utf-8')) + title_str = u":index:`%s`" % item.title block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') - output.write(block.encode('utf-8')) - output.write(u'\n') - output.write(u'\n') + rst = u"%s\n%s\n\n%s\n" % (title_str, + u'~' * len(title_str), + block) + output.write(rst.encode('utf-8')) if opts.writerst: output.close() From 32de4148bc4509c37315a10046edc0cc36e10fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 09:30:14 -0400 Subject: [PATCH 07/22] use glob patterns correctly in index --- docs/plugins/lyrics.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 2fe7422f9..f7f9c2189 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -122,8 +122,9 @@ index of song titles:: .. toctree:: :maxdepth: 1 + :glob: - artists + artists/* Then the correct format can be generated with one of:: From d330353e1c1d5cdc261b5ed2be286c7500bf087d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 14:19:25 -0400 Subject: [PATCH 08/22] rename the skip option to local skip was a misnomer: we actually skip "unfetched" lyrics. this means it's somewhat of a double-negative and really confusing. --local is clearer, although less in opposition with --force --- beetsplug/lyrics.py | 8 ++++---- docs/plugins/lyrics.rst | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 959506e99..38c0f73fb 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -598,7 +598,7 @@ class LyricsPlugin(plugins.BeetsPlugin): "76V-uFL5jks5dNvcGCdarqFjDhP9c", 'fallback': None, 'force': False, - 'skip': False, + 'local': False, 'sources': self.SOURCES, }) self.config['bing_client_secret'].redact = True @@ -673,9 +673,9 @@ class LyricsPlugin(plugins.BeetsPlugin): help=u'always re-download lyrics', ) cmd.parser.add_option( - u'-s', u'--skip', dest='skip_fetched', + u'-l', u'--local', dest='local_only', action='store_true', default=False, - help=u'skip already fetched lyrics', + help=u'do not fetch missing lyrics', ) def func(lib, opts, args): @@ -686,7 +686,7 @@ class LyricsPlugin(plugins.BeetsPlugin): album = False output = sys.stdout for item in lib.items(ui.decargs(args)): - if not opts.skip_fetched and not self.config['skip']: + if not opts.local_only and not self.config['local']: self.fetch_item_lyrics( lib, item, write, opts.force_refetch or self.config['force'], diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index f7f9c2189..5682ddbc0 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -133,8 +133,9 @@ Then the correct format can be generated with one of:: sphinx-build -b html . _build/html The ``-f`` option forces the command to fetch lyrics, even for tracks that -already have lyrics. Inversely, the ``-s`` option skips lyrics that -are not locally available, to dump lyrics faster. +already have lyrics. Inversely, the ``-l`` option restricts operations +to lyrics that are locally available, to show lyrics faster without +retrying them over the network all the time. .. _activate-google-custom-search: From 91de8aac84eb3058cf44d7e5b3f86970adc28a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 14:46:26 -0400 Subject: [PATCH 09/22] move rst writer to a different function this simplifies and clarifies the code, although we need to call the writerst function twice to wrap up at the end of the loop --- beetsplug/lyrics.py | 78 ++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 38c0f73fb..f32e34bfd 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function import difflib +import errno import itertools import json import struct @@ -30,7 +31,6 @@ from unidecode import unidecode import warnings import six from six.moves import urllib -import sys try: from bs4 import SoupStrainer, BeautifulSoup @@ -605,6 +605,14 @@ class LyricsPlugin(plugins.BeetsPlugin): self.config['google_API_key'].redact = True self.config['google_engine_ID'].redact = True self.config['genius_api_key'].redact = True + # state information for the RST writer + # the current artist + self.artist = u'Unknown artist' + # the current album, False means no album yet + self.album = False + # the current rst file content. None means the file is not + # open yet. + self.rst = None available_sources = list(self.SOURCES) sources = plugins.sanitize_choices( @@ -682,9 +690,6 @@ class LyricsPlugin(plugins.BeetsPlugin): # The "write to files" option corresponds to the # import_write config value. write = ui.should_write() - artist = u'Unknown artist' - album = False - output = sys.stdout for item in lib.items(ui.decargs(args)): if not opts.local_only and not self.config['local']: self.fetch_item_lyrics( @@ -695,40 +700,47 @@ class LyricsPlugin(plugins.BeetsPlugin): if opts.printlyr: ui.print_(item.lyrics) if opts.writerst: - if artist != item.artist: - artist = item.artist - if output != sys.stdout: - output.close() - slug = re.sub(r'\W+', '-', - unidecode(artist).lower()) - path = os.path.join(opts.writerst, slug + u'.rst') - output = open(path, 'w') - rst = u'''%s + self.writerst(opts.writerst, item) + if opts.writerst: + # flush last artist + self.writerst(opts.writerst, None) + cmd.func = func + return [cmd] + + def writerst(self, directory, item): + """Write the item to an RST file + + this will keep state (in the `rst` variable) in order to avoid + writing continuously to the same files + """ + if item is None or self.artist != item.artist: + if self.rst is not None: + slug = re.sub(r'\W+', '-', unidecode(self.artist).lower()) + path = os.path.join(directory, 'artists', slug + u'.rst') + with open(path, 'w') as output: + output.write(self.rst.encode('utf-8')) + self.rst = None + if item is None: + return + self.artist = item.artist + self.rst = u'''%s %s .. contents:: :local: -''' % (artist, u'=' * len(artist)) - output.write(rst.encode('utf-8')) - if album != item.album: - tmpalbum = album = item.album - if album == '': - tmpalbum = u'Unknown album' - rst = u"%s\n%s\n\n" % (tmpalbum, - u'-' * len(tmpalbum)) - output.write(rst.encode('utf-8')) - title_str = u":index:`%s`" % item.title - block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') - rst = u"%s\n%s\n\n%s\n" % (title_str, - u'~' * len(title_str), - block) - output.write(rst.encode('utf-8')) - if opts.writerst: - output.close() - - cmd.func = func - return [cmd] +''' % (self.artist, u'=' * len(self.artist)) + if self.album != item.album: + tmpalbum = self.album = item.album + if self.album == '': + tmpalbum = u'Unknown album' + self.rst += u"%s\n%s\n\n" % (tmpalbum, + u'-' * len(tmpalbum)) + title_str = u":index:`%s`" % item.title + block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') + self.rst += u"%s\n%s\n\n%s\n" % (title_str, + u'~' * len(title_str), + block) def imported(self, session, task): """Import hook for fetching lyrics automatically. From e6adb5e7da94fa3ff3175c2a00fbf6bbe67153ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 14:47:36 -0400 Subject: [PATCH 10/22] cosmetic: do not use needless heredoc --- beetsplug/lyrics.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index f32e34bfd..a1af67363 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -723,13 +723,8 @@ class LyricsPlugin(plugins.BeetsPlugin): if item is None: return self.artist = item.artist - self.rst = u'''%s -%s - -.. contents:: - :local: - -''' % (self.artist, u'=' * len(self.artist)) + self.rst = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ + % (self.artist, u'=' * len(self.artist)) if self.album != item.album: tmpalbum = self.album = item.album if self.album == '': From f6674287589c489417a0bc99306b12f735cab89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 15:43:13 -0400 Subject: [PATCH 11/22] write sphinx base files we write the artists files in a subdirectory, to avoid infinite recursions or flooding the current directory needlessly. this way, the user has a good base structure and can just chain the command into sphinx to continue building the next format, after possible tweaks. --- beetsplug/lyrics.py | 60 +++++++++++++++++++++++++++++++++ docs/plugins/lyrics.rst | 75 +++++++++++++++-------------------------- 2 files changed, 88 insertions(+), 47 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index a1af67363..29bab25a6 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -690,6 +690,8 @@ class LyricsPlugin(plugins.BeetsPlugin): # The "write to files" option corresponds to the # import_write config value. write = ui.should_write() + if opts.writerst: + self.writerst_indexes(opts.writerst) for item in lib.items(ui.decargs(args)): if not opts.local_only and not self.config['local']: self.fetch_item_lyrics( @@ -704,6 +706,13 @@ class LyricsPlugin(plugins.BeetsPlugin): if opts.writerst: # flush last artist self.writerst(opts.writerst, None) + ui.print_(u'RST files generated. to build, use one of:') + ui.print_(u' sphinx-build -b html %s _build/html' + % opts.writerst) + ui.print_(u' sphinx-build -b epub %s _build/epub' + % opts.writerst) + ui.print_(u' sphinx-build -b latex %s _build/latex && make -C _build/latex all-pdf' + % opts.writerst) cmd.func = func return [cmd] @@ -737,6 +746,57 @@ class LyricsPlugin(plugins.BeetsPlugin): u'~' * len(title_str), block) + def writerst_indexes(self, directory): + """Write conf.py and index.rst files necessary for Sphinx + + We write minimal configurations that are necessary for Sphinx + to operate. We do not overwrite existing files so that + customizations are respected.""" + try: + os.makedirs(os.path.join(directory, 'artists')) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + indexfile = os.path.join(directory, 'index.rst') + if not os.path.exists(indexfile): + with open(indexfile, 'w') as output: + output.write(u'''Lyrics +====== + +* :ref:`Song index ` +* :ref:`search` + +Artist index: + +.. toctree:: + :maxdepth: 1 + :glob: + + artists/* +''') + conffile = os.path.join(directory, 'conf.py') + if not os.path.exists(conffile): + with open(conffile, 'w') as output: + output.write(u'''# -*- coding: utf-8 -*- +master_doc = 'index' +project = u'Lyrics' +copyright = u'none' +author = u'Various Authors' +latex_documents = [ + (master_doc, 'Lyrics.tex', project, + author, 'manual'), +] +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright +epub_exclude_files = ['search.html'] +epub_tocdepth = 1 +epub_tocdup = False +''') + def imported(self, session, task): """Import hook for fetching lyrics automatically. """ diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 5682ddbc0..0c6d68546 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -84,59 +84,40 @@ lyrics will be added to the beets database and, if ``import.write`` is on, embedded into files' metadata. The ``-p`` option to the ``lyrics`` command makes it print lyrics out to the -console so you can view the fetched (or previously-stored) lyrics. The -``-r directory`` option similarly shows all lyrics as an RST (ReStructuredText) -document structure located in ``directory`` (which defaults to the -current directory). That document, in turn, can be parsed by tools like Sphinx -to generate HTML, ePUB or PDF formatted documents. Use, for example, -the following ``conf.py``:: - - # -*- coding: utf-8 -*- - master_doc = 'index' - project = u'Lyrics' - copyright = u'none' - author = u'Various Authors' - latex_documents = [ - (master_doc, 'Lyrics.tex', project, - author, 'manual'), - ] - epub_title = project - epub_author = author - epub_publisher = author - epub_copyright = copyright - epub_exclude_files = ['search.html'] - epub_tocdepth = 1 - epub_tocdup = False - -Then the output can be written to ``index.rst``. An alternative is to -use the following ``index.rst`` file, which will also generate an -index of song titles:: - - Lyrics - ====== - - * :ref:`Song index ` - * :ref:`search` - - Artist index: - - .. toctree:: - :maxdepth: 1 - :glob: - - artists/* - -Then the correct format can be generated with one of:: - - sphinx-build -b epub3 . _build/epub - sphinx-build -b latex . _build/latex - sphinx-build -b html . _build/html +console so you can view the fetched (or previously-stored) lyrics. The ``-f`` option forces the command to fetch lyrics, even for tracks that already have lyrics. Inversely, the ``-l`` option restricts operations to lyrics that are locally available, to show lyrics faster without retrying them over the network all the time. +Rendering lyrics into other formats +----------------------------------- + +The ``-r directory`` option similarly renders all lyrics as an RST +(ReStructuredText) document structure located in ``directory`` (which +defaults to the current directory). That directory, in turn, can be +parsed by tools like Sphinx to generate HTML, ePUB or PDF formatted +documents. A minimal ``conf.py`` and ``index.rst`` files are created +the first time the command is ran, to provide templates that can be +modified. They are not overwritten on subsequent runs. + +Sphinx supports various `builders +`_, but here are a +few suggestions. + + * build a HTML version:: + + sphinx-build -b html . _build/html + + * build an ePUB3 formatted file, usable on ebook-readers:: + + sphinx-build -b epub3 . _build/epub + + * build a PDF file, which incidentally also builds a LaTeX file:: + + sphinx-build -b latex %s _build/latex && make -C _build/latex all-pdf + .. _activate-google-custom-search: Activate Google custom search From 0bcd16f1ab04cd77319ad5f2eb1f44308f674cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 16:21:41 -0400 Subject: [PATCH 12/22] deal with encoding issues in python3 when we encode explicitly, we return bytes, so open files as binary --- beetsplug/lyrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 29bab25a6..c36839b29 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -726,7 +726,7 @@ class LyricsPlugin(plugins.BeetsPlugin): if self.rst is not None: slug = re.sub(r'\W+', '-', unidecode(self.artist).lower()) path = os.path.join(directory, 'artists', slug + u'.rst') - with open(path, 'w') as output: + with open(path, 'wb') as output: output.write(self.rst.encode('utf-8')) self.rst = None if item is None: @@ -761,7 +761,7 @@ class LyricsPlugin(plugins.BeetsPlugin): raise indexfile = os.path.join(directory, 'index.rst') if not os.path.exists(indexfile): - with open(indexfile, 'w') as output: + with open(indexfile, 'wb') as output: output.write(u'''Lyrics ====== @@ -778,7 +778,7 @@ Artist index: ''') conffile = os.path.join(directory, 'conf.py') if not os.path.exists(conffile): - with open(conffile, 'w') as output: + with open(conffile, 'wb') as output: output.write(u'''# -*- coding: utf-8 -*- master_doc = 'index' project = u'Lyrics' From 5d8c15980ef5fa854ab17dba5ab83489d640bead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 15 Jul 2017 16:24:07 -0400 Subject: [PATCH 13/22] fix flake8 warning --- beetsplug/lyrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index c36839b29..73e7e11b1 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -711,7 +711,8 @@ class LyricsPlugin(plugins.BeetsPlugin): % opts.writerst) ui.print_(u' sphinx-build -b epub %s _build/epub' % opts.writerst) - ui.print_(u' sphinx-build -b latex %s _build/latex && make -C _build/latex all-pdf' + ui.print_((u' sphinx-build -b latex %s _build/latex ' + u'&& make -C _build/latex all-pdf') % opts.writerst) cmd.func = func return [cmd] From c655a13c4103308dea3ae936198c21ddb733d502 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 16 Jul 2017 10:02:14 -0400 Subject: [PATCH 14/22] Use title case in docs --- docs/plugins/lyrics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 0c6d68546..872c95b5c 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -91,7 +91,7 @@ already have lyrics. Inversely, the ``-l`` option restricts operations to lyrics that are locally available, to show lyrics faster without retrying them over the network all the time. -Rendering lyrics into other formats +Rendering Lyrics into Other Formats ----------------------------------- The ``-r directory`` option similarly renders all lyrics as an RST @@ -120,7 +120,7 @@ few suggestions. .. _activate-google-custom-search: -Activate Google custom search +Activate Google Custom Search ------------------------------ Using the Google backend requires `BeautifulSoup`_, which you can install From d44eda56c50d1cbe6ad70406717ab3c44e4e80d0 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 16 Jul 2017 10:07:32 -0400 Subject: [PATCH 15/22] Docs refinements for lyrics ReST output --- docs/plugins/lyrics.rst | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/plugins/lyrics.rst b/docs/plugins/lyrics.rst index 872c95b5c..85e3021f9 100644 --- a/docs/plugins/lyrics.rst +++ b/docs/plugins/lyrics.rst @@ -88,33 +88,37 @@ console so you can view the fetched (or previously-stored) lyrics. The ``-f`` option forces the command to fetch lyrics, even for tracks that already have lyrics. Inversely, the ``-l`` option restricts operations -to lyrics that are locally available, to show lyrics faster without -retrying them over the network all the time. +to lyrics that are locally available, which show lyrics faster without using +the network at all. Rendering Lyrics into Other Formats ----------------------------------- -The ``-r directory`` option similarly renders all lyrics as an RST -(ReStructuredText) document structure located in ``directory`` (which -defaults to the current directory). That directory, in turn, can be -parsed by tools like Sphinx to generate HTML, ePUB or PDF formatted -documents. A minimal ``conf.py`` and ``index.rst`` files are created -the first time the command is ran, to provide templates that can be -modified. They are not overwritten on subsequent runs. +The ``-r directory`` option renders all lyrics as `reStructuredText`_ (ReST) +documents in ``directory`` (by default, the current directory). That +directory, in turn, can be parsed by tools like `Sphinx`_ to generate HTML, +ePUB, or PDF documents. + +A minimal ``conf.py`` and ``index.rst`` files are created the first time the +command is run. They are not overwritten on subsequent runs, so you can safely +modify these files to customize the output. + +.. _Sphinx: http://www.sphinx-doc.org/ +.. _reStructuredText: http://docutils.sourceforge.net/rst.html Sphinx supports various `builders `_, but here are a few suggestions. - * build a HTML version:: + * Build an HTML version:: sphinx-build -b html . _build/html - * build an ePUB3 formatted file, usable on ebook-readers:: + * Build an ePUB3 formatted file, usable on ebook readers:: - sphinx-build -b epub3 . _build/epub + sphinx-build -b epub3 . _build/epub - * build a PDF file, which incidentally also builds a LaTeX file:: + * Build a PDF file, which incidentally also builds a LaTeX file:: sphinx-build -b latex %s _build/latex && make -C _build/latex all-pdf From 813cf97686bcaebce2ee5c68f99681713cd039c9 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 16 Jul 2017 10:10:41 -0400 Subject: [PATCH 16/22] Better metavariable for lyrics --help output --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 73e7e11b1..7fb13150b 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -672,7 +672,7 @@ class LyricsPlugin(plugins.BeetsPlugin): ) cmd.parser.add_option( u'-r', u'--write-rst', dest='writerst', - action='store', default='.', + action='store', default='.', metavar='directory', help=u'write lyrics to given directory as RST files', ) cmd.parser.add_option( From 9de94378b91fb619206ab5356ea080965233ba6b Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 16 Jul 2017 10:14:49 -0400 Subject: [PATCH 17/22] An even shorter metavariable --- beetsplug/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 7fb13150b..841847c8c 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -672,7 +672,7 @@ class LyricsPlugin(plugins.BeetsPlugin): ) cmd.parser.add_option( u'-r', u'--write-rst', dest='writerst', - action='store', default='.', metavar='directory', + action='store', default='.', metavar='dir', help=u'write lyrics to given directory as RST files', ) cmd.parser.add_option( From cf75b1548e48f91cabceadd7c3faa8eab589e30f Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Sun, 16 Jul 2017 10:15:45 -0400 Subject: [PATCH 18/22] Changelog entry for #2628 --- docs/changelog.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 35785cc3c..e30080f49 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,11 @@ Changelog 1.4.6 (in development) ---------------------- -Changelog goes here! +New features: + +* :doc:`/plugins/lyrics`: The plugin can now produce reStructuredText files + for beautiful, readable books of lyrics. Thanks to :user:`anarcat`. + :bug:`2628` Fixes: From 7e0a48a46d7fe7b3d920db3f7e52cd9ca4978ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 17 Jul 2017 08:49:40 -0400 Subject: [PATCH 19/22] s/rest/rest/ --- beetsplug/lyrics.py | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 841847c8c..9dba98a39 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -605,14 +605,14 @@ class LyricsPlugin(plugins.BeetsPlugin): self.config['google_API_key'].redact = True self.config['google_engine_ID'].redact = True self.config['genius_api_key'].redact = True - # state information for the RST writer + # state information for the ReST writer # the current artist self.artist = u'Unknown artist' # the current album, False means no album yet self.album = False - # the current rst file content. None means the file is not + # the current rest file content. None means the file is not # open yet. - self.rst = None + self.rest = None available_sources = list(self.SOURCES) sources = plugins.sanitize_choices( @@ -671,9 +671,9 @@ class LyricsPlugin(plugins.BeetsPlugin): help=u'print lyrics to console', ) cmd.parser.add_option( - u'-r', u'--write-rst', dest='writerst', + u'-r', u'--write-rest', dest='writerest', action='store', default='.', metavar='dir', - help=u'write lyrics to given directory as RST files', + help=u'write lyrics to given directory as ReST files', ) cmd.parser.add_option( u'-f', u'--force', dest='force_refetch', @@ -690,8 +690,8 @@ class LyricsPlugin(plugins.BeetsPlugin): # The "write to files" option corresponds to the # import_write config value. write = ui.should_write() - if opts.writerst: - self.writerst_indexes(opts.writerst) + if opts.writerest: + self.writerest_indexes(opts.writerest) for item in lib.items(ui.decargs(args)): if not opts.local_only and not self.config['local']: self.fetch_item_lyrics( @@ -701,53 +701,53 @@ class LyricsPlugin(plugins.BeetsPlugin): if item.lyrics: if opts.printlyr: ui.print_(item.lyrics) - if opts.writerst: - self.writerst(opts.writerst, item) - if opts.writerst: + if opts.writerest: + self.writerest(opts.writerest, item) + if opts.writerest: # flush last artist - self.writerst(opts.writerst, None) - ui.print_(u'RST files generated. to build, use one of:') + self.writerest(opts.writerest, None) + ui.print_(u'ReST files generated. to build, use one of:') ui.print_(u' sphinx-build -b html %s _build/html' - % opts.writerst) + % opts.writerest) ui.print_(u' sphinx-build -b epub %s _build/epub' - % opts.writerst) + % opts.writerest) ui.print_((u' sphinx-build -b latex %s _build/latex ' u'&& make -C _build/latex all-pdf') - % opts.writerst) + % opts.writerest) cmd.func = func return [cmd] - def writerst(self, directory, item): - """Write the item to an RST file + def writerest(self, directory, item): + """Write the item to an ReST file - this will keep state (in the `rst` variable) in order to avoid - writing continuously to the same files + This will keep state (in the `rest` variable) in order to avoid + writing continuously to the same files. """ if item is None or self.artist != item.artist: - if self.rst is not None: + if self.rest is not None: slug = re.sub(r'\W+', '-', unidecode(self.artist).lower()) path = os.path.join(directory, 'artists', slug + u'.rst') with open(path, 'wb') as output: - output.write(self.rst.encode('utf-8')) - self.rst = None + output.write(self.rest.encode('utf-8')) + self.rest = None if item is None: return self.artist = item.artist - self.rst = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ - % (self.artist, u'=' * len(self.artist)) + self.rest = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ + % (self.artist, u'=' * len(self.artist)) if self.album != item.album: tmpalbum = self.album = item.album if self.album == '': tmpalbum = u'Unknown album' - self.rst += u"%s\n%s\n\n" % (tmpalbum, - u'-' * len(tmpalbum)) + self.rest += u"%s\n%s\n\n" % (tmpalbum, + u'-' * len(tmpalbum)) title_str = u":index:`%s`" % item.title block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') - self.rst += u"%s\n%s\n\n%s\n" % (title_str, - u'~' * len(title_str), - block) + self.rest += u"%s\n%s\n\n%s\n" % (title_str, + u'~' * len(title_str), + block) - def writerst_indexes(self, directory): + def writerest_indexes(self, directory): """Write conf.py and index.rst files necessary for Sphinx We write minimal configurations that are necessary for Sphinx From 6d58110bd2e46b32f36ddbcd588ef137f8915274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 17 Jul 2017 08:50:19 -0400 Subject: [PATCH 20/22] move heredocs to top-level globals --- beetsplug/lyrics.py | 68 ++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 9dba98a39..07e865c91 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -77,6 +77,41 @@ URL_CHARACTERS = { } USER_AGENT = 'beets/{}'.format(beets.__version__) +# the content for the base index.rst generated +REST_INDEX_TEMPLATE = u'''Lyrics +====== + +* :ref:`Song index ` +* :ref:`search` + +Artist index: + +.. toctree:: + :maxdepth: 1 + :glob: + + artists/* +''' + +# the content for the base conf.py generated +REST_CONF_TEMPLATE = u'''# -*- coding: utf-8 -*- +master_doc = 'index' +project = u'Lyrics' +copyright = u'none' +author = u'Various Authors' +latex_documents = [ + (master_doc, 'Lyrics.tex', project, + author, 'manual'), +] +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright +epub_exclude_files = ['search.html'] +epub_tocdepth = 1 +epub_tocdup = False +''' + # Utilities. @@ -763,40 +798,11 @@ class LyricsPlugin(plugins.BeetsPlugin): indexfile = os.path.join(directory, 'index.rst') if not os.path.exists(indexfile): with open(indexfile, 'wb') as output: - output.write(u'''Lyrics -====== - -* :ref:`Song index ` -* :ref:`search` - -Artist index: - -.. toctree:: - :maxdepth: 1 - :glob: - - artists/* -''') + output.write(REST_INDEX_TEMPLATE) conffile = os.path.join(directory, 'conf.py') if not os.path.exists(conffile): with open(conffile, 'wb') as output: - output.write(u'''# -*- coding: utf-8 -*- -master_doc = 'index' -project = u'Lyrics' -copyright = u'none' -author = u'Various Authors' -latex_documents = [ - (master_doc, 'Lyrics.tex', project, - author, 'manual'), -] -epub_title = project -epub_author = author -epub_publisher = author -epub_copyright = copyright -epub_exclude_files = ['search.html'] -epub_tocdepth = 1 -epub_tocdup = False -''') + output.write(REST_CONF_TEMPLATE) def imported(self, session, task): """Import hook for fetching lyrics automatically. From b6e42ee2e886da109cbc1262e8558ab61966082b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 17 Jul 2017 08:55:07 -0400 Subject: [PATCH 21/22] fix another unicode error the unicode strings are not binary - rely on Python to do the right thing here instead of encoding a string we know is already properly encoded --- beetsplug/lyrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index 07e865c91..e19014d11 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -797,11 +797,11 @@ class LyricsPlugin(plugins.BeetsPlugin): raise indexfile = os.path.join(directory, 'index.rst') if not os.path.exists(indexfile): - with open(indexfile, 'wb') as output: + with open(indexfile, 'w') as output: output.write(REST_INDEX_TEMPLATE) conffile = os.path.join(directory, 'conf.py') if not os.path.exists(conffile): - with open(conffile, 'wb') as output: + with open(conffile, 'w') as output: output.write(REST_CONF_TEMPLATE) def imported(self, session, task): From 93966ed4ee134cc7f727636ea4e5203e230b1e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 17 Jul 2017 09:00:22 -0400 Subject: [PATCH 22/22] strip whitespace in titles this would cause problems with songs that had trailing spaces with the index directive --- beetsplug/lyrics.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/beetsplug/lyrics.py b/beetsplug/lyrics.py index e19014d11..7c81d3a81 100644 --- a/beetsplug/lyrics.py +++ b/beetsplug/lyrics.py @@ -760,7 +760,8 @@ class LyricsPlugin(plugins.BeetsPlugin): """ if item is None or self.artist != item.artist: if self.rest is not None: - slug = re.sub(r'\W+', '-', unidecode(self.artist).lower()) + slug = re.sub(r'\W+', '-', + unidecode(self.artist.strip()).lower()) path = os.path.join(directory, 'artists', slug + u'.rst') with open(path, 'wb') as output: output.write(self.rest.encode('utf-8')) @@ -769,14 +770,15 @@ class LyricsPlugin(plugins.BeetsPlugin): return self.artist = item.artist self.rest = u"%s\n%s\n\n.. contents::\n :local:\n\n" \ - % (self.artist, u'=' * len(self.artist)) + % (self.artist.strip(), + u'=' * len(self.artist.strip())) if self.album != item.album: tmpalbum = self.album = item.album if self.album == '': tmpalbum = u'Unknown album' - self.rest += u"%s\n%s\n\n" % (tmpalbum, - u'-' * len(tmpalbum)) - title_str = u":index:`%s`" % item.title + self.rest += u"%s\n%s\n\n" % (tmpalbum.strip(), + u'-' * len(tmpalbum.strip())) + title_str = u":index:`%s`" % item.title.strip() block = u'| ' + item.lyrics.replace(u'\n', u'\n| ') self.rest += u"%s\n%s\n\n%s\n" % (title_str, u'~' * len(title_str),