From c5075b28551d8552d2dd35fd8f3d08969f24906c Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 18:24:59 +0200 Subject: [PATCH 1/6] Create a cached template() function We were previously doing calls to Template() directly, sometimes in a loop. This caused the same template to be recompiled over and over. This commit introduces a function template() which caches the results, so that multiple calls with the same template string does not require recompilation. --- beets/dbcore/db.py | 4 ++-- beets/library.py | 6 +++--- beets/ui/__init__.py | 4 ++-- beets/util/functemplate.py | 4 ++++ docs/changelog.rst | 3 +++ 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/beets/dbcore/db.py b/beets/dbcore/db.py index 97a4a7ce3..3195b52c9 100755 --- a/beets/dbcore/db.py +++ b/beets/dbcore/db.py @@ -25,7 +25,7 @@ import sqlite3 import contextlib import beets -from beets.util.functemplate import Template +from beets.util import functemplate from beets.util import py3_path from beets.dbcore import types from .query import MatchQuery, NullSort, TrueQuery @@ -597,7 +597,7 @@ class Model(object): """ # Perform substitution. if isinstance(template, six.string_types): - template = Template(template) + template = functemplate.template(template) return template.substitute(self.formatted(for_path), self._template_funcs()) diff --git a/beets/library.py b/beets/library.py index d49d67227..5786ce9d9 100644 --- a/beets/library.py +++ b/beets/library.py @@ -31,7 +31,7 @@ from beets import plugins from beets import util from beets.util import bytestring_path, syspath, normpath, samefile, \ MoveOperation -from beets.util.functemplate import Template +from beets.util.functemplate import template, Template from beets import dbcore from beets.dbcore import types import beets @@ -855,7 +855,7 @@ class Item(LibModel): if isinstance(path_format, Template): subpath_tmpl = path_format else: - subpath_tmpl = Template(path_format) + subpath_tmpl = template(path_format) # Evaluate the selected template. subpath = self.evaluate_template(subpath_tmpl, True) @@ -1134,7 +1134,7 @@ class Album(LibModel): image = bytestring_path(image) item_dir = item_dir or self.item_dir() - filename_tmpl = Template( + filename_tmpl = template( beets.config['art_filename'].as_str()) subpath = self.evaluate_template(filename_tmpl, True) if beets.config['asciify_paths']: diff --git a/beets/ui/__init__.py b/beets/ui/__init__.py index 327db6b04..622a1e7f0 100644 --- a/beets/ui/__init__.py +++ b/beets/ui/__init__.py @@ -36,7 +36,7 @@ from beets import logging from beets import library from beets import plugins from beets import util -from beets.util.functemplate import Template +from beets.util.functemplate import template from beets import config from beets.util import confit, as_string from beets.autotag import mb @@ -616,7 +616,7 @@ def get_path_formats(subview=None): subview = subview or config['paths'] for query, view in subview.items(): query = PF_KEY_QUERIES.get(query, query) # Expand common queries. - path_formats.append((query, Template(view.as_str()))) + path_formats.append((query, template(view.as_str()))) return path_formats diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 6a34a3bb3..57ea1a394 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -35,6 +35,7 @@ import dis import types import sys import six +import functools SYMBOL_DELIM = u'$' FUNC_DELIM = u'%' @@ -552,6 +553,9 @@ def _parse(template): parts.append(remainder) return Expression(parts) +@functools.lru_cache(maxsize=128) +def template(fmt): + return Template(fmt) # External interface. diff --git a/docs/changelog.rst b/docs/changelog.rst index d88bf0b12..085345353 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -117,6 +117,9 @@ Some improvements have been focused on improving beets' performance: to be displayed. Thanks to :user:`pprkut`. :bug:`3089` +* Querying the library was further improved by reusing compiled teamplates + instead of compiling them over and over again. + Thanks to :user:`SimonPersson`. * :doc:`/plugins/absubmit`, :doc:`/plugins/badfiles`: Analysis now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. From 7df4e23b134c20fa292dbe4c35b333002e5f33f0 Mon Sep 17 00:00:00 2001 From: Simon Persson Date: Thu, 9 May 2019 19:27:31 +0200 Subject: [PATCH 2/6] Fix formatting, and add python2 support. --- beets/util/functemplate.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 57ea1a394..9c625b7f9 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -553,12 +553,21 @@ def _parse(template): parts.append(remainder) return Expression(parts) -@functools.lru_cache(maxsize=128) + +# Decorator that enables lru_cache on py3, and no caching on py2. +def cached(func): + if six.PY2: + # Sorry python2 users, no caching for you :( + return func + return functools.lru_cache(maxsize=128)(func) + + +@cached def template(fmt): return Template(fmt) -# External interface. +# External interface. class Template(object): """A string template, including text, Symbols, and Calls. """ From 0d190e7fad17d335b9bba17741167b2b3a9fc77f Mon Sep 17 00:00:00 2001 From: David Logie Date: Thu, 9 May 2019 19:08:43 +0100 Subject: [PATCH 3/6] Use NullPaddedInt for all r128_album_gain fields. --- beets/library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beets/library.py b/beets/library.py index d49d67227..e3e2a6876 100644 --- a/beets/library.py +++ b/beets/library.py @@ -935,7 +935,7 @@ class Album(LibModel): 'releasegroupdisambig': types.STRING, 'rg_album_gain': types.NULL_FLOAT, 'rg_album_peak': types.NULL_FLOAT, - 'r128_album_gain': types.PaddedInt(6), + 'r128_album_gain': types.NullPaddedInt(6), 'original_year': types.PaddedInt(4), 'original_month': types.PaddedInt(2), 'original_day': types.PaddedInt(2), From d236e1edff56255e5632c3dbbf4c9e4b2cb3a16c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 9 May 2019 14:17:11 -0400 Subject: [PATCH 4/6] Changelog refinement for #3258 --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 085345353..8608f1b6f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -117,9 +117,10 @@ Some improvements have been focused on improving beets' performance: to be displayed. Thanks to :user:`pprkut`. :bug:`3089` -* Querying the library was further improved by reusing compiled teamplates - instead of compiling them over and over again. +* Another query optimization works by compiling templates once and reusing + them instead of recompiling them to print out each matching object. Thanks to :user:`SimonPersson`. + :bug:`3258` * :doc:`/plugins/absubmit`, :doc:`/plugins/badfiles`: Analysis now works in parallel (on Python 3 only). Thanks to :user:`bemeurer`. From ff1d43ddf92f5c5a85d72766e01208566bfc1d1c Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 9 May 2019 14:21:38 -0400 Subject: [PATCH 5/6] Refine @cached decorator from #3258 Don't restrict to Python 2 precisely. --- beets/util/functemplate.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/beets/util/functemplate.py b/beets/util/functemplate.py index 9c625b7f9..5d9900f0b 100644 --- a/beets/util/functemplate.py +++ b/beets/util/functemplate.py @@ -554,12 +554,15 @@ def _parse(template): return Expression(parts) -# Decorator that enables lru_cache on py3, and no caching on py2. def cached(func): - if six.PY2: - # Sorry python2 users, no caching for you :( + """Like the `functools.lru_cache` decorator, but works (as a no-op) + on Python < 3.2. + """ + if hasattr(functools, 'lru_cache'): + return functools.lru_cache(maxsize=128)(func) + else: + # Do nothing when lru_cache is not available. return func - return functools.lru_cache(maxsize=128)(func) @cached From 23da057cebf35e82193cdf4b9e56064ae49b6f41 Mon Sep 17 00:00:00 2001 From: Adrian Sampson Date: Thu, 9 May 2019 14:22:48 -0400 Subject: [PATCH 6/6] tox: Don't use Python 3.8 by default 3.8.0 final is not released yet. This default set is meant to be a reasonable list for quick iteration during development. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e3250bd6b..8736f0f3c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27-test, py37-test, py38-test, py27-flake8, docs +envlist = py27-test, py37-test, py27-flake8, docs # The exhaustive list of environments is: # envlist = py{27,34,35}-{test,cov}, py{27,34,35}-flake8, docs