From 581bf768ca9c8daee2e93ad4b97e86b726c936d9 Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 3 May 2014 13:55:21 +0200 Subject: [PATCH 01/12] add buckets plugin + tests Add a new template functions %bucket(text, field) for path formatting. --- beetsplug/bucket.py | 115 ++++++++++++++++++++++++++++++++++++++++++++ test/test_bucket.py | 76 +++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 beetsplug/bucket.py create mode 100644 test/test_bucket.py diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py new file mode 100644 index 000000000..1b8eec4d7 --- /dev/null +++ b/beetsplug/bucket.py @@ -0,0 +1,115 @@ +# This file is part of beets. +# Copyright 2014, Fabrice Laporte. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Enrich path formatting with %bucket_alpha and %bucket_date functions +""" + +from datetime import datetime +import logging +import re +import string +from beets import plugins +# from beets import config + +log = logging.getLogger('beets') + + +def extract_years(lst): + """Extract years from a list of strings""" + + def make_date(s): + """Convert string representing a year to int + """ + d = int(s) + if d < 100: # two digits imply it is 20th century + d = 1900 + d + return d + + res = [] + for bucket in lst: + yearspan_str = re.findall('\d+', bucket) + yearspan = [make_date(x) for x in yearspan_str] + res.append(yearspan) + return res + + +class BucketPlugin(plugins.BeetsPlugin): + def __init__(self): + super(BucketPlugin, self).__init__() + self.template_funcs['bucket'] = self._tmpl_bucket + + self.config.add({ + 'bucket_year': [], + 'bucket_alpha': [], + }) + self.setup() + + def setup(self): + """Setup plugin from config options + """ + + yearranges = extract_years(self.config['bucket_year'].get()) + self.yearbounds = sorted([y for ys in yearranges for y in ys]) + self.yearranges = [self.make_year_range(b) for b in yearranges] + self.alpharanges = [self.make_alpha_range(b) for b in + self.config['bucket_alpha'].get()] + log.debug(self.alpharanges) + + def make_year_range(self, ys): + """Express year-span as a list of years [from...to]. + If input year-span only contain the from year, the to is defined + as the from year of the next year-span minus one. + """ + if len(ys) == 1: # miss upper bound + lb_idx = self.yearbounds.index(ys[0]) + try: + ys.append(self.yearbounds[lb_idx + 1]) + except: + ys.append(datetime.now().year + 1) + return range(ys[0], ys[1]) + + def make_alpha_range(self, s): + """Express chars range as a list of chars [from...to] + """ + bucket = sorted([x for x in s.lower() if x.isalnum()]) + beginIdx = string.ascii_lowercase.index(bucket[0]) + endIdx = string.ascii_lowercase.index(bucket[-1]) + return string.ascii_lowercase[beginIdx:endIdx + 1] + + def find_bucket_timerange(self, date): + """Find folder whose range contains date + 1960-1970 + 60s-70s + """ + for (i, r) in enumerate(self.yearranges): + if int(date) in r: + return self.config['bucket_year'].get()[i] + return date + + def find_bucket_alpha(self, s): + for (i, r) in enumerate(self.alpharanges): + if s.lower()[0] in r: + return self.config['bucket_alpha'].get()[i] + return s[0].upper() + + def _tmpl_bucket(self, text, field=None): + if not field and text.isdigit(): + field = 'year' + + if field == 'year': + func = self.find_bucket_timerange + else: + func = self.find_bucket_alpha + + return func(text) diff --git a/test/test_bucket.py b/test/test_bucket.py new file mode 100644 index 000000000..1a45bc2c2 --- /dev/null +++ b/test/test_bucket.py @@ -0,0 +1,76 @@ +# This file is part of beets. +# Copyright 2014, Fabrice Laporte. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +"""Tests for the 'bucket' plugin.""" + +from _common import unittest +from beetsplug import bucket +from beets import config + +from helper import TestHelper + + +class BucketPluginTest(unittest.TestCase, TestHelper): + def setUp(self): + self.setup_beets() + self.plugin = bucket.BucketPlugin() + + def tearDown(self): + self.teardown_beets() + + def _setup_config(self, bucket_year=[], bucket_alpha=[]): + config['bucket']['bucket_year'] = bucket_year + config['bucket']['bucket_alpha'] = bucket_alpha + self.plugin.setup() + + def test_year_single_year(self): + """If a single year is given, folder represents a range from this year + to the next 'from year' of next folder.""" + self._setup_config(bucket_year=['50', '70']) + + self.assertEqual(self.plugin._tmpl_bucket('1959'), '50') + self.assertEqual(self.plugin._tmpl_bucket('1969'), '50') + + def test_year_single_year_last_folder(self): + """Last folder of a range extends from its year to current year.""" + self._setup_config(bucket_year=['50', '70']) + self.assertEqual(self.plugin._tmpl_bucket('1999'), '70') + + def test_year_two_years(self): + self._setup_config(bucket_year=['50-59', '1960-69']) + self.assertEqual(self.plugin._tmpl_bucket('1954'), '50-59') + + def test_year_out_of_range(self): + """If no range match, return the year""" + self._setup_config(bucket_year=['50-59', '1960-69']) + self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') + + def test_alpha_all_chars(self): + self._setup_config(bucket_alpha=['ABCD', 'FGH', 'IJKL']) + self.assertEqual(self.plugin._tmpl_bucket('garry'), 'FGH') + + def test_alpha_first_last_chars(self): + self._setup_config(bucket_alpha=['A-D', 'F-H', 'I-Z']) + self.assertEqual(self.plugin._tmpl_bucket('garry'), 'F-H') + + def test_alpha_out_of_range(self): + """If no range match, return the initial""" + self.assertEqual(self.plugin._tmpl_bucket('errol'), 'E') + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 31569baba79fece6cc0e67f3b0f65c744198fb9c Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 3 May 2014 23:46:31 +0200 Subject: [PATCH 02/12] fix range upper bound + tests added --- beetsplug/bucket.py | 6 ++---- test/test_bucket.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index 1b8eec4d7..a814f86c2 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -64,7 +64,6 @@ class BucketPlugin(plugins.BeetsPlugin): self.yearranges = [self.make_year_range(b) for b in yearranges] self.alpharanges = [self.make_alpha_range(b) for b in self.config['bucket_alpha'].get()] - log.debug(self.alpharanges) def make_year_range(self, ys): """Express year-span as a list of years [from...to]. @@ -76,8 +75,8 @@ class BucketPlugin(plugins.BeetsPlugin): try: ys.append(self.yearbounds[lb_idx + 1]) except: - ys.append(datetime.now().year + 1) - return range(ys[0], ys[1]) + ys.append(datetime.now().year) + return range(ys[0], ys[-1] + 1) def make_alpha_range(self, s): """Express chars range as a list of chars [from...to] @@ -111,5 +110,4 @@ class BucketPlugin(plugins.BeetsPlugin): func = self.find_bucket_timerange else: func = self.find_bucket_alpha - return func(text) diff --git a/test/test_bucket.py b/test/test_bucket.py index 1a45bc2c2..3b16420f4 100644 --- a/test/test_bucket.py +++ b/test/test_bucket.py @@ -45,11 +45,17 @@ class BucketPluginTest(unittest.TestCase, TestHelper): def test_year_single_year_last_folder(self): """Last folder of a range extends from its year to current year.""" self._setup_config(bucket_year=['50', '70']) - self.assertEqual(self.plugin._tmpl_bucket('1999'), '70') + self.assertEqual(self.plugin._tmpl_bucket('2014'), '70') + self.assertEqual(self.plugin._tmpl_bucket('2015'), '2015') def test_year_two_years(self): self._setup_config(bucket_year=['50-59', '1960-69']) - self.assertEqual(self.plugin._tmpl_bucket('1954'), '50-59') + self.assertEqual(self.plugin._tmpl_bucket('1959'), '50-59') + + def test_year_multiple_years(self): + self._setup_config(bucket_year=['1950,51,52,53']) + self.assertEqual(self.plugin._tmpl_bucket('1953'), '1950,51,52,53') + self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') def test_year_out_of_range(self): """If no range match, return the year""" From b1d10ed5f9b23f909ec263a41d670e5b42d6cc31 Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 3 May 2014 23:48:23 +0200 Subject: [PATCH 03/12] add bucket plugin doc --- docs/plugins/bucket.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/plugins/bucket.rst diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst new file mode 100644 index 000000000..1537477f8 --- /dev/null +++ b/docs/plugins/bucket.rst @@ -0,0 +1,34 @@ +Bucket Plugin +============== + +The ``bucket`` plugin helps you keep a balanced files tree for your library +by gathering your files into buckets folders representing ranges. +This kind of files organization is usually used to classify your music by +periods (eg *1960s*, *1970s* etc), or to divide bloated folders into smaller +subfolders by grouping albums/artist alphabetically (eg *A-F*, *G-M*, *N-Z*). +To use plugin, enable it by including `bucket` into `plugins` line of your +beets config. The plugin provides a template function called `%bucket` for +use in path format expressions:: + + paths: + default: /%bucket($year)/%bucket($artist)/$albumartist-$album-$year + +You must then define what ranges representations you allow in the `bucket:` +section in the config file : + + bucket: + bucket_alpha: ['A-F', 'G-M', 'N-Z'] + bucket_year: ['1980s', '1990s', '2000s'] + +The `bucket_year` parameter is used for all substitutions occuring on the +`$year` field, while `bucket_alpha` takes care of the others textual fields. + +The definition of a range is somewhat loose, and multiple formats are allowed : + +- for alpha ranges: the range is defined by the lowest and highest (ascii-wise) +alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. +- for year ranges: digits characters are extracted, and in case of doubt XXth +century is assumed. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are + equivalent. +If no upper bound is given, the range is extended to current year (unless a +later range is defined). eg *'1975'* encompasses all years from 1975 until now. From 67b3d78b72cac356e555be46b54cfe7468aba3a7 Mon Sep 17 00:00:00 2001 From: "Fabrice L." Date: Sun, 4 May 2014 00:04:18 +0200 Subject: [PATCH 04/12] Update bucket.rst --- docs/plugins/bucket.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst index 1537477f8..3a232e9fb 100644 --- a/docs/plugins/bucket.rst +++ b/docs/plugins/bucket.rst @@ -29,6 +29,6 @@ The definition of a range is somewhat loose, and multiple formats are allowed : alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. - for year ranges: digits characters are extracted, and in case of doubt XXth century is assumed. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are - equivalent. -If no upper bound is given, the range is extended to current year (unless a + equivalent. If no upper bound is given, the range is extended to current year (unless a later range is defined). eg *'1975'* encompasses all years from 1975 until now. + From 8a7f82f354d4fd3635a898e1cc66938cd64aed8c Mon Sep 17 00:00:00 2001 From: "Fabrice L." Date: Sun, 4 May 2014 07:09:36 +0200 Subject: [PATCH 05/12] Update bucket.rst --- docs/plugins/bucket.rst | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst index 3a232e9fb..a38b52078 100644 --- a/docs/plugins/bucket.rst +++ b/docs/plugins/bucket.rst @@ -6,29 +6,25 @@ by gathering your files into buckets folders representing ranges. This kind of files organization is usually used to classify your music by periods (eg *1960s*, *1970s* etc), or to divide bloated folders into smaller subfolders by grouping albums/artist alphabetically (eg *A-F*, *G-M*, *N-Z*). -To use plugin, enable it by including `bucket` into `plugins` line of your -beets config. The plugin provides a template function called `%bucket` for +To use plugin, enable it by including ``bucket`` into ``plugins`` line of your +beets config. The plugin provides a template function called ``%bucket`` for use in path format expressions:: paths: default: /%bucket($year)/%bucket($artist)/$albumartist-$album-$year -You must then define what ranges representations you allow in the `bucket:` -section in the config file : +You must then define what ranges representations you allow in the ``bucket:`` +section of the config file : bucket: - bucket_alpha: ['A-F', 'G-M', 'N-Z'] + bucket_alpha: ['A-F', 'G-M', 'N-Z'] bucket_year: ['1980s', '1990s', '2000s'] -The `bucket_year` parameter is used for all substitutions occuring on the -`$year` field, while `bucket_alpha` takes care of the others textual fields. +The ``bucket_year`` parameter is used for all substitutions occuring on the +``$year`` field, while ``bucket_alpha`` takes care of the others textual fields. The definition of a range is somewhat loose, and multiple formats are allowed : -- for alpha ranges: the range is defined by the lowest and highest (ascii-wise) -alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. -- for year ranges: digits characters are extracted, and in case of doubt XXth -century is assumed. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are - equivalent. If no upper bound is given, the range is extended to current year (unless a -later range is defined). eg *'1975'* encompasses all years from 1975 until now. +- for alpha ranges: the range is defined by the lowest and highest (ascii-wise) alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. +- for year ranges: digits characters are extracted, and in case of doubt XXth century is assumed. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are equivalent. If no upper bound is given, the range is extended to current year (unless a later range is defined). eg *'1975'* encompasses all years from 1975 until now. From a138c4989e031ab93da2232ef953cb924edb4a4a Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sun, 4 May 2014 07:41:02 +0200 Subject: [PATCH 06/12] include bucket.rst in toc tree --- docs/plugins/bucket.rst | 6 +++--- docs/plugins/index.rst | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst index a38b52078..0302ed005 100644 --- a/docs/plugins/bucket.rst +++ b/docs/plugins/bucket.rst @@ -2,7 +2,7 @@ Bucket Plugin ============== The ``bucket`` plugin helps you keep a balanced files tree for your library -by gathering your files into buckets folders representing ranges. +by grouping your files into buckets folders representing ranges. This kind of files organization is usually used to classify your music by periods (eg *1960s*, *1970s* etc), or to divide bloated folders into smaller subfolders by grouping albums/artist alphabetically (eg *A-F*, *G-M*, *N-Z*). @@ -17,7 +17,7 @@ You must then define what ranges representations you allow in the ``bucket:`` section of the config file : bucket: - bucket_alpha: ['A-F', 'G-M', 'N-Z'] + bucket_alpha: ['A-F', 'G-M', 'N-Z'] bucket_year: ['1980s', '1990s', '2000s'] The ``bucket_year`` parameter is used for all substitutions occuring on the @@ -25,6 +25,6 @@ The ``bucket_year`` parameter is used for all substitutions occuring on the The definition of a range is somewhat loose, and multiple formats are allowed : -- for alpha ranges: the range is defined by the lowest and highest (ascii-wise) alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. +- for alpha ranges: the range is defined by the lowest and highest (ascii-wise) alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. - for year ranges: digits characters are extracted, and in case of doubt XXth century is assumed. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are equivalent. If no upper bound is given, the range is extended to current year (unless a later range is defined). eg *'1975'* encompasses all years from 1975 until now. diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 05a0b7aef..f620927bc 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -58,6 +58,7 @@ by typing ``beet version``. fromfilename ftintitle keyfinder + bucket Autotagger Extensions --------------------- @@ -102,6 +103,8 @@ Path Formats * :doc:`rewrite`: Substitute values in path formats. * :doc:`the`: Move patterns in path formats (i.e., move "a" and "the" to the end). +* :doc:`bucket`: Group your files into bucket directories that cover different +field values ranges. Interoperability ---------------- From 1e7e456f213d409b18c8ab46ea074e33217aba80 Mon Sep 17 00:00:00 2001 From: "Fabrice L." Date: Sun, 4 May 2014 08:33:04 +0200 Subject: [PATCH 07/12] Update index.rst indent line --- docs/plugins/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index f620927bc..eab7a69b9 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -104,7 +104,7 @@ Path Formats * :doc:`the`: Move patterns in path formats (i.e., move "a" and "the" to the end). * :doc:`bucket`: Group your files into bucket directories that cover different -field values ranges. + field values ranges. Interoperability ---------------- From 509af59d4ed3db685dc3681c811bb0cd936c3dba Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sun, 4 May 2014 08:51:11 +0200 Subject: [PATCH 08/12] cleanup docstrings and add tests --- beetsplug/bucket.py | 20 +++++++++++--------- test/test_bucket.py | 5 +++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index a814f86c2..5473a4b65 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -20,7 +20,6 @@ import logging import re import string from beets import plugins -# from beets import config log = logging.getLogger('beets') @@ -58,7 +57,6 @@ class BucketPlugin(plugins.BeetsPlugin): def setup(self): """Setup plugin from config options """ - yearranges = extract_years(self.config['bucket_year'].get()) self.yearbounds = sorted([y for ys in yearranges for y in ys]) self.yearranges = [self.make_year_range(b) for b in yearranges] @@ -66,9 +64,10 @@ class BucketPlugin(plugins.BeetsPlugin): self.config['bucket_alpha'].get()] def make_year_range(self, ys): - """Express year-span as a list of years [from...to]. - If input year-span only contain the from year, the to is defined - as the from year of the next year-span minus one. + """Express year-range as a list of years [from...to]. + If input year-range only contains the 'from' year, the 'to' is + defined as the 'from' year of the next year-range minus one or is + set to current year if there is no next year-range. """ if len(ys) == 1: # miss upper bound lb_idx = self.yearbounds.index(ys[0]) @@ -79,7 +78,8 @@ class BucketPlugin(plugins.BeetsPlugin): return range(ys[0], ys[-1] + 1) def make_alpha_range(self, s): - """Express chars range as a list of chars [from...to] + """Extract alphanumerics from string and return sorted list of chars + [from...to] """ bucket = sorted([x for x in s.lower() if x.isalnum()]) beginIdx = string.ascii_lowercase.index(bucket[0]) @@ -87,9 +87,8 @@ class BucketPlugin(plugins.BeetsPlugin): return string.ascii_lowercase[beginIdx:endIdx + 1] def find_bucket_timerange(self, date): - """Find folder whose range contains date - 1960-1970 - 60s-70s + """Return year-range bucket that matches given date or return the date + if no matching bucket. """ for (i, r) in enumerate(self.yearranges): if int(date) in r: @@ -97,6 +96,9 @@ class BucketPlugin(plugins.BeetsPlugin): return date def find_bucket_alpha(self, s): + """Return alpha-range bucket that matches given string or return the + string initial if no matching bucket. + """ for (i, r) in enumerate(self.alpharanges): if s.lower()[0] in r: return self.config['bucket_alpha'].get()[i] diff --git a/test/test_bucket.py b/test/test_bucket.py index 3b16420f4..b76bd973d 100644 --- a/test/test_bucket.py +++ b/test/test_bucket.py @@ -61,6 +61,8 @@ class BucketPluginTest(unittest.TestCase, TestHelper): """If no range match, return the year""" self._setup_config(bucket_year=['50-59', '1960-69']) self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') + self._setup_config(bucket_year=[]) + self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') def test_alpha_all_chars(self): self._setup_config(bucket_alpha=['ABCD', 'FGH', 'IJKL']) @@ -72,6 +74,9 @@ class BucketPluginTest(unittest.TestCase, TestHelper): def test_alpha_out_of_range(self): """If no range match, return the initial""" + self._setup_config(bucket_alpha=['ABCD', 'FGH', 'IJKL']) + self.assertEqual(self.plugin._tmpl_bucket('errol'), 'E') + self._setup_config(bucket_alpha=[]) self.assertEqual(self.plugin._tmpl_bucket('errol'), 'E') From 4add189608c19ff70847c763b30f9015adc981a9 Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Thu, 8 May 2014 01:23:32 +0200 Subject: [PATCH 09/12] Add option to extrapolate year buckets names - spans are now tracked via a list of dicts instead of 2 lists previously (simpler code) - extend_year_spans() pregenerates all possible ranges at plugin setup stage - a BucketError is now raised if declared bucket format not accepted --- beetsplug/bucket.py | 200 ++++++++++++++++++++++++++++++---------- docs/plugins/bucket.rst | 18 +++- test/test_bucket.py | 52 ++++++++--- 3 files changed, 205 insertions(+), 65 deletions(-) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index 5473a4b65..9a6939601 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -12,37 +12,152 @@ # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -"""Enrich path formatting with %bucket_alpha and %bucket_date functions +"""Provides %bucket_alpha and %bucket_year functions for path formatting. """ from datetime import datetime import logging import re import string +from itertools import tee, izip from beets import plugins log = logging.getLogger('beets') -def extract_years(lst): - """Extract years from a list of strings""" +class BucketError(Exception): + pass - def make_date(s): - """Convert string representing a year to int + +def pairwise(iterable): + "s -> (s0,s1), (s1,s2), (s2, s3), ..." + a, b = tee(iterable) + next(b, None) + return izip(a, b) + + +def span_from_str(span_str): + """Build a span dict from the span string representation. + """ + + def normalize_year(d, yearfrom): + """Convert string to a 4 digits year """ - d = int(s) - if d < 100: # two digits imply it is 20th century - d = 1900 + d + if yearfrom < 100: + raise BucketError("Bucket 'from' year %d must be expressed on 4 " + "digits" % yearfrom) + # if two digits only, pick closest year that ends by these two + # digits starting from yearfrom + if d < 100: + if (d % 100) < (yearfrom % 100): + d = (yearfrom - yearfrom % 100) + 100 + d + else: + d = (yearfrom - yearfrom % 100) + d return d - res = [] - for bucket in lst: - yearspan_str = re.findall('\d+', bucket) - yearspan = [make_date(x) for x in yearspan_str] - res.append(yearspan) + years = [int(x) for x in re.findall('\d+', span_str)] + years = [normalize_year(x, years[0]) for x in years] + + res = {'from': years[0], 'str': span_str} + if len(years) > 1: + res['to'] = years[-1] return res +def complete_year_spans(spans): + """Set the `to` value of spans if empty and sort them chronologically. + """ + spans.sort(key=lambda x: x['from']) + for (x, y) in pairwise(spans): + if 'to' not in x: + x['to'] = y['from'] - 1 + if spans and 'to' not in spans[-1]: + spans[-1]['to'] = datetime.now().year + + +def extend_year_spans(spans, spanlen, start=1900, end=2014): + """Add new spans to given spans list so that every year of [start,end] + belongs to a span. + """ + extended_spans = spans[:] + for (x, y) in pairwise(spans): + # if a gap between two spans, fill the gap with as much spans of + # spanlen length as necessary + for span_from in range(x['to'] + 1, y['from'], spanlen): + extended_spans.append({'from': span_from}) + # Create spans prior to declared ones + for span_from in range(spans[0]['from'] - spanlen, start, -spanlen): + extended_spans.append({'from': span_from}) + # Create spans after the declared ones + for span_from in range(spans[-1]['to'] + 1, end, spanlen): + extended_spans.append({'from': span_from}) + + complete_year_spans(extended_spans) + return extended_spans + + +def build_year_spans(year_spans_str): + """Build a chronologically ordered list of spans dict from unordered spans + stringlist. + """ + spans = [] + for elem in year_spans_str: + spans.append(span_from_str(elem)) + complete_year_spans(spans) + return spans + + +def str2fmt(s): + """Deduces formatting syntax from a span string. + """ + regex = re.compile("(?P\D*)(?P\d+)(?P\D*)" + "(?P\d*)(?P\D*)") + m = re.match(regex, s) + + def year_format(year): + return '%%0%dd' % len(year) + + res = {'fromnchars': len(m.group('fromyear')), + 'tonchars': len(m.group('toyear'))} + res['fmt'] = "%s%%s%s%s%s" % (m.group('bef'), + m.group('sep'), + '%s' if res['tonchars'] else '', + m.group('after')) + return res + + +def format_span(fmt, yearfrom, yearto, fromnchars, tonchars): + """Return a span string representation. + """ + args = (str(yearfrom)[-fromnchars:]) + if tonchars: + args = (str(yearfrom)[-fromnchars:], str(yearto)[-tonchars:]) + return fmt % args + + +def extract_modes(spans): + """Extract the most common spans lengths and representation formats + """ + rangelen = sorted([x['to'] - x['from'] + 1 for x in spans]) + deflen = sorted(rangelen, key=rangelen.count)[-1] + reprs = [str2fmt(x['str']) for x in spans] + deffmt = sorted(reprs, key=reprs.count)[-1] + return deflen, deffmt + + +def build_alpha_spans(alpha_spans_str): + """Extract alphanumerics from string and return sorted list of chars + [from...to] + """ + spans = [] + for elem in alpha_spans_str: + bucket = sorted([x for x in elem.lower() if x.isalnum()]) + beginIdx = string.ascii_lowercase.index(bucket[0]) + endIdx = string.ascii_lowercase.index(bucket[-1]) + spans.append(string.ascii_lowercase[beginIdx:endIdx + 1]) + return spans + + class BucketPlugin(plugins.BeetsPlugin): def __init__(self): super(BucketPlugin, self).__init__() @@ -51,56 +166,43 @@ class BucketPlugin(plugins.BeetsPlugin): self.config.add({ 'bucket_year': [], 'bucket_alpha': [], + 'extrapolate': False }) self.setup() def setup(self): """Setup plugin from config options """ - yearranges = extract_years(self.config['bucket_year'].get()) - self.yearbounds = sorted([y for ys in yearranges for y in ys]) - self.yearranges = [self.make_year_range(b) for b in yearranges] - self.alpharanges = [self.make_alpha_range(b) for b in - self.config['bucket_alpha'].get()] + self.year_spans = build_year_spans(self.config['bucket_year'].get()) + if self.year_spans and self.config['extrapolate']: + [self.ys_len_mode, + self.ys_repr_mode] = extract_modes(self.year_spans) + self.year_spans = extend_year_spans(self.year_spans, + self.ys_len_mode) - def make_year_range(self, ys): - """Express year-range as a list of years [from...to]. - If input year-range only contains the 'from' year, the 'to' is - defined as the 'from' year of the next year-range minus one or is - set to current year if there is no next year-range. - """ - if len(ys) == 1: # miss upper bound - lb_idx = self.yearbounds.index(ys[0]) - try: - ys.append(self.yearbounds[lb_idx + 1]) - except: - ys.append(datetime.now().year) - return range(ys[0], ys[-1] + 1) + self.alpha_spans = build_alpha_spans(self.config['bucket_alpha'].get()) - def make_alpha_range(self, s): - """Extract alphanumerics from string and return sorted list of chars - [from...to] - """ - bucket = sorted([x for x in s.lower() if x.isalnum()]) - beginIdx = string.ascii_lowercase.index(bucket[0]) - endIdx = string.ascii_lowercase.index(bucket[-1]) - return string.ascii_lowercase[beginIdx:endIdx + 1] - - def find_bucket_timerange(self, date): - """Return year-range bucket that matches given date or return the date + def find_bucket_year(self, year): + """Return bucket that matches given year or return the year if no matching bucket. """ - for (i, r) in enumerate(self.yearranges): - if int(date) in r: - return self.config['bucket_year'].get()[i] - return date + for ys in self.year_spans: + if ys['from'] <= int(year) <= ys['to']: + if 'str' in ys: + return ys['str'] + else: + return format_span(self.ys_repr_mode['fmt'], + ys['from'], ys['to'], + self.ys_repr_mode['fromnchars'], + self.ys_repr_mode['tonchars']) + return year def find_bucket_alpha(self, s): """Return alpha-range bucket that matches given string or return the string initial if no matching bucket. """ - for (i, r) in enumerate(self.alpharanges): - if s.lower()[0] in r: + for (i, span) in enumerate(self.alpha_spans): + if s.lower()[0] in span: return self.config['bucket_alpha'].get()[i] return s[0].upper() @@ -109,7 +211,7 @@ class BucketPlugin(plugins.BeetsPlugin): field = 'year' if field == 'year': - func = self.find_bucket_timerange + func = self.find_bucket_year else: func = self.find_bucket_alpha return func(text) diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst index 0302ed005..ac1291aa8 100644 --- a/docs/plugins/bucket.rst +++ b/docs/plugins/bucket.rst @@ -5,8 +5,8 @@ The ``bucket`` plugin helps you keep a balanced files tree for your library by grouping your files into buckets folders representing ranges. This kind of files organization is usually used to classify your music by periods (eg *1960s*, *1970s* etc), or to divide bloated folders into smaller -subfolders by grouping albums/artist alphabetically (eg *A-F*, *G-M*, *N-Z*). -To use plugin, enable it by including ``bucket`` into ``plugins`` line of your +subfolders by grouping albums/artists alphabetically (eg *A-F*, *G-M*, *N-Z*). +To use this plugin, enable it by including ``bucket`` into ``plugins`` line of your beets config. The plugin provides a template function called ``%bucket`` for use in path format expressions:: @@ -26,5 +26,17 @@ The ``bucket_year`` parameter is used for all substitutions occuring on the The definition of a range is somewhat loose, and multiple formats are allowed : - for alpha ranges: the range is defined by the lowest and highest (ascii-wise) alphanumeric characters. eg *'ABCD'*, *'A-D'*, *'A->D'*, *[AD]* are equivalent. -- for year ranges: digits characters are extracted, and in case of doubt XXth century is assumed. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are equivalent. If no upper bound is given, the range is extended to current year (unless a later range is defined). eg *'1975'* encompasses all years from 1975 until now. +- for year ranges: digits characters are extracted and the two extremes years define the range. eg *'1975-77'*, *'1975,76,77'* and *'1975-1977'* are equivalent. If no upper bound is given, the range is extended to current year (unless a later range is defined). eg *'1975'* encompasses all years from 1975 until now. + +If you want to group your files into many small year ranges, you don't have to +enumerate them all in `bucket_year` parameter but can activate the ``extrapolate`` +option instead. This option will generate year bucket names by reproducing characteristics +of declared buckets. + + bucket: + bucket_year: ['2000-05'] + extrapolate: true + +is enough to make the plugin return a five years range for any input year. + diff --git a/test/test_bucket.py b/test/test_bucket.py index b76bd973d..2b4d546c8 100644 --- a/test/test_bucket.py +++ b/test/test_bucket.py @@ -29,46 +29,72 @@ class BucketPluginTest(unittest.TestCase, TestHelper): def tearDown(self): self.teardown_beets() - def _setup_config(self, bucket_year=[], bucket_alpha=[]): + def _setup_config(self, bucket_year=[], bucket_alpha=[], + extrapolate=False): config['bucket']['bucket_year'] = bucket_year config['bucket']['bucket_alpha'] = bucket_alpha + config['bucket']['extrapolate'] = extrapolate self.plugin.setup() def test_year_single_year(self): - """If a single year is given, folder represents a range from this year - to the next 'from year' of next folder.""" - self._setup_config(bucket_year=['50', '70']) - - self.assertEqual(self.plugin._tmpl_bucket('1959'), '50') - self.assertEqual(self.plugin._tmpl_bucket('1969'), '50') + """If a single year is given, range starts from this year and stops at + the year preceding the one of next bucket.""" + self._setup_config(bucket_year=['1950s', '1970s']) + self.assertEqual(self.plugin._tmpl_bucket('1959'), '1950s') + self.assertEqual(self.plugin._tmpl_bucket('1969'), '1950s') def test_year_single_year_last_folder(self): - """Last folder of a range extends from its year to current year.""" - self._setup_config(bucket_year=['50', '70']) - self.assertEqual(self.plugin._tmpl_bucket('2014'), '70') + """If a single year is given for the last bucket, extend it to current + year.""" + self._setup_config(bucket_year=['1950', '1970']) + self.assertEqual(self.plugin._tmpl_bucket('2014'), '1970') self.assertEqual(self.plugin._tmpl_bucket('2015'), '2015') def test_year_two_years(self): - self._setup_config(bucket_year=['50-59', '1960-69']) - self.assertEqual(self.plugin._tmpl_bucket('1959'), '50-59') + """Buckets can be named with the 'from-to' syntax.""" + self._setup_config(bucket_year=['1950-59', '1960-1969']) + self.assertEqual(self.plugin._tmpl_bucket('1959'), '1950-59') + self.assertEqual(self.plugin._tmpl_bucket('1969'), '1960-1969') def test_year_multiple_years(self): + """Buckets can be named by listing all the years""" self._setup_config(bucket_year=['1950,51,52,53']) self.assertEqual(self.plugin._tmpl_bucket('1953'), '1950,51,52,53') self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') def test_year_out_of_range(self): """If no range match, return the year""" - self._setup_config(bucket_year=['50-59', '1960-69']) + self._setup_config(bucket_year=['1950-59', '1960-69']) self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') self._setup_config(bucket_year=[]) self.assertEqual(self.plugin._tmpl_bucket('1974'), '1974') + def test_year_out_of_range_extrapolate(self): + """If no defined range match, extrapolate all ranges using the most + common syntax amongst existing buckets and return the matching one.""" + self._setup_config(bucket_year=['1950-59', '1960-69'], + extrapolate=True) + self.assertEqual(self.plugin._tmpl_bucket('1914'), '1910-19') + # pick single year format + self._setup_config(bucket_year=['1962-81', '2002', '2012'], + extrapolate=True) + self.assertEqual(self.plugin._tmpl_bucket('1983'), '1982') + # pick from-end format + self._setup_config(bucket_year=['1962-81', '2002', '2012-14'], + extrapolate=True) + self.assertEqual(self.plugin._tmpl_bucket('1983'), '1982-01') + # extrapolate add ranges, but never modifies existing ones + self._setup_config(bucket_year=['1932', '1942', '1952', '1962-81', + '2002'], extrapolate=True) + self.assertEqual(self.plugin._tmpl_bucket('1975'), '1962-81') + def test_alpha_all_chars(self): + """Alphabet buckets can be named by listing all their chars""" self._setup_config(bucket_alpha=['ABCD', 'FGH', 'IJKL']) self.assertEqual(self.plugin._tmpl_bucket('garry'), 'FGH') def test_alpha_first_last_chars(self): + """Alphabet buckets can be named by listing the 'from-to' syntax""" self._setup_config(bucket_alpha=['A-D', 'F-H', 'I-Z']) self.assertEqual(self.plugin._tmpl_bucket('garry'), 'F-H') From 0d5050bdcecf2257a746fff62bca8cfd794c062f Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Thu, 8 May 2014 16:13:17 +0200 Subject: [PATCH 10/12] fix handling of digits chars in alpha buckets Characters were search in string.ascii_lowercase that does not contain digits chars. --- beetsplug/bucket.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index 9a6939601..c4b886bc1 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -150,11 +150,12 @@ def build_alpha_spans(alpha_spans_str): [from...to] """ spans = [] + ASCII_DIGITS = string.digits + string.ascii_lowercase for elem in alpha_spans_str: bucket = sorted([x for x in elem.lower() if x.isalnum()]) - beginIdx = string.ascii_lowercase.index(bucket[0]) - endIdx = string.ascii_lowercase.index(bucket[-1]) - spans.append(string.ascii_lowercase[beginIdx:endIdx + 1]) + beginIdx = ASCII_DIGITS.index(bucket[0]) + endIdx = ASCII_DIGITS.index(bucket[-1]) + spans.append(ASCII_DIGITS[beginIdx:endIdx + 1]) return spans From 21feab7ab1f2a8f9617c9f2fe0a66951383cd7ce Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 10 May 2014 10:55:38 +0200 Subject: [PATCH 11/12] add tests to check bad buckets definitions message is now printed to the user when buckets declared in the config file cannot be parsed --- beetsplug/bucket.py | 24 ++++++++++++++++++------ docs/plugins/bucket.rst | 2 +- test/test_bucket.py | 27 +++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index c4b886bc1..230cf7fb1 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -20,7 +20,7 @@ import logging import re import string from itertools import tee, izip -from beets import plugins +from beets import plugins, ui log = logging.getLogger('beets') @@ -44,8 +44,8 @@ def span_from_str(span_str): """Convert string to a 4 digits year """ if yearfrom < 100: - raise BucketError("Bucket 'from' year %d must be expressed on 4 " - "digits" % yearfrom) + raise BucketError("%d must be expressed on 4 digits" % yearfrom) + # if two digits only, pick closest year that ends by these two # digits starting from yearfrom if d < 100: @@ -56,7 +56,14 @@ def span_from_str(span_str): return d years = [int(x) for x in re.findall('\d+', span_str)] - years = [normalize_year(x, years[0]) for x in years] + if not years: + raise ui.UserError("invalid range defined for year bucket '%s': no " + "year found" % span_str) + try: + years = [normalize_year(x, years[0]) for x in years] + except BucketError as exc: + raise ui.UserError("invalid range defined for year bucket '%s': %s" % + (span_str, exc)) res = {'from': years[0], 'str': span_str} if len(years) > 1: @@ -153,8 +160,13 @@ def build_alpha_spans(alpha_spans_str): ASCII_DIGITS = string.digits + string.ascii_lowercase for elem in alpha_spans_str: bucket = sorted([x for x in elem.lower() if x.isalnum()]) - beginIdx = ASCII_DIGITS.index(bucket[0]) - endIdx = ASCII_DIGITS.index(bucket[-1]) + if bucket: + beginIdx = ASCII_DIGITS.index(bucket[0]) + endIdx = ASCII_DIGITS.index(bucket[-1]) + else: + raise ui.UserError("invalid range defined for alpha bucket '%s'" + " : no alphanumeric character found" % + elem) spans.append(ASCII_DIGITS[beginIdx:endIdx + 1]) return spans diff --git a/docs/plugins/bucket.rst b/docs/plugins/bucket.rst index ac1291aa8..173d3d147 100644 --- a/docs/plugins/bucket.rst +++ b/docs/plugins/bucket.rst @@ -37,6 +37,6 @@ of declared buckets. bucket_year: ['2000-05'] extrapolate: true -is enough to make the plugin return a five years range for any input year. +is enough to make the plugin return an enclosing five years range for any input year. diff --git a/test/test_bucket.py b/test/test_bucket.py index 2b4d546c8..e6679f3c1 100644 --- a/test/test_bucket.py +++ b/test/test_bucket.py @@ -14,9 +14,10 @@ """Tests for the 'bucket' plugin.""" +from nose.tools import raises from _common import unittest from beetsplug import bucket -from beets import config +from beets import config, ui from helper import TestHelper @@ -95,8 +96,9 @@ class BucketPluginTest(unittest.TestCase, TestHelper): def test_alpha_first_last_chars(self): """Alphabet buckets can be named by listing the 'from-to' syntax""" - self._setup_config(bucket_alpha=['A-D', 'F-H', 'I-Z']) + self._setup_config(bucket_alpha=['0->9','A->D', 'F-H', 'I->Z']) self.assertEqual(self.plugin._tmpl_bucket('garry'), 'F-H') + self.assertEqual(self.plugin._tmpl_bucket('2pac'), '0->9') def test_alpha_out_of_range(self): """If no range match, return the initial""" @@ -105,6 +107,27 @@ class BucketPluginTest(unittest.TestCase, TestHelper): self._setup_config(bucket_alpha=[]) self.assertEqual(self.plugin._tmpl_bucket('errol'), 'E') + @raises(ui.UserError) + def test_bad_alpha_range_def(self): + """If bad alpha range definition, a UserError is raised""" + self._setup_config(bucket_alpha=['$%']) + self.assertEqual(self.plugin._tmpl_bucket('errol'), 'E') + + @raises(ui.UserError) + def test_bad_year_range_def_no4digits(self): + """If bad year range definition, a UserError is raised. + Range origin must be expressed on 4 digits.""" + self._setup_config(bucket_year=['62-64']) + # from year must be expressed on 4 digits + self.assertEqual(self.plugin._tmpl_bucket('1963'), '62-64') + + @raises(ui.UserError) + def test_bad_year_range_def_nodigits(self): + """If bad year range definition, a UserError is raised. + At least the range origin must be declared.""" + self._setup_config(bucket_year=['nodigits']) + self.assertEqual(self.plugin._tmpl_bucket('1963'), '62-64') + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 02f7e78fd36fda65d2169f1a52995f1a7e2af6af Mon Sep 17 00:00:00 2001 From: Fabrice Laporte Date: Sat, 10 May 2014 10:57:44 +0200 Subject: [PATCH 12/12] fix flake8 warnings --- beetsplug/bucket.py | 2 +- test/test_bucket.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beetsplug/bucket.py b/beetsplug/bucket.py index 230cf7fb1..f31339aa1 100644 --- a/beetsplug/bucket.py +++ b/beetsplug/bucket.py @@ -63,7 +63,7 @@ def span_from_str(span_str): years = [normalize_year(x, years[0]) for x in years] except BucketError as exc: raise ui.UserError("invalid range defined for year bucket '%s': %s" % - (span_str, exc)) + (span_str, exc)) res = {'from': years[0], 'str': span_str} if len(years) > 1: diff --git a/test/test_bucket.py b/test/test_bucket.py index e6679f3c1..f8b86bf6d 100644 --- a/test/test_bucket.py +++ b/test/test_bucket.py @@ -96,7 +96,7 @@ class BucketPluginTest(unittest.TestCase, TestHelper): def test_alpha_first_last_chars(self): """Alphabet buckets can be named by listing the 'from-to' syntax""" - self._setup_config(bucket_alpha=['0->9','A->D', 'F-H', 'I->Z']) + self._setup_config(bucket_alpha=['0->9', 'A->D', 'F-H', 'I->Z']) self.assertEqual(self.plugin._tmpl_bucket('garry'), 'F-H') self.assertEqual(self.plugin._tmpl_bucket('2pac'), '0->9')