From 10a4e14045ce55305f874054650ff406f408de37 Mon Sep 17 00:00:00 2001
From: Matteo Mecucci <djinn.rtl@gmail.com>
Date: Sat, 28 Apr 2012 14:07:20 +0200
Subject: [PATCH 1/3] Added regexps to queries, use with additional column
 before pattern eg 'title::^Quiet' or ':^Quiet'.

---
 beets/library.py | 83 ++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 74 insertions(+), 9 deletions(-)

diff --git a/beets/library.py b/beets/library.py
index 4d0fb0328..810b19464 100644
--- a/beets/library.py
+++ b/beets/library.py
@@ -361,6 +361,21 @@ class SubstringQuery(FieldQuery):
         value = getattr(item, self.field) or ''
         return self.pattern.lower() in value.lower()
 
+class RegexpQuery(FieldQuery):
+    """A query that matches a regular expression in a specific item field."""
+    def __init__(self, field, pattern):
+        super(RegexpQuery, self).__init__(field, pattern)
+        self.regexp = re.compile(pattern)
+
+    def clause(self):
+        clause = self.field + " REGEXP ?"
+        subvals = [self.pattern]
+        return clause, subvals
+
+    def match(self, item):
+        value = getattr(item, self.field) or ''
+        return self.regexp.match(value) is not None
+
 class BooleanQuery(MatchQuery):
     """Matches a boolean field. Pattern should either be a boolean or a
     string reflecting a boolean.
@@ -417,17 +432,23 @@ class CollectionQuery(Query):
                                 r'(\S+?)'   # the keyword
                                 r'(?<!\\):' # unescaped :
                            r')?'
+                           r'((?<!\\):?)'  # unescaped : for regexps
                            r'(.+)',        # the term itself
                            re.I)            # case-insensitive
     @classmethod
     def _parse_query_part(cls, part):
         """Takes a query in the form of a key/value pair separated by a
-        colon. Returns pair (key, term) where key is None if the search
-        term has no key.
+        colon. An additional colon before the value indicates that the
+        value is a regular expression.
+        Returns tuple (key, term, is_regexp) where key is None if
+        the search term has no key and is_regexp indicates whether term
+        is a regular expression or not.
 
         For instance,
-        parse_query('stapler') == (None, 'stapler')
-        parse_query('color:red') == ('color', 'red')
+        parse_query('stapler') == (None, 'stapler', false)
+        parse_query('color:red') == ('color', 'red', false)
+        parse_query(':^Quiet') == (None, '^Quiet', true)
+        parse_query('color::b..e') == ('color', 'b..e', true)
 
         Colons may be 'escaped' with a backslash to disable the keying
         behavior.
@@ -435,7 +456,7 @@ class CollectionQuery(Query):
         part = part.strip()
         match = cls._pq_regex.match(part)
         if match:
-            return match.group(1), match.group(2).replace(r'\:', ':')
+            return match.group(1), match.group(3).replace(r'\:', ':'), match.group(2)==':'
 
     @classmethod
     def from_strings(cls, query_parts, default_fields=None, all_keys=ITEM_KEYS):
@@ -449,21 +470,28 @@ class CollectionQuery(Query):
             res = cls._parse_query_part(part)
             if not res:
                 continue
-            key, pattern = res
+            key, pattern, is_regexp = res
             if key is None: # No key specified.
                 if os.sep in pattern and 'path' in all_keys:
                     # This looks like a path.
                     subqueries.append(PathQuery(pattern))
                 else:
                     # Match any field.
-                    subqueries.append(AnySubstringQuery(pattern,
-                                                        default_fields))
+                    if is_regexp:
+                        subqueries.append(
+                            AnyRegexpQuery(pattern, default_fields))
+                    else:
+                        subqueries.append(
+                            AnySubstringQuery(pattern, default_fields))
             elif key.lower() == 'comp': # a boolean field
                 subqueries.append(BooleanQuery(key.lower(), pattern))
             elif key.lower() == 'path' and 'path' in all_keys:
                 subqueries.append(PathQuery(pattern))
             elif key.lower() in all_keys: # ignore unrecognized keys
-                subqueries.append(SubstringQuery(key.lower(), pattern))
+                if is_regexp:
+                    subqueries.append(RegexpQuery(key.lower(), pattern))
+                else:
+                    subqueries.append(SubstringQuery(key.lower(), pattern))
             elif key.lower() == 'singleton':
                 subqueries.append(SingletonQuery(util.str2bool(pattern)))
         if not subqueries: # no terms in query
@@ -508,6 +536,37 @@ class AnySubstringQuery(CollectionQuery):
                 return True
         return False
 
+class AnyRegexpQuery(CollectionQuery):
+    """A query that matches a regexp in any of a list of metadata
+    fields.
+    """
+    def __init__(self, pattern, fields=None):
+        """Create a query for regexp over the sequence of fields
+        given. If no fields are given, all available fields are
+        used.
+        """
+        self.regexp = re.compile(pattern)
+        self.fields = fields or ITEM_KEYS_WRITABLE
+
+        subqueries = []
+        for field in self.fields:
+            subqueries.append(RegexpQuery(field, pattern))
+        super(AnyRegexpQuery, self).__init__(subqueries)
+
+    def clause(self):
+        return self.clause_with_joiner('or')
+
+    def match(self, item):
+        for fld in self.fields:
+            try:
+                val = getattr(item, fld)
+            except KeyError:
+                continue
+            if isinstance(val, basestring) and \
+               self.regexp.match(val) is not None:
+                return True
+        return False
+
 class MutableCollectionQuery(CollectionQuery):
     """A collection query whose subqueries may be modified after the
     query is initialized.
@@ -782,6 +841,12 @@ class Library(BaseLibrary):
         self.conn = sqlite3.connect(self.path, timeout)
         self.conn.row_factory = sqlite3.Row
             # this way we can access our SELECT results like dictionaries
+
+        def regexp(expr, item):
+            reg = re.compile(expr)
+            return reg.search(item) is not None
+
+        self.conn.create_function("REGEXP", 2, regexp)
         
         self._make_table('items', item_fields)
         self._make_table('albums', album_fields)

From 46ec5db3feb66908d1350d7528e243e27ce94796 Mon Sep 17 00:00:00 2001
From: Matteo Mecucci <djinn.rtl@gmail.com>
Date: Sat, 28 Apr 2012 15:17:43 +0200
Subject: [PATCH 2/3] Added unit tests for regexps in query expressions.

---
 test/test_query.py | 101 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 90 insertions(+), 11 deletions(-)

diff --git a/test/test_query.py b/test/test_query.py
index cad6da336..a572ee3f2 100644
--- a/test/test_query.py
+++ b/test/test_query.py
@@ -27,27 +27,37 @@ some_item = _common.item()
 class QueryParseTest(unittest.TestCase):
     def test_one_basic_term(self):
         q = 'test'
-        r = (None, 'test')
+        r = (None, 'test', False)
         self.assertEqual(pqp(q), r)
-    
+
     def test_one_keyed_term(self):
         q = 'test:val'
-        r = ('test', 'val')
+        r = ('test', 'val', False)
         self.assertEqual(pqp(q), r)
 
     def test_colon_at_end(self):
         q = 'test:'
-        r = (None, 'test:')
+        r = (None, 'test:', False)
         self.assertEqual(pqp(q), r)
-    
-    def test_colon_at_start(self):
-        q = ':test'
-        r = (None, ':test')
+
+    def test_one_basic_regexp(self):
+        q = r':regexp'
+        r = (None, 'regexp', True)
         self.assertEqual(pqp(q), r)
-    
+
+    def test_keyed_regexp(self):
+        q = r'test::regexp'
+        r = ('test', 'regexp', True)
+        self.assertEqual(pqp(q), r)
+
     def test_escaped_colon(self):
         q = r'test\:val'
-        r = (None, 'test:val')
+        r = (None, 'test:val', False)
+        self.assertEqual(pqp(q), r)
+
+    def test_escaped_colon_in_regexp(self):
+        q = r':test\:regexp'
+        r = (None, 'test:regexp', True)
         self.assertEqual(pqp(q), r)
 
 class AnySubstringQueryTest(unittest.TestCase):
@@ -67,6 +77,27 @@ class AnySubstringQueryTest(unittest.TestCase):
         q = beets.library.AnySubstringQuery('title', ['artist'])
         self.assertRaises(StopIteration, self.lib.items(q).next)
 
+class AnyRegexpQueryTest(unittest.TestCase):
+    def setUp(self):
+        self.lib = beets.library.Library(':memory:')
+        self.lib.add(some_item)
+
+    def test_no_restriction(self):
+        q = beets.library.AnyRegexpQuery(r'^the ti')
+        self.assertEqual(self.lib.items(q).next().title, 'the title')
+
+    def test_restriction_completeness(self):
+        q = beets.library.AnyRegexpQuery(r'^the ti', ['title'])
+        self.assertEqual(self.lib.items(q).next().title, 'the title')
+
+    def test_restriction_soundness(self):
+        q = beets.library.AnyRegexpQuery(r'^the ti', ['artist'])
+        self.assertRaises(StopIteration, self.lib.items(q).next)
+
+    def test_restriction_soundness_2(self):
+        q = beets.library.AnyRegexpQuery(r'the ti$', ['title'])
+        self.assertRaises(StopIteration, self.lib.items(q).next)
+
 
 # Convenient asserts for matching items.
 class AssertsMixin(object):
@@ -103,12 +134,24 @@ class GetTest(unittest.TestCase, AssertsMixin):
         self.assert_matched(results, 'Littlest Things')
         self.assert_done(results)
     
+    def test_get_one_keyed_regexp(self):
+        q = r'artist::L.+y'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Littlest Things')
+        self.assert_done(results)
+    
     def test_get_one_unkeyed_term(self):
         q = 'Terry'
         results = self.lib.items(q)
         self.assert_matched(results, 'Boracay')
         self.assert_done(results)
     
+    def test_get_one_unkeyed_regexp(self):
+        q = r':y$'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Boracay')
+        self.assert_done(results)
+    
     def test_get_no_matches(self):
         q = 'popebear'
         results = self.lib.items(q)
@@ -125,6 +168,15 @@ class GetTest(unittest.TestCase, AssertsMixin):
         self.assert_matched(results, 'Lovers Who Uncover')
         self.assert_done(results)
     
+    def test_regexp_case_sensitive(self):
+        q = r':UNCoVER'
+        results = self.lib.items(q)
+        self.assert_done(results)
+        q = r':Uncover'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Lovers Who Uncover')
+        self.assert_done(results)
+    
     def test_term_case_insensitive_with_key(self):
         q = 'album:stiLL'
         results = self.lib.items(q)
@@ -145,6 +197,14 @@ class GetTest(unittest.TestCase, AssertsMixin):
         self.assert_matched(results, 'Boracay')
         self.assert_done(results)
     
+    def test_unkeyed_regexp_matches_multiple_columns(self):
+        q = r':^T'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Take Pills')
+        self.assert_matched(results, 'Lovers Who Uncover')
+        self.assert_matched(results, 'Boracay')
+        self.assert_done(results)
+    
     def test_keyed_term_matches_only_one_column(self):
         q = 'artist:little'
         results = self.lib.items(q)
@@ -152,13 +212,32 @@ class GetTest(unittest.TestCase, AssertsMixin):
         self.assert_matched(results, 'Boracay')
         self.assert_done(results)
     
-    def test_mulitple_terms_narrow_search(self):
+    def test_keyed_regexp_matches_only_one_column(self):
+        q = r'album::\sS'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Littlest Things')
+        self.assert_matched(results, 'Lovers Who Uncover')
+        self.assert_done(results)
+    
+    def test_multiple_terms_narrow_search(self):
         q = 'little ones'
         results = self.lib.items(q)
         self.assert_matched(results, 'Lovers Who Uncover')
         self.assert_matched(results, 'Boracay')
         self.assert_done(results)
 
+    def test_multiple_regexps_narrow_search(self):
+        q = r':\sS :^T'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Lovers Who Uncover')
+        self.assert_done(results)
+
+    def test_mixed_terms_regexps_narrow_search(self):
+        q = r':\sS lily'
+        results = self.lib.items(q)
+        self.assert_matched(results, 'Littlest Things')
+        self.assert_done(results)
+
 class MemoryGetTest(unittest.TestCase, AssertsMixin):
     def setUp(self):
         self.album_item = _common.item()

From f4a0595b329c9e89cf6ca9f624d044797d4520b6 Mon Sep 17 00:00:00 2001
From: Matteo Mecucci <djinn.rtl@gmail.com>
Date: Sat, 28 Apr 2012 15:18:51 +0200
Subject: [PATCH 3/3] Fixed the regexp function used with sqlite3.

---
 beets/library.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/beets/library.py b/beets/library.py
index 810b19464..330af9284 100644
--- a/beets/library.py
+++ b/beets/library.py
@@ -837,14 +837,23 @@ class Library(BaseLibrary):
         self.art_filename = bytestring_path(art_filename)
         self.replacements = replacements
         
+        # uncomment for extra traceback on error
+        #sqlite3.enable_callback_tracebacks(1)
+
         self.timeout = timeout
         self.conn = sqlite3.connect(self.path, timeout)
         self.conn.row_factory = sqlite3.Row
             # this way we can access our SELECT results like dictionaries
 
         def regexp(expr, item):
-            reg = re.compile(expr)
-            return reg.search(item) is not None
+            if item == None:
+                return False
+            try:
+                reg = re.compile(expr)
+                res = reg.search(item)
+                return res is not None
+            except:
+                return False
 
         self.conn.create_function("REGEXP", 2, regexp)