From f2c8e9ff07846e4578bf9d9f185b136334843645 Mon Sep 17 00:00:00 2001 From: Diego Moreda Date: Mon, 16 Nov 2015 21:15:48 +0100 Subject: [PATCH 1/7] Add "not" query operator, initial draft * Add support for user friendly "not" operator in queries without requiring to use regular expressions, via adding NotQuery to beets.dbcore.query. * Update the prefix list at library.parse_query_parts() and the query parsing mechanisms in order to create a NotQuery when the prefix is found. * Add two TestCases, NotQueryMatchTest as the negated version of MatchTest, and the more generic NotQueryTest for testing the integration of the NotQuery with the rest of the existing Query subclasses. --- beets/dbcore/query.py | 18 ++++ beets/dbcore/queryparse.py | 22 +++- beets/library.py | 4 +- test/test_query.py | 200 +++++++++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index c04e734b8..4681f2044 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -447,6 +447,24 @@ class OrQuery(MutableCollectionQuery): return any([q.match(item) for q in self.subqueries]) +class NotQuery(MutableCollectionQuery): + """A query that matches the negation of its `subquery`, as a shorcut for + performing `not(subquery)` without using regular expressions. + + TODO: revise class hierarchy, probably limiting to one subquery + """ + def clause(self): + clause, subvals = self.clause_with_joiner('TODO') + if clause: + return 'not ({0})'.format(clause), subvals + else: + # special case for RegexpQuery, (None, ()) + return clause, subvals + + def match(self, item): + return not all([q.match(item) for q in self.subqueries]) + + class TrueQuery(Query): """A query that always matches.""" def clause(self): diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index 89b7bd64a..88bcbd601 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -96,17 +96,33 @@ def construct_query_part(model_cls, prefixes, query_part): key, pattern, query_class = \ parse_query_part(query_part, query_classes, prefixes) + # Handle negation: set negation flag, and recover the original query_class. + negate = query_class is query.NotQuery + if negate: + query_class = query_classes.get(key, query.SubstringQuery) + # No key specified. if key is None: if issubclass(query_class, query.FieldQuery): # The query type matches a specific field, but none was # specified. So we use a version of the query that matches # any field. - return query.AnyFieldQuery(pattern, model_cls._search_fields, - query_class) + q = query.AnyFieldQuery(pattern, model_cls._search_fields, + query_class) + if negate: + return query.NotQuery([q]) + else: + return q else: # Other query type. - return query_class(pattern) + if negate: + return query.NotQuery([query_class(pattern)]) + else: + return query_class(pattern) + else: + if negate: + return query.NotQuery([query_class(key.lower(), pattern, + key in model_cls._fields)]) key = key.lower() return query_class(key.lower(), pattern, key in model_cls._fields) diff --git a/beets/library.py b/beets/library.py index cad39c232..37ab005b8 100644 --- a/beets/library.py +++ b/beets/library.py @@ -1105,7 +1105,9 @@ def parse_query_parts(parts, model_cls): special path query detection. """ # Get query types and their prefix characters. - prefixes = {':': dbcore.query.RegexpQuery} + prefixes = {':': dbcore.query.RegexpQuery, + # TODO: decide on a better symbol, or other syntax + u'\u00ac': dbcore.query.NotQuery} prefixes.update(plugins.queries()) # Special-case path-like queries, which are non-field queries diff --git a/test/test_query.py b/test/test_query.py index 0515c2910..68558c054 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -714,6 +714,206 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) +class NotQueryMatchTest(_common.TestCase): + """Test `query.NotQuery` matching against a single item, using the same + cases and assertions as on `MatchTest`, plus assertion on the negated + queries (ie. assertTrue(q) -> assertFalse(NotQuery(q))). + """ + def setUp(self): + super(NotQueryMatchTest, self).setUp() + self.item = _common.item() + + def test_regex_match_positive(self): + q = dbcore.query.RegexpQuery('album', '^the album$') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_regex_match_negative(self): + q = dbcore.query.RegexpQuery('album', '^album$') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_regex_match_non_string_value(self): + q = dbcore.query.RegexpQuery('disc', '^6$') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_substring_match_positive(self): + q = dbcore.query.SubstringQuery('album', 'album') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_substring_match_negative(self): + q = dbcore.query.SubstringQuery('album', 'ablum') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_substring_match_non_string_value(self): + q = dbcore.query.SubstringQuery('disc', '6') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_year_match_positive(self): + q = dbcore.query.NumericQuery('year', '1') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_year_match_negative(self): + q = dbcore.query.NumericQuery('year', '10') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_bitrate_range_positive(self): + q = dbcore.query.NumericQuery('bitrate', '100000..200000') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_bitrate_range_negative(self): + q = dbcore.query.NumericQuery('bitrate', '200000..300000') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + + def test_open_range(self): + q = dbcore.query.NumericQuery('bitrate', '100000..') + dbcore.query.NotQuery((q,)) + + +class NotQueryTest(DummyDataTestCase): + """Test `query.NotQuery` against the dummy data: + - `test_type_xxx`: tests for the negation of a particular XxxQuery class. + + TODO: add test_type_bytes, for ByteQuery? + """ + def assertNegationProperties(self, q): + """Given a Query `q`, assert that: + - q OR not(q) == all items + - q AND not(q) == 0 + - not(not(q)) == q + """ + not_q = dbcore.query.NotQuery([q]) + # assert using OrQuery, AndQuery + q_or = dbcore.query.OrQuery([q, not_q]) + q_and = dbcore.query.AndQuery([q, not_q]) + self.assert_items_matched_all(self.lib.items(q_or)) + self.assert_items_matched(self.lib.items(q_and), []) + + # assert manually checking the item titles + all_titles = set([i.title for i in self.lib.items()]) + q_results = set([i.title for i in self.lib.items(q)]) + not_q_results = set([i.title for i in self.lib.items(not_q)]) + self.assertEqual(q_results.union(not_q_results), all_titles) + self.assertEqual(q_results.intersection(not_q_results), set()) + + # round trip + not_not_q = dbcore.query.NotQuery([not_q]) + self.assertEqual([i.title for i in self.lib.items(q)], + [i.title for i in self.lib.items(not_not_q)]) + + def test_type_and(self): + # not(a and b) <-> not(a) or not(b) + q = dbcore.query.AndQuery([dbcore.query.BooleanQuery('comp', True), + dbcore.query.NumericQuery('year', '2002')]) + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['foo bar', 'beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_anyfield(self): + q = dbcore.query.AnyFieldQuery('foo', ['title', 'artist', 'album'], + dbcore.query.SubstringQuery) + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['baz qux']) + self.assertNegationProperties(q) + + def test_type_boolean(self): + q = dbcore.query.BooleanQuery('comp', True) + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_date(self): + q = dbcore.query.DateQuery('mtime', '0.0') + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, []) + self.assertNegationProperties(q) + + def test_type_false(self): + q = dbcore.query.FalseQuery() + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched_all(not_results) + self.assertNegationProperties(q) + + def test_type_match(self): + q = dbcore.query.MatchQuery('year', '2003') + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['foo bar', 'baz qux']) + self.assertNegationProperties(q) + + def test_type_none(self): + q = dbcore.query.NoneQuery('rg_track_gain') + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, []) + self.assertNegationProperties(q) + + def test_type_numeric(self): + q = dbcore.query.NumericQuery('year', '2001..2002') + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_or(self): + # not(a or b) <-> not(a) and not(b) + q = dbcore.query.OrQuery([dbcore.query.BooleanQuery('comp', True), + dbcore.query.NumericQuery('year', '2002')]) + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_regexp(self): + q = dbcore.query.RegexpQuery('artist', '^t') + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['foo bar']) + self.assertNegationProperties(q) + + def test_type_substring(self): + q = dbcore.query.SubstringQuery('album', 'ba') + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_true(self): + q = dbcore.query.TrueQuery() + not_results = self.lib.items(dbcore.query.NotQuery([q])) + self.assert_items_matched(not_results, []) + self.assertNegationProperties(q) + + def test_fast_vs_slow(self): + """Test that the results are the same regardless of the `fast` flag + for negated `FieldQuery`s. + + TODO: investigate NoneQuery(fast=False), as it is raising + AttributeError: type object 'NoneQuery' has no attribute 'field' + at NoneQuery.match() (due to being @classmethod, and no self?) + """ + classes = [(dbcore.query.DateQuery, ['mtime', '0.0']), + (dbcore.query.MatchQuery, ['artist', 'one']), + # (dbcore.query.NoneQuery, ['rg_track_gain']), + (dbcore.query.NumericQuery, ['year', '2002']), + (dbcore.query.StringFieldQuery, ['year', '2001']), + (dbcore.query.RegexpQuery, ['album', '^.a']), + (dbcore.query.SubstringQuery, ['title', 'x'])] + + for klass, args in classes: + q_fast = dbcore.query.NotQuery([klass(*(args + [True]))]) + q_slow = dbcore.query.NotQuery([klass(*(args + [False]))]) + + try: + self.assertEqual([i.title for i in self.lib.items(q_fast)], + [i.title for i in self.lib.items(q_slow)]) + except NotImplementedError: + # ignore classes that do not provide `fast` implementation + pass + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__) From dd8b80e32004129c587134ee51a916155cdbf811 Mon Sep 17 00:00:00 2001 From: Diego Moreda Date: Wed, 18 Nov 2015 14:54:20 +0100 Subject: [PATCH 2/7] Make NotQuery subclass Query, update tests * Modify NotQuery so it subclasses Query instead of MutableCollectionQuery. * Update instances where NotQuery objects are created on tests and queryparse, as NotQuery expects a single Query as a parameter to the constructor instead of a list of Queries. --- beets/dbcore/query.py | 22 +++++++++++----- beets/dbcore/queryparse.py | 13 +++++---- test/test_query.py | 54 +++++++++++++++++++------------------- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 4681f2044..b4c1d1445 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -447,14 +447,14 @@ class OrQuery(MutableCollectionQuery): return any([q.match(item) for q in self.subqueries]) -class NotQuery(MutableCollectionQuery): +class NotQuery(Query): """A query that matches the negation of its `subquery`, as a shorcut for - performing `not(subquery)` without using regular expressions. + performing `not(subquery)` without using regular expressions.""" + def __init__(self, subquery): + self.subquery = subquery - TODO: revise class hierarchy, probably limiting to one subquery - """ def clause(self): - clause, subvals = self.clause_with_joiner('TODO') + clause, subvals = self.subquery.clause() if clause: return 'not ({0})'.format(clause), subvals else: @@ -462,7 +462,17 @@ class NotQuery(MutableCollectionQuery): return clause, subvals def match(self, item): - return not all([q.match(item) for q in self.subqueries]) + return not self.subquery.match(item) + + def __repr__(self): + return "{0.__class__.__name__}({0.subquery})".format(self) + + def __eq__(self, other): + return super(NotQuery, self).__eq__(other) and \ + self.subquery == other.subquery + + def __hash__(self): + return hash(('not', hash(self.subquery))) class TrueQuery(Query): diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index 88bcbd601..565fe26f1 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -110,22 +110,21 @@ def construct_query_part(model_cls, prefixes, query_part): q = query.AnyFieldQuery(pattern, model_cls._search_fields, query_class) if negate: - return query.NotQuery([q]) + return query.NotQuery(q) else: return q else: # Other query type. if negate: - return query.NotQuery([query_class(pattern)]) + return query.NotQuery(query_class(pattern)) else: return query_class(pattern) - else: - if negate: - return query.NotQuery([query_class(key.lower(), pattern, - key in model_cls._fields)]) key = key.lower() - return query_class(key.lower(), pattern, key in model_cls._fields) + q = query_class(key.lower(), pattern, key in model_cls._fields) + if negate: + return query.NotQuery(q) + return q def query_from_strings(query_cls, model_cls, prefixes, query_parts): diff --git a/test/test_query.py b/test/test_query.py index 68558c054..ae5a84b88 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -726,56 +726,56 @@ class NotQueryMatchTest(_common.TestCase): def test_regex_match_positive(self): q = dbcore.query.RegexpQuery('album', '^the album$') self.assertTrue(q.match(self.item)) - self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) def test_regex_match_negative(self): q = dbcore.query.RegexpQuery('album', '^album$') self.assertFalse(q.match(self.item)) - self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) def test_regex_match_non_string_value(self): q = dbcore.query.RegexpQuery('disc', '^6$') self.assertTrue(q.match(self.item)) - self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) def test_substring_match_positive(self): q = dbcore.query.SubstringQuery('album', 'album') self.assertTrue(q.match(self.item)) - self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) def test_substring_match_negative(self): q = dbcore.query.SubstringQuery('album', 'ablum') self.assertFalse(q.match(self.item)) - self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) def test_substring_match_non_string_value(self): q = dbcore.query.SubstringQuery('disc', '6') self.assertTrue(q.match(self.item)) - self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) def test_year_match_positive(self): q = dbcore.query.NumericQuery('year', '1') self.assertTrue(q.match(self.item)) - self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) def test_year_match_negative(self): q = dbcore.query.NumericQuery('year', '10') self.assertFalse(q.match(self.item)) - self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) def test_bitrate_range_positive(self): q = dbcore.query.NumericQuery('bitrate', '100000..200000') self.assertTrue(q.match(self.item)) - self.assertFalse(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) def test_bitrate_range_negative(self): q = dbcore.query.NumericQuery('bitrate', '200000..300000') self.assertFalse(q.match(self.item)) - self.assertTrue(dbcore.query.NotQuery((q,)).match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) def test_open_range(self): q = dbcore.query.NumericQuery('bitrate', '100000..') - dbcore.query.NotQuery((q,)) + dbcore.query.NotQuery(q) class NotQueryTest(DummyDataTestCase): @@ -790,7 +790,7 @@ class NotQueryTest(DummyDataTestCase): - q AND not(q) == 0 - not(not(q)) == q """ - not_q = dbcore.query.NotQuery([q]) + not_q = dbcore.query.NotQuery(q) # assert using OrQuery, AndQuery q_or = dbcore.query.OrQuery([q, not_q]) q_and = dbcore.query.AndQuery([q, not_q]) @@ -805,7 +805,7 @@ class NotQueryTest(DummyDataTestCase): self.assertEqual(q_results.intersection(not_q_results), set()) # round trip - not_not_q = dbcore.query.NotQuery([not_q]) + not_not_q = dbcore.query.NotQuery(not_q) self.assertEqual([i.title for i in self.lib.items(q)], [i.title for i in self.lib.items(not_not_q)]) @@ -813,50 +813,50 @@ class NotQueryTest(DummyDataTestCase): # not(a and b) <-> not(a) or not(b) q = dbcore.query.AndQuery([dbcore.query.BooleanQuery('comp', True), dbcore.query.NumericQuery('year', '2002')]) - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['foo bar', 'beets 4 eva']) self.assertNegationProperties(q) def test_type_anyfield(self): q = dbcore.query.AnyFieldQuery('foo', ['title', 'artist', 'album'], dbcore.query.SubstringQuery) - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['baz qux']) self.assertNegationProperties(q) def test_type_boolean(self): q = dbcore.query.BooleanQuery('comp', True) - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['beets 4 eva']) self.assertNegationProperties(q) def test_type_date(self): q = dbcore.query.DateQuery('mtime', '0.0') - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, []) self.assertNegationProperties(q) def test_type_false(self): q = dbcore.query.FalseQuery() - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched_all(not_results) self.assertNegationProperties(q) def test_type_match(self): q = dbcore.query.MatchQuery('year', '2003') - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['foo bar', 'baz qux']) self.assertNegationProperties(q) def test_type_none(self): q = dbcore.query.NoneQuery('rg_track_gain') - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, []) self.assertNegationProperties(q) def test_type_numeric(self): q = dbcore.query.NumericQuery('year', '2001..2002') - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['beets 4 eva']) self.assertNegationProperties(q) @@ -864,25 +864,25 @@ class NotQueryTest(DummyDataTestCase): # not(a or b) <-> not(a) and not(b) q = dbcore.query.OrQuery([dbcore.query.BooleanQuery('comp', True), dbcore.query.NumericQuery('year', '2002')]) - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['beets 4 eva']) self.assertNegationProperties(q) def test_type_regexp(self): q = dbcore.query.RegexpQuery('artist', '^t') - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['foo bar']) self.assertNegationProperties(q) def test_type_substring(self): q = dbcore.query.SubstringQuery('album', 'ba') - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, ['beets 4 eva']) self.assertNegationProperties(q) def test_type_true(self): q = dbcore.query.TrueQuery() - not_results = self.lib.items(dbcore.query.NotQuery([q])) + not_results = self.lib.items(dbcore.query.NotQuery(q)) self.assert_items_matched(not_results, []) self.assertNegationProperties(q) @@ -903,8 +903,8 @@ class NotQueryTest(DummyDataTestCase): (dbcore.query.SubstringQuery, ['title', 'x'])] for klass, args in classes: - q_fast = dbcore.query.NotQuery([klass(*(args + [True]))]) - q_slow = dbcore.query.NotQuery([klass(*(args + [False]))]) + q_fast = dbcore.query.NotQuery(klass(*(args + [True]))) + q_slow = dbcore.query.NotQuery(klass(*(args + [False]))) try: self.assertEqual([i.title for i in self.lib.items(q_fast)], From 0293504a24c2265c41e3ef235bf23b8b314b2c89 Mon Sep 17 00:00:00 2001 From: Diego Moreda Date: Wed, 18 Nov 2015 16:59:50 +0100 Subject: [PATCH 3/7] Update negation query syntax, prefix always first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update the negation query sintax so that the negation prefix ("-" or "¬") is always the first part of a query, instead of after the ":" separator. * Modify queryparse, so that the detection of the negation is done inside parse_query_part() (using the modified PARSE_QUERY_PART_REGEX, and returning the negation flag) instead of construct_query_part. * Revert the prefixes dict on beets.library to the original dict (only one item, with the RegexpQuery prefix). --- beets/dbcore/queryparse.py | 22 +++++++++++----------- beets/library.py | 4 +--- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index 565fe26f1..ca01f0433 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -23,6 +23,10 @@ import beets PARSE_QUERY_PART_REGEX = re.compile( # Non-capturing optional segment for the keyword. + r'(?:' + ur'(-|\u00ac)' # Negation prefixes + r')?' + r'(?:' r'(\S+?)' # The field key. r'(? Date: Wed, 18 Nov 2015 17:27:22 +0100 Subject: [PATCH 4/7] Add NotQuery syntax tests * Add tests to NotQueryTest for testing the results of using queries with negation. * Fix issue on test_dbcore due to the modifications on the tuple returned by parse_query_part (the number of elements was changed from 3 to 4). --- test/test_dbcore.py | 2 +- test/test_query.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 6fc07d0f6..04fe60d46 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -351,7 +351,7 @@ class QueryParseTest(unittest.TestCase): part, {'year': dbcore.query.NumericQuery}, {':': dbcore.query.RegexpQuery}, - ) + )[:-1] # remove the negate flag def test_one_basic_term(self): q = 'test' diff --git a/test/test_query.py b/test/test_query.py index ae5a84b88..64328233f 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -781,6 +781,7 @@ class NotQueryMatchTest(_common.TestCase): class NotQueryTest(DummyDataTestCase): """Test `query.NotQuery` against the dummy data: - `test_type_xxx`: tests for the negation of a particular XxxQuery class. + - `test_get_yyy`: tests on query strings (similar to `GetTest`) TODO: add test_type_bytes, for ByteQuery? """ @@ -886,6 +887,44 @@ class NotQueryTest(DummyDataTestCase): self.assert_items_matched(not_results, []) self.assertNegationProperties(q) + def test_get_prefixes_keyed(self): + """Test both negation prefixes on a keyed query.""" + q0 = '-title:qux' + q1 = '\u00actitle:qux' + results0 = self.lib.items(q0) + results1 = self.lib.items(q1) + self.assert_items_matched(results0, ['foo bar', 'beets 4 eva']) + self.assert_items_matched(results1, ['foo bar', 'beets 4 eva']) + + def test_get_prefixes_unkeyed(self): + """Test both negation prefixes on an unkeyed query.""" + q0 = '-qux' + q1 = '\u00acqux' + results0 = self.lib.items(q0) + results1 = self.lib.items(q1) + self.assert_items_matched(results0, ['foo bar', 'beets 4 eva']) + self.assert_items_matched(results1, ['foo bar', 'beets 4 eva']) + + def test_get_keyed_regexp(self): + q = r'-artist::t.+r' + results = self.lib.items(q) + self.assert_items_matched(results, ['foo bar', 'baz qux']) + + def test_get_one_unkeyed_regexp(self): + q = r'-:x$' + results = self.lib.items(q) + self.assert_items_matched(results, ['foo bar', 'beets 4 eva']) + + def test_get_multiple_terms(self): + q = 'baz -bar' + results = self.lib.items(q) + self.assert_items_matched(results, ['baz qux']) + + def test_get_mixed_terms(self): + q = 'baz -title:bar' + results = self.lib.items(q) + self.assert_items_matched(results, ['baz qux']) + def test_fast_vs_slow(self): """Test that the results are the same regardless of the `fast` flag for negated `FieldQuery`s. From e69b3b3c5d38a36868fdafaa2f540f65e1227c71 Mon Sep 17 00:00:00 2001 From: Diego Moreda Date: Wed, 18 Nov 2015 17:40:47 +0100 Subject: [PATCH 5/7] Fix NotQuery assertion by using sets, not lists * Fix issue with an assertion that was order-sensitive and caused a problem on some tox runs. --- test/test_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_query.py b/test/test_query.py index 64328233f..5720c65cf 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -807,8 +807,8 @@ class NotQueryTest(DummyDataTestCase): # round trip not_not_q = dbcore.query.NotQuery(not_q) - self.assertEqual([i.title for i in self.lib.items(q)], - [i.title for i in self.lib.items(not_not_q)]) + self.assertEqual(set([i.title for i in self.lib.items(q)]), + set([i.title for i in self.lib.items(not_not_q)])) def test_type_and(self): # not(a and b) <-> not(a) or not(b) From 40bfed756b8cef2ce52b9449602511f78cab8e11 Mon Sep 17 00:00:00 2001 From: Diego Moreda Date: Thu, 19 Nov 2015 18:04:47 +0100 Subject: [PATCH 6/7] Revise not query syntax, cleanup, modify docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revise the NotQuery syntax, replacing the '¬' character with '^'. Fix tests to conform to this change, and cleanup the PARSE_QUERY_PART_REGEX. * Modify parse_query_part() docstring to mention the negate parameter on the returned tuple, and added an example. --- beets/dbcore/queryparse.py | 17 ++++++++--------- test/test_query.py | 6 +++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index ca01f0433..c7b75a90c 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -23,9 +23,7 @@ import beets PARSE_QUERY_PART_REGEX = re.compile( # Non-capturing optional segment for the keyword. - r'(?:' - ur'(-|\u00ac)' # Negation prefixes - r')?' + r'(-|\^)?' # Negation prefixes. r'(?:' r'(\S+?)' # The field key. @@ -41,9 +39,9 @@ PARSE_QUERY_PART_REGEX = re.compile( def parse_query_part(part, query_classes={}, prefixes={}, default_class=query.SubstringQuery): """Take a query in the form of a key/value pair separated by a - colon and return a tuple of `(key, value, cls)`. `key` may be None, + colon and return a tuple of `(key, value, cls, negate)`. `key` may be None, indicating that any field may be matched. `cls` is a subclass of - `FieldQuery`. + `FieldQuery`. `negate` is a boolean indicating if the query is negated. The optional `query_classes` parameter maps field names to default query types; `default_class` is the fallback. `prefixes` is a map @@ -57,10 +55,11 @@ def parse_query_part(part, query_classes={}, prefixes={}, class is available, `default_class` is used. For instance, - 'stapler' -> (None, 'stapler', SubstringQuery) - 'color:red' -> ('color', 'red', SubstringQuery) - ':^Quiet' -> (None, '^Quiet', RegexpQuery) - 'color::b..e' -> ('color', 'b..e', RegexpQuery) + 'stapler' -> (None, 'stapler', SubstringQuery, False) + 'color:red' -> ('color', 'red', SubstringQuery, False) + ':^Quiet' -> (None, '^Quiet', RegexpQuery, False) + 'color::b..e' -> ('color', 'b..e', RegexpQuery, False) + '-color:red' -> ('color', 'red', SubstringQuery, True) Prefixes may be "escaped" with a backslash to disable the keying behavior. diff --git a/test/test_query.py b/test/test_query.py index 5720c65cf..c415b7f1c 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -890,7 +890,7 @@ class NotQueryTest(DummyDataTestCase): def test_get_prefixes_keyed(self): """Test both negation prefixes on a keyed query.""" q0 = '-title:qux' - q1 = '\u00actitle:qux' + q1 = '^title:qux' results0 = self.lib.items(q0) results1 = self.lib.items(q1) self.assert_items_matched(results0, ['foo bar', 'beets 4 eva']) @@ -899,13 +899,13 @@ class NotQueryTest(DummyDataTestCase): def test_get_prefixes_unkeyed(self): """Test both negation prefixes on an unkeyed query.""" q0 = '-qux' - q1 = '\u00acqux' + q1 = '^qux' results0 = self.lib.items(q0) results1 = self.lib.items(q1) self.assert_items_matched(results0, ['foo bar', 'beets 4 eva']) self.assert_items_matched(results1, ['foo bar', 'beets 4 eva']) - def test_get_keyed_regexp(self): + def test_get_one_keyed_regexp(self): q = r'-artist::t.+r' results = self.lib.items(q) self.assert_items_matched(results, ['foo bar', 'baz qux']) From 51bf6a1c9ffd7f8c4e10511eb58a5284248594ec Mon Sep 17 00:00:00 2001 From: Diego Moreda Date: Fri, 20 Nov 2015 18:06:22 +0100 Subject: [PATCH 7/7] Add documentation for NotQuery, cleanup * Add changelog and query.rst documentation entries for the usage of negated queries. * Cleanup NotQuery class as suggested during code review (PEP conforming docstring, clarification on empty clause match behaviour). --- beets/dbcore/query.py | 6 ++++-- docs/changelog.rst | 4 ++++ docs/reference/query.rst | 29 +++++++++++++++++++++++++++++ test/test_query.py | 2 -- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index b4c1d1445..348394f0a 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -449,7 +449,8 @@ class OrQuery(MutableCollectionQuery): class NotQuery(Query): """A query that matches the negation of its `subquery`, as a shorcut for - performing `not(subquery)` without using regular expressions.""" + performing `not(subquery)` without using regular expressions. + """ def __init__(self, subquery): self.subquery = subquery @@ -458,7 +459,8 @@ class NotQuery(Query): if clause: return 'not ({0})'.format(clause), subvals else: - # special case for RegexpQuery, (None, ()) + # If there is no clause, there is nothing to negate. All the logic + # is handled by match() for slow queries. return clause, subvals def match(self, item): diff --git a/docs/changelog.rst b/docs/changelog.rst index 700a16d6d..81dcdd2b7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,10 @@ New: :doc:`/plugins/discogs` also adopts the same setting. * :doc:`/plugins/embyupdate`: A plugin to trigger a library refresh on a `Emby Server`_ if database changed. +* Queries can now use "not" logic: if you prepend a query term with "-" or + "^", items or albums matching that term will be excluded from the results. + For example, ``beet ls foo ^artist:bar`` will get all the items matching + `foo` but whose artist do not match `bar`. See :ref:`not_query`. :bug:`819` For developers: diff --git a/docs/reference/query.rst b/docs/reference/query.rst index 452681c35..6d83537d1 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -181,6 +181,35 @@ Find all items with a file modification time between 2008-12-01 and $ beet ls 'mtime:2008-12-01..2008-12-02' +.. _not_query: + +Query Term Negation +------------------- + +Query terms can also be negated, acting like a Boolean "not", by prepending +them with ``-`` or ``^``. This has the effect of returning all the items that +do **not** match the query term. For example, this command:: + + $ beet list ^love + +matches all the songs in the library that do not have "love" in any of their +fields. + +Negation can be combined with the rest of the query mechanisms, allowing to +negate specific fields, regular expressions, etc. For example, this command:: + + $ beet list -a artist:dylan ^year:1980..1990 "^album::the(y)?" + +matches all the albums with an artist containing "dylan", but excluding those +released on the eighties and those that have "the" or "they" on the title. + +Note that the ``-`` character is treated by most shells as a reserved character +for passing arguments, and as such needs to be escaped if using it for query +negation. In most UNIX derivatives shells, using a double dash ``--`` +(indicating that everything after that point should not be treated as +arguments) before the query terms should prevent conflicts, such as:: + + $ beet list -a -- artist:dylan -year:1980..1990 "-album::the(y)?" .. _pathquery: diff --git a/test/test_query.py b/test/test_query.py index c415b7f1c..e6b2d0499 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -782,8 +782,6 @@ class NotQueryTest(DummyDataTestCase): """Test `query.NotQuery` against the dummy data: - `test_type_xxx`: tests for the negation of a particular XxxQuery class. - `test_get_yyy`: tests on query strings (similar to `GetTest`) - - TODO: add test_type_bytes, for ByteQuery? """ def assertNegationProperties(self, q): """Given a Query `q`, assert that: