diff --git a/beets/dbcore/query.py b/beets/dbcore/query.py index 1a7876675..424d51e7c 100644 --- a/beets/dbcore/query.py +++ b/beets/dbcore/query.py @@ -447,6 +447,36 @@ class OrQuery(MutableCollectionQuery): return any([q.match(item) for q in self.subqueries]) +class NotQuery(Query): + """A query that matches the negation of its `subquery`, as a shorcut for + performing `not(subquery)` without using regular expressions. + """ + def __init__(self, subquery): + self.subquery = subquery + + def clause(self): + clause, subvals = self.subquery.clause() + if clause: + return 'not ({0})'.format(clause), subvals + else: + # If there is no clause, there is nothing to negate. All the logic + # is handled by match() for slow queries. + return clause, subvals + + def match(self, item): + return not self.subquery.match(item) + + def __repr__(self): + return "{0.__class__.__name__}({0.subquery})".format(self) + + def __eq__(self, other): + return super(NotQuery, self).__eq__(other) and \ + self.subquery == other.subquery + + def __hash__(self): + return hash(('not', hash(self.subquery))) + + class TrueQuery(Query): """A query that always matches.""" def clause(self): diff --git a/beets/dbcore/queryparse.py b/beets/dbcore/queryparse.py index 34eea6b16..53b6a026e 100644 --- a/beets/dbcore/queryparse.py +++ b/beets/dbcore/queryparse.py @@ -24,6 +24,8 @@ import beets PARSE_QUERY_PART_REGEX = re.compile( # Non-capturing optional segment for the keyword. + r'(-|\^)?' # Negation prefixes. + r'(?:' r'(\S+?)' # The field key. r'(? (None, 'stapler', SubstringQuery) - 'color:red' -> ('color', 'red', SubstringQuery) - ':^Quiet' -> (None, '^Quiet', RegexpQuery) - 'color::b..e' -> ('color', 'b..e', RegexpQuery) + 'stapler' -> (None, 'stapler', SubstringQuery, False) + 'color:red' -> ('color', 'red', SubstringQuery, False) + ':^Quiet' -> (None, '^Quiet', RegexpQuery, False) + 'color::b..e' -> ('color', 'b..e', RegexpQuery, False) + '-color:red' -> ('color', 'red', SubstringQuery, True) Prefixes may be "escaped" with a backslash to disable the keying behavior. @@ -65,18 +68,19 @@ def parse_query_part(part, query_classes={}, prefixes={}, part = part.strip() match = PARSE_QUERY_PART_REGEX.match(part) - assert match # Regex should always match. - key = match.group(1) - term = match.group(2).replace('\:', ':') + assert match # Regex should always match + negate = bool(match.group(1)) + key = match.group(2) + term = match.group(3).replace('\:', ':') # Match the search term against the list of prefixes. for pre, query_class in prefixes.items(): if term.startswith(pre): - return key, term[len(pre):], query_class + return key, term[len(pre):], query_class, negate # No matching prefix: use type-based or fallback/default query. query_class = query_classes.get(key, default_class) - return key, term, query_class + return key, term, query_class, negate def construct_query_part(model_cls, prefixes, query_part): @@ -94,7 +98,7 @@ def construct_query_part(model_cls, prefixes, query_part): query_classes[k] = t.query # Parse the string. - key, pattern, query_class = \ + key, pattern, query_class, negate = \ parse_query_part(query_part, query_classes, prefixes) # No key specified. @@ -103,14 +107,24 @@ def construct_query_part(model_cls, prefixes, query_part): # The query type matches a specific field, but none was # specified. So we use a version of the query that matches # any field. - return query.AnyFieldQuery(pattern, model_cls._search_fields, - query_class) + q = query.AnyFieldQuery(pattern, model_cls._search_fields, + query_class) + if negate: + return query.NotQuery(q) + else: + return q else: # Other query type. - return query_class(pattern) + if negate: + return query.NotQuery(query_class(pattern)) + else: + return query_class(pattern) key = key.lower() - return query_class(key.lower(), pattern, key in model_cls._fields) + q = query_class(key.lower(), pattern, key in model_cls._fields) + if negate: + return query.NotQuery(q) + return q def query_from_strings(query_cls, model_cls, prefixes, query_parts): diff --git a/docs/changelog.rst b/docs/changelog.rst index 700a16d6d..81dcdd2b7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,10 @@ New: :doc:`/plugins/discogs` also adopts the same setting. * :doc:`/plugins/embyupdate`: A plugin to trigger a library refresh on a `Emby Server`_ if database changed. +* Queries can now use "not" logic: if you prepend a query term with "-" or + "^", items or albums matching that term will be excluded from the results. + For example, ``beet ls foo ^artist:bar`` will get all the items matching + `foo` but whose artist do not match `bar`. See :ref:`not_query`. :bug:`819` For developers: diff --git a/docs/reference/query.rst b/docs/reference/query.rst index 452681c35..6d83537d1 100644 --- a/docs/reference/query.rst +++ b/docs/reference/query.rst @@ -181,6 +181,35 @@ Find all items with a file modification time between 2008-12-01 and $ beet ls 'mtime:2008-12-01..2008-12-02' +.. _not_query: + +Query Term Negation +------------------- + +Query terms can also be negated, acting like a Boolean "not", by prepending +them with ``-`` or ``^``. This has the effect of returning all the items that +do **not** match the query term. For example, this command:: + + $ beet list ^love + +matches all the songs in the library that do not have "love" in any of their +fields. + +Negation can be combined with the rest of the query mechanisms, allowing to +negate specific fields, regular expressions, etc. For example, this command:: + + $ beet list -a artist:dylan ^year:1980..1990 "^album::the(y)?" + +matches all the albums with an artist containing "dylan", but excluding those +released on the eighties and those that have "the" or "they" on the title. + +Note that the ``-`` character is treated by most shells as a reserved character +for passing arguments, and as such needs to be escaped if using it for query +negation. In most UNIX derivatives shells, using a double dash ``--`` +(indicating that everything after that point should not be treated as +arguments) before the query terms should prevent conflicts, such as:: + + $ beet list -a -- artist:dylan -year:1980..1990 "-album::the(y)?" .. _pathquery: diff --git a/test/test_dbcore.py b/test/test_dbcore.py index 49fca2bb3..4e1f2dd37 100644 --- a/test/test_dbcore.py +++ b/test/test_dbcore.py @@ -352,7 +352,7 @@ class QueryParseTest(unittest.TestCase): part, {'year': dbcore.query.NumericQuery}, {':': dbcore.query.RegexpQuery}, - ) + )[:-1] # remove the negate flag def test_one_basic_term(self): q = 'test' diff --git a/test/test_query.py b/test/test_query.py index 36df04fd0..dae752581 100644 --- a/test/test_query.py +++ b/test/test_query.py @@ -715,6 +715,243 @@ class NoneQueryTest(unittest.TestCase, TestHelper): self.assertInResult(item, matched) +class NotQueryMatchTest(_common.TestCase): + """Test `query.NotQuery` matching against a single item, using the same + cases and assertions as on `MatchTest`, plus assertion on the negated + queries (ie. assertTrue(q) -> assertFalse(NotQuery(q))). + """ + def setUp(self): + super(NotQueryMatchTest, self).setUp() + self.item = _common.item() + + def test_regex_match_positive(self): + q = dbcore.query.RegexpQuery('album', '^the album$') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) + + def test_regex_match_negative(self): + q = dbcore.query.RegexpQuery('album', '^album$') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) + + def test_regex_match_non_string_value(self): + q = dbcore.query.RegexpQuery('disc', '^6$') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) + + def test_substring_match_positive(self): + q = dbcore.query.SubstringQuery('album', 'album') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) + + def test_substring_match_negative(self): + q = dbcore.query.SubstringQuery('album', 'ablum') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) + + def test_substring_match_non_string_value(self): + q = dbcore.query.SubstringQuery('disc', '6') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) + + def test_year_match_positive(self): + q = dbcore.query.NumericQuery('year', '1') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) + + def test_year_match_negative(self): + q = dbcore.query.NumericQuery('year', '10') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) + + def test_bitrate_range_positive(self): + q = dbcore.query.NumericQuery('bitrate', '100000..200000') + self.assertTrue(q.match(self.item)) + self.assertFalse(dbcore.query.NotQuery(q).match(self.item)) + + def test_bitrate_range_negative(self): + q = dbcore.query.NumericQuery('bitrate', '200000..300000') + self.assertFalse(q.match(self.item)) + self.assertTrue(dbcore.query.NotQuery(q).match(self.item)) + + def test_open_range(self): + q = dbcore.query.NumericQuery('bitrate', '100000..') + dbcore.query.NotQuery(q) + + +class NotQueryTest(DummyDataTestCase): + """Test `query.NotQuery` against the dummy data: + - `test_type_xxx`: tests for the negation of a particular XxxQuery class. + - `test_get_yyy`: tests on query strings (similar to `GetTest`) + """ + def assertNegationProperties(self, q): + """Given a Query `q`, assert that: + - q OR not(q) == all items + - q AND not(q) == 0 + - not(not(q)) == q + """ + not_q = dbcore.query.NotQuery(q) + # assert using OrQuery, AndQuery + q_or = dbcore.query.OrQuery([q, not_q]) + q_and = dbcore.query.AndQuery([q, not_q]) + self.assert_items_matched_all(self.lib.items(q_or)) + self.assert_items_matched(self.lib.items(q_and), []) + + # assert manually checking the item titles + all_titles = set([i.title for i in self.lib.items()]) + q_results = set([i.title for i in self.lib.items(q)]) + not_q_results = set([i.title for i in self.lib.items(not_q)]) + self.assertEqual(q_results.union(not_q_results), all_titles) + self.assertEqual(q_results.intersection(not_q_results), set()) + + # round trip + not_not_q = dbcore.query.NotQuery(not_q) + self.assertEqual(set([i.title for i in self.lib.items(q)]), + set([i.title for i in self.lib.items(not_not_q)])) + + def test_type_and(self): + # not(a and b) <-> not(a) or not(b) + q = dbcore.query.AndQuery([dbcore.query.BooleanQuery('comp', True), + dbcore.query.NumericQuery('year', '2002')]) + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['foo bar', 'beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_anyfield(self): + q = dbcore.query.AnyFieldQuery('foo', ['title', 'artist', 'album'], + dbcore.query.SubstringQuery) + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['baz qux']) + self.assertNegationProperties(q) + + def test_type_boolean(self): + q = dbcore.query.BooleanQuery('comp', True) + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_date(self): + q = dbcore.query.DateQuery('mtime', '0.0') + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, []) + self.assertNegationProperties(q) + + def test_type_false(self): + q = dbcore.query.FalseQuery() + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched_all(not_results) + self.assertNegationProperties(q) + + def test_type_match(self): + q = dbcore.query.MatchQuery('year', '2003') + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['foo bar', 'baz qux']) + self.assertNegationProperties(q) + + def test_type_none(self): + q = dbcore.query.NoneQuery('rg_track_gain') + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, []) + self.assertNegationProperties(q) + + def test_type_numeric(self): + q = dbcore.query.NumericQuery('year', '2001..2002') + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_or(self): + # not(a or b) <-> not(a) and not(b) + q = dbcore.query.OrQuery([dbcore.query.BooleanQuery('comp', True), + dbcore.query.NumericQuery('year', '2002')]) + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_regexp(self): + q = dbcore.query.RegexpQuery('artist', '^t') + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['foo bar']) + self.assertNegationProperties(q) + + def test_type_substring(self): + q = dbcore.query.SubstringQuery('album', 'ba') + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, ['beets 4 eva']) + self.assertNegationProperties(q) + + def test_type_true(self): + q = dbcore.query.TrueQuery() + not_results = self.lib.items(dbcore.query.NotQuery(q)) + self.assert_items_matched(not_results, []) + self.assertNegationProperties(q) + + def test_get_prefixes_keyed(self): + """Test both negation prefixes on a keyed query.""" + q0 = '-title:qux' + q1 = '^title:qux' + results0 = self.lib.items(q0) + results1 = self.lib.items(q1) + self.assert_items_matched(results0, ['foo bar', 'beets 4 eva']) + self.assert_items_matched(results1, ['foo bar', 'beets 4 eva']) + + def test_get_prefixes_unkeyed(self): + """Test both negation prefixes on an unkeyed query.""" + q0 = '-qux' + q1 = '^qux' + results0 = self.lib.items(q0) + results1 = self.lib.items(q1) + self.assert_items_matched(results0, ['foo bar', 'beets 4 eva']) + self.assert_items_matched(results1, ['foo bar', 'beets 4 eva']) + + def test_get_one_keyed_regexp(self): + q = r'-artist::t.+r' + results = self.lib.items(q) + self.assert_items_matched(results, ['foo bar', 'baz qux']) + + def test_get_one_unkeyed_regexp(self): + q = r'-:x$' + results = self.lib.items(q) + self.assert_items_matched(results, ['foo bar', 'beets 4 eva']) + + def test_get_multiple_terms(self): + q = 'baz -bar' + results = self.lib.items(q) + self.assert_items_matched(results, ['baz qux']) + + def test_get_mixed_terms(self): + q = 'baz -title:bar' + results = self.lib.items(q) + self.assert_items_matched(results, ['baz qux']) + + def test_fast_vs_slow(self): + """Test that the results are the same regardless of the `fast` flag + for negated `FieldQuery`s. + + TODO: investigate NoneQuery(fast=False), as it is raising + AttributeError: type object 'NoneQuery' has no attribute 'field' + at NoneQuery.match() (due to being @classmethod, and no self?) + """ + classes = [(dbcore.query.DateQuery, ['mtime', '0.0']), + (dbcore.query.MatchQuery, ['artist', 'one']), + # (dbcore.query.NoneQuery, ['rg_track_gain']), + (dbcore.query.NumericQuery, ['year', '2002']), + (dbcore.query.StringFieldQuery, ['year', '2001']), + (dbcore.query.RegexpQuery, ['album', '^.a']), + (dbcore.query.SubstringQuery, ['title', 'x'])] + + for klass, args in classes: + q_fast = dbcore.query.NotQuery(klass(*(args + [True]))) + q_slow = dbcore.query.NotQuery(klass(*(args + [False]))) + + try: + self.assertEqual([i.title for i in self.lib.items(q_fast)], + [i.title for i in self.lib.items(q_slow)]) + except NotImplementedError: + # ignore classes that do not provide `fast` implementation + pass + + def suite(): return unittest.TestLoader().loadTestsFromName(__name__)