From f5d658c58fd2132876fb1a6ab636ebc606cbeb28 Mon Sep 17 00:00:00 2001 From: steini Date: Mon, 4 Mar 2013 05:55:43 +0000 Subject: [PATCH 1/5] An attempt at flexible attributes with plugin namespaces. Mostly untested. --- beets/library.py | 102 ++++++++++++++++++++++++++++++++++++++--------- beets/plugins.py | 31 +++++++++----- 2 files changed, 104 insertions(+), 29 deletions(-) diff --git a/beets/library.py b/beets/library.py index 6b7b087ee..8a2ea92d4 100644 --- a/beets/library.py +++ b/beets/library.py @@ -229,6 +229,28 @@ def format_for_path(value, key=None, pathmod=None): class InvalidFieldError(Exception): pass +class FixedDict(dict): + """A dict where keys can not be added after creation and keys are marked + as 'dirty' when their values are changed. + Accepts a list of `keys` and an optional `values` which should + be a dict compatible object. + Any key not in `values` will be implicitly added and given a `None` value. + """ + def __init__(self, keys, values=None): + values = values or {} + super(FixedDict, self).__init__(values) + seti = super(FixedDict, self).__setitem__ + for key in keys: + if not key in self: + seti(key, None) + self.dirty = {} + + def __setitem__(self, key, value): + if key not in self: + raise KeyError('{} is not a valid key.'.format(key)) + elif value != self[key]: + super(FixedDict, self).__setitem__(key, value) + self.dirty[key] = True # Library items (songs). @@ -252,12 +274,34 @@ class Item(object): def _fill_record(self, values): self.record = {} + for key in ITEM_KEYS: try: setattr(self, key, values[key]) except KeyError: setattr(self, key, None) + #make the initial, empty flexible attribute dict + flexattrns = plugins.item_fields() + self.flexattrs = FixedDict(flexattrns.keys()) + for key in self.flexattrs.iterkeys(): + self.flexattrs[key] = FixedDict(flexattrns[key]) + + #treat flexattrs values as they would come from the database + if values['flexkeys'] and values['flexvalues'] and values['flexns']: + flexkeys = values['flexkeys'].split(',') + flexvalues = values['flexvalues'].split(',') + flexns = values['flexns'].split(',') + for i, key in enumerate(flexkeys): + namespace = flexns[i] + if namespace not in self.flexattrs: + #not an active plugin, skip + continue + value = flexvalues[i] + if not self.flexattrs.has_key(namespace): + self.flexattrs[namespace] = {} + self.flexattrs[flexns[i]][key] = flexvalues[i] + def _clear_dirty(self): self.dirty = {} for key in ITEM_KEYS: @@ -266,7 +310,6 @@ class Item(object): def __repr__(self): return 'Item(' + repr(self.record) + ')' - # Item field accessors. def __getattr__(self, key): @@ -293,7 +336,7 @@ class Item(object): elif isinstance(value, buffer): value = str(value) - if key in ITEM_KEYS or key in plugins.item_fields(): + if key in ITEM_KEYS: # If the value changed, mark the field as dirty. if (key not in self.record) or (self.record[key] != value): self.record[key] = value @@ -303,6 +346,15 @@ class Item(object): else: super(Item, self).__setattr__(key, value) + def __getitem__(self, key): + """Entry point to flexible attributes. + `key` should be a plugin name or an otherwise + valid flexible attr namespace. + """ + if key in self.flexattrs: + return self.flexattrs[key] + else: + raise KeyError('{} is not a valid attribute namespace'.format(key)) # Interaction with file metadata. @@ -453,7 +505,7 @@ class Query(object): to substitute in for ?s in the query. """ clause, subvals = self.clause() - return ('SELECT ' + columns + ' FROM items WHERE ' + clause, subvals) + return ('SELECT ' + columns + ' FROM unified_items WHERE ' + clause, subvals) def count(self, tx): """Returns `(num, length)` where `num` is the number of items in @@ -461,7 +513,7 @@ class Query(object): length in seconds. """ clause, subvals = self.clause() - statement = 'SELECT COUNT(id), SUM(length) FROM items WHERE ' + clause + statement = 'SELECT COUNT(id), SUM(length) FROM unified_items WHERE ' + clause result = tx.query(statement, subvals)[0] return (result[0], result[1] or 0.0) @@ -1122,11 +1174,22 @@ class Library(BaseLibrary): CREATE TABLE IF NOT EXISTS {0}_attributes ( id INTEGER PRIMARY KEY, entity_id INTEGER, + namespace TEXT, key TEXT, value TEXT, - UNIQUE(entity_id, key) ON CONFLICT REPLACE); + UNIQUE(entity_id, key, namespace) ON CONFLICT REPLACE); CREATE INDEX IF NOT EXISTS {0}_id_attribute ON {0}_attributes (entity_id); + CREATE VIEW IF NOT EXISTS unified_{0}s AS + SELECT + GROUP_CONCAT(ia.key) AS flexkeys, + GROUP_CONCAT(ia.value) AS flexvalues, + GROUP_CONCAT(ia.namespace) AS flexns, + {0}s.* + FROM {0}s + LEFT JOIN {0}_attributes AS ia + ON {0}s.id == ia.entity_id + GROUP BY {0}s.id; """.format(entity)) def _connection(self): @@ -1280,7 +1343,7 @@ class Library(BaseLibrary): load_id = item.id with self.transaction() as tx: - rows = tx.query('SELECT * FROM items WHERE id=?', (load_id,)) + rows = tx.query('SELECT * FROM unified_items WHERE id=?', (load_id,)) item._fill_record(rows[0]) item._clear_dirty() @@ -1309,15 +1372,18 @@ class Library(BaseLibrary): subvars.append(store_id) tx.mutate(query, subvars) - # Flexible attributes. - for key in plugins.item_fields(): - if item.dirty.get(key) or store_all: - tx.mutate( - 'INSERT INTO item_attributes (entity_id, key, value)' - ' VALUES (?, ?, ?)', - (store_id, key, getattr(item, key)) - ) - + #flexible attributes + flexins = '''INSERT INTO item_attributes + (entity_id, key, value, namespace) + VALUES (?,?,?);''' + flexup = '''UPDATE item_attributes SET value = ? WHERE + entity_id = ? AND key = ? AND namespace = ?;''' + for ns,attrs in item.flexattrs.iteritems(): + for key in attrs.dirty: + #TODO: use an upsert method to store attrs + tx.mutate(flexins, (store_id, key, attrs[key], ns)) + attrs.dirty.clear() + item._clear_dirty() self._memotable = {} @@ -1415,7 +1481,7 @@ class Library(BaseLibrary): super_query = AndQuery(queries) where, subvals = super_query.clause() - sql = "SELECT * FROM items " + \ + sql = "SELECT * FROM unified_items " + \ "WHERE " + where + \ " ORDER BY %s, album, disc, track" % \ _orelse("artist_sort", "artist") @@ -1431,7 +1497,7 @@ class Library(BaseLibrary): """Fetch an Item by its ID. Returns None if no match is found. """ with self.transaction() as tx: - rows = tx.query("SELECT * FROM items WHERE id=?", (id,)) + rows = tx.query("SELECT * FROM unified_items WHERE id=?", (id,)) it = ResultIterator(rows) try: return it.next() @@ -1552,7 +1618,7 @@ class Album(BaseAlbum): """ with self._library.transaction() as tx: rows = tx.query( - 'SELECT * FROM items WHERE album_id=?', + 'SELECT * FROM unified_items WHERE album_id=?', (self.id,) ) return ResultIterator(rows) diff --git a/beets/plugins.py b/beets/plugins.py index 177fce032..c8875ec34 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -48,8 +48,6 @@ class BeetsPlugin(object): self.template_funcs = {} if not self.template_fields: self.template_fields = {} - self.item_fields = [] - self.album_fields = [] def commands(self): """Should return a list of beets.ui.Subcommand objects for @@ -95,6 +93,18 @@ class BeetsPlugin(object): """ return {} + def item_attributes(self): + """Returns a list of registered flexible attribute fields + for `Item` entities. + """ + return [] + + def album_attributes(self): + """Returns a list of registered flexible attribute fields + for `Album` entities. + """ + return [] + listeners = None @classmethod @@ -291,24 +301,23 @@ def import_stages(): return stages def item_fields(): - """Get a list of strings indicating registered flexible Item - attributes. + """Get a dict of string lists indicating registered + flexible Item attributes in the form {'pluginname':[fields...]}. """ - fields = [] + fields = {} for plugin in find_plugins(): - fields += plugin.item_fields + fields[plugin.name] = plugin.item_attributes() return fields def album_fields(): - """Get a list of strings indicating registered flexible Album - attributes. + """Get a dict of string lists indicating registered + flexible Album attributes in the form {'pluginname':[fields...]}. """ - fields = [] + fields = {} for plugin in find_plugins(): - fields += plugin.album_fields + fields[plugin.name] = plugin.album_attributes() return fields - # Event dispatch. def event_handlers(): From fbd85ef6baa0542bf9dc91c535e621e5c9397ee7 Mon Sep 17 00:00:00 2001 From: steini Date: Tue, 5 Mar 2013 05:20:03 +0000 Subject: [PATCH 2/5] Started hacking `CollectionQuery` to search flexible attribute fields. --- beets/library.py | 52 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 8a2ea92d4..41cde2c37 100644 --- a/beets/library.py +++ b/beets/library.py @@ -521,16 +521,23 @@ class FieldQuery(Query): """An abstract query that searches in a specific field for a pattern. """ - def __init__(self, field, pattern): + def __init__(self, field, pattern, namespace=None, entity='item'): self.field = field self.pattern = pattern + self.namespace = namespace + self.entity = entity class MatchQuery(FieldQuery): """A query that looks for exact matches in an item field.""" def clause(self): + pattern = self.pattern if self.field == 'path' and isinstance(pattern, str): pattern = buffer(pattern) + if self.namespace: + #give a flexible attribute clause + c = 'key = ? AND value = ? AND namespace = ?' + return c, [self.field, pattern, self.namespace] return self.field + " = ?", [pattern] def match(self, item): @@ -551,7 +558,7 @@ class SubstringQuery(FieldQuery): class RegexpQuery(FieldQuery): """A query that matches a regular expression in a specific item field.""" - def __init__(self, field, pattern): + def __init__(self, field, pattern, namespace=None, entity='item'): super(RegexpQuery, self).__init__(field, pattern) self.regexp = re.compile(pattern) @@ -605,14 +612,47 @@ class CollectionQuery(Query): """Returns a clause created by joining together the clauses of all subqueries with the string joiner (padded by spaces). """ + entity = None + flexclause_parts = [] + flexsubvals = [] clause_parts = [] subvals = [] for subq in self.subqueries: - subq_clause, subq_subvals = subq.clause() - clause_parts.append('(' + subq_clause + ')') - subvals += subq_subvals + if subq.namespace: + #it's a flex attr query, initiate -ugly hack- mode + entity = subq.entity + clauses = flexclause_parts + subs = flexsubvals + subq_clause, subq_subvals = subq.clause() + if joiner.lower() == 'and': + #flexible attrs need this nested mess for AND queries + subq_clause = ''' + EXISTS (SELECT 1 FROM {0}_attributes AS ia + WHERE {1} AND ia.entity_id = unified_{0}s.id) + '''.format(entity, '('+subq_clause+')') + else: + clauses = clause_parts + subs = subvals + subq_clause, subq_subvals = subq.clause() + clauses.append('(' + subq_clause + ')') + subs += subq_subvals + clause = (' ' + joiner + ' ').join(clause_parts) - return clause, subvals + + if not flexclause_parts: + return clause, subvals + + fclause = (' ' + joiner + ' ').join(flexclause_parts) + if joiner.lower() == 'and': + pass + else: + flexorclause = ''' + id IN ( + SELECT entity_id FROM {0}_attributes + WHERE {1}) + ''' + fclause = flexorclause.format(entity, fclause) + return clause+' '+joiner+' '+fclause, subvals+flexsubvals # Regular expression for _parse_query_part, below. _pq_regex = re.compile( From 8e342a16a14f3cb61c86e91c81425410b5e1ce3b Mon Sep 17 00:00:00 2001 From: steini Date: Wed, 6 Mar 2013 14:38:17 +0000 Subject: [PATCH 3/5] Fixed up a couple of query classes for flex attr support. --- beets/library.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 41cde2c37..42aa4639a 100644 --- a/beets/library.py +++ b/beets/library.py @@ -548,7 +548,12 @@ class SubstringQuery(FieldQuery): def clause(self): search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%') .replace('_','\\_')) + '%' - clause = self.field + " like ? escape '\\'" + if self.namespace: + clause = """key = {0} AND namespace = {1} + AND value like ? escape '\\'""".format( + self.field, self.namespace) + else: + clause = self.field + " like ? escape '\\'" subvals = [search] return clause, subvals @@ -559,11 +564,15 @@ class SubstringQuery(FieldQuery): class RegexpQuery(FieldQuery): """A query that matches a regular expression in a specific item field.""" def __init__(self, field, pattern, namespace=None, entity='item'): - super(RegexpQuery, self).__init__(field, pattern) + super(RegexpQuery, self).__init__(field, pattern, namespace, entity) self.regexp = re.compile(pattern) def clause(self): - clause = self.field + " REGEXP ?" + if self.namespace: + clause = 'key = {0} AND namespace = {1} AND value REGEXP ?'.format( + self.field, self.namespace) + else: + clause = self.field + " REGEXP ?" subvals = [self.pattern] return clause, subvals @@ -1416,11 +1425,8 @@ class Library(BaseLibrary): flexins = '''INSERT INTO item_attributes (entity_id, key, value, namespace) VALUES (?,?,?);''' - flexup = '''UPDATE item_attributes SET value = ? WHERE - entity_id = ? AND key = ? AND namespace = ?;''' for ns,attrs in item.flexattrs.iteritems(): for key in attrs.dirty: - #TODO: use an upsert method to store attrs tx.mutate(flexins, (store_id, key, attrs[key], ns)) attrs.dirty.clear() From 7d1f67881ebaf22b73dd3d17eb4b2a19b668627b Mon Sep 17 00:00:00 2001 From: steini Date: Wed, 6 Mar 2013 18:29:22 +0000 Subject: [PATCH 4/5] ironing out some kinks --- beets/library.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/beets/library.py b/beets/library.py index 42aa4639a..dc4925a7f 100644 --- a/beets/library.py +++ b/beets/library.py @@ -288,10 +288,13 @@ class Item(object): self.flexattrs[key] = FixedDict(flexattrns[key]) #treat flexattrs values as they would come from the database - if values['flexkeys'] and values['flexvalues'] and values['flexns']: + k = values.keys() + if 'flexkeys' in k and 'flexvalues' in k and 'flexns' in k: flexkeys = values['flexkeys'].split(',') flexvalues = values['flexvalues'].split(',') flexns = values['flexns'].split(',') + log.debug('flexkeys: %s - flexvals: %s - flexns: %s', + flexkeys, flexvalues, flexns) for i, key in enumerate(flexkeys): namespace = flexns[i] if namespace not in self.flexattrs: @@ -300,7 +303,7 @@ class Item(object): value = flexvalues[i] if not self.flexattrs.has_key(namespace): self.flexattrs[namespace] = {} - self.flexattrs[flexns[i]][key] = flexvalues[i] + self.flexattrs[flexns[i]][key] = flexvalues[i] def _clear_dirty(self): self.dirty = {} @@ -627,7 +630,7 @@ class CollectionQuery(Query): clause_parts = [] subvals = [] for subq in self.subqueries: - if subq.namespace: + if hasattr(subq, 'namespace') and subq.namespace: #it's a flex attr query, initiate -ugly hack- mode entity = subq.entity clauses = flexclause_parts @@ -661,7 +664,13 @@ class CollectionQuery(Query): WHERE {1}) ''' fclause = flexorclause.format(entity, fclause) - return clause+' '+joiner+' '+fclause, subvals+flexsubvals + if clause and fclause: + clause = clause+' '+joiner+' '+fclause + elif clause: + clause = clause + elif fclause: + clause = fclause + return clause, subvals+flexsubvals # Regular expression for _parse_query_part, below. _pq_regex = re.compile( @@ -1424,7 +1433,7 @@ class Library(BaseLibrary): #flexible attributes flexins = '''INSERT INTO item_attributes (entity_id, key, value, namespace) - VALUES (?,?,?);''' + VALUES (?,?,?,?);''' for ns,attrs in item.flexattrs.iteritems(): for key in attrs.dirty: tx.mutate(flexins, (store_id, key, attrs[key], ns)) @@ -1508,7 +1517,7 @@ class Library(BaseLibrary): # "Add" the artist to the query. query = AndQuery((query, MatchQuery('albumartist', artist))) where, subvals = query.clause() - sql = "SELECT * FROM albums " + \ + sql = "SELECT * FROM unified_albums " + \ "WHERE " + where + \ " ORDER BY %s, album" % \ _orelse("albumartist_sort", "albumartist") From 410bdf6dddaa9cddc869baaad96796c5e0700d80 Mon Sep 17 00:00:00 2001 From: steini Date: Wed, 6 Mar 2013 23:25:31 +0000 Subject: [PATCH 5/5] Code for reading attributes from concatenated flex columns into dicts moved to an outside methods to make it usable for both BaseAlbum and Item objects. --- beets/library.py | 67 +++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/beets/library.py b/beets/library.py index dc4925a7f..d52820383 100644 --- a/beets/library.py +++ b/beets/library.py @@ -252,6 +252,38 @@ class FixedDict(dict): super(FixedDict, self).__setitem__(key, value) self.dirty[key] = True + +def set_flexattrs(obj, values): + """Create the initial flexible attribute dict for `obj` (e.g. Item/Album) + from given `values` which should be a dict compatible row of values + from the database. + + This sets the `flexattrs` attribute on `obj`. + """ + #make the initial, empty flexible attribute dict + flexattrns = plugins.item_fields() + flexattrs = FixedDict(flexattrns.keys()) + for key in flexattrs.iterkeys(): + flexattrs[key] = FixedDict(flexattrns[key]) + + k = values.keys() + if 'flexkeys' in k and 'flexvalues' in k and 'flexns' in k \ + and values['flexkeys']: + flexkeys = values['flexkeys'].split(',') + flexvalues = values['flexvalues'].split(',') + flexns = values['flexns'].split(',') + log.debug('flexkeys: %s - flexvals: %s - flexns: %s', + flexkeys, flexvalues, flexns) + for i, key in enumerate(flexkeys): + namespace = flexns[i] + if namespace not in flexattrs: + #not an active plugin, skip + continue + if not flexattrs.has_key(namespace): + flexattrs[namespace] = {} + flexattrs[flexns[i]][key] = flexvalues[i] + object.__setattr__(obj, 'flexattrs', flexattrs) + # Library items (songs). class Item(object): @@ -281,29 +313,7 @@ class Item(object): except KeyError: setattr(self, key, None) - #make the initial, empty flexible attribute dict - flexattrns = plugins.item_fields() - self.flexattrs = FixedDict(flexattrns.keys()) - for key in self.flexattrs.iterkeys(): - self.flexattrs[key] = FixedDict(flexattrns[key]) - - #treat flexattrs values as they would come from the database - k = values.keys() - if 'flexkeys' in k and 'flexvalues' in k and 'flexns' in k: - flexkeys = values['flexkeys'].split(',') - flexvalues = values['flexvalues'].split(',') - flexns = values['flexns'].split(',') - log.debug('flexkeys: %s - flexvals: %s - flexns: %s', - flexkeys, flexvalues, flexns) - for i, key in enumerate(flexkeys): - namespace = flexns[i] - if namespace not in self.flexattrs: - #not an active plugin, skip - continue - value = flexvalues[i] - if not self.flexattrs.has_key(namespace): - self.flexattrs[namespace] = {} - self.flexattrs[flexns[i]][key] = flexvalues[i] + set_flexattrs(self, values) def _clear_dirty(self): self.dirty = {} @@ -1048,6 +1058,7 @@ class BaseAlbum(object): def __init__(self, library, record): super(BaseAlbum, self).__setattr__('_library', library) super(BaseAlbum, self).__setattr__('_record', record) + set_flexattrs(self, record) def __getattr__(self, key): """Get the value for an album attribute.""" @@ -1073,6 +1084,16 @@ class BaseAlbum(object): else: super(BaseAlbum, self).__setattr__(key, value) + def __getitem__(self, key): + """Entry point to flexible attributes. + `key` should be a plugin name or an otherwise + valid flexible attr namespace. + """ + if key in self.flexattrs: + return self.flexattrs[key] + else: + raise KeyError('{} is not a valid attribute namespace'.format(key)) + def load(self): """Refresh this album's cached metadata from the library. """