diff --git a/beets/library.py b/beets/library.py index 6b7b087ee..d52820383 100644 --- a/beets/library.py +++ b/beets/library.py @@ -229,6 +229,60 @@ def format_for_path(value, key=None, pathmod=None): class InvalidFieldError(Exception): pass +class FixedDict(dict): + """A dict where keys can not be added after creation and keys are marked + as 'dirty' when their values are changed. + Accepts a list of `keys` and an optional `values` which should + be a dict compatible object. + Any key not in `values` will be implicitly added and given a `None` value. + """ + def __init__(self, keys, values=None): + values = values or {} + super(FixedDict, self).__init__(values) + seti = super(FixedDict, self).__setitem__ + for key in keys: + if not key in self: + seti(key, None) + self.dirty = {} + + def __setitem__(self, key, value): + if key not in self: + raise KeyError('{} is not a valid key.'.format(key)) + elif value != self[key]: + super(FixedDict, self).__setitem__(key, value) + self.dirty[key] = True + + +def set_flexattrs(obj, values): + """Create the initial flexible attribute dict for `obj` (e.g. Item/Album) + from given `values` which should be a dict compatible row of values + from the database. + + This sets the `flexattrs` attribute on `obj`. + """ + #make the initial, empty flexible attribute dict + flexattrns = plugins.item_fields() + flexattrs = FixedDict(flexattrns.keys()) + for key in flexattrs.iterkeys(): + flexattrs[key] = FixedDict(flexattrns[key]) + + k = values.keys() + if 'flexkeys' in k and 'flexvalues' in k and 'flexns' in k \ + and values['flexkeys']: + flexkeys = values['flexkeys'].split(',') + flexvalues = values['flexvalues'].split(',') + flexns = values['flexns'].split(',') + log.debug('flexkeys: %s - flexvals: %s - flexns: %s', + flexkeys, flexvalues, flexns) + for i, key in enumerate(flexkeys): + namespace = flexns[i] + if namespace not in flexattrs: + #not an active plugin, skip + continue + if not flexattrs.has_key(namespace): + flexattrs[namespace] = {} + flexattrs[flexns[i]][key] = flexvalues[i] + object.__setattr__(obj, 'flexattrs', flexattrs) # Library items (songs). @@ -252,12 +306,15 @@ class Item(object): def _fill_record(self, values): self.record = {} + for key in ITEM_KEYS: try: setattr(self, key, values[key]) except KeyError: setattr(self, key, None) + set_flexattrs(self, values) + def _clear_dirty(self): self.dirty = {} for key in ITEM_KEYS: @@ -266,7 +323,6 @@ class Item(object): def __repr__(self): return 'Item(' + repr(self.record) + ')' - # Item field accessors. def __getattr__(self, key): @@ -293,7 +349,7 @@ class Item(object): elif isinstance(value, buffer): value = str(value) - if key in ITEM_KEYS or key in plugins.item_fields(): + if key in ITEM_KEYS: # If the value changed, mark the field as dirty. if (key not in self.record) or (self.record[key] != value): self.record[key] = value @@ -303,6 +359,15 @@ class Item(object): else: super(Item, self).__setattr__(key, value) + def __getitem__(self, key): + """Entry point to flexible attributes. + `key` should be a plugin name or an otherwise + valid flexible attr namespace. + """ + if key in self.flexattrs: + return self.flexattrs[key] + else: + raise KeyError('{} is not a valid attribute namespace'.format(key)) # Interaction with file metadata. @@ -453,7 +518,7 @@ class Query(object): to substitute in for ?s in the query. """ clause, subvals = self.clause() - return ('SELECT ' + columns + ' FROM items WHERE ' + clause, subvals) + return ('SELECT ' + columns + ' FROM unified_items WHERE ' + clause, subvals) def count(self, tx): """Returns `(num, length)` where `num` is the number of items in @@ -461,7 +526,7 @@ class Query(object): length in seconds. """ clause, subvals = self.clause() - statement = 'SELECT COUNT(id), SUM(length) FROM items WHERE ' + clause + statement = 'SELECT COUNT(id), SUM(length) FROM unified_items WHERE ' + clause result = tx.query(statement, subvals)[0] return (result[0], result[1] or 0.0) @@ -469,16 +534,23 @@ class FieldQuery(Query): """An abstract query that searches in a specific field for a pattern. """ - def __init__(self, field, pattern): + def __init__(self, field, pattern, namespace=None, entity='item'): self.field = field self.pattern = pattern + self.namespace = namespace + self.entity = entity class MatchQuery(FieldQuery): """A query that looks for exact matches in an item field.""" def clause(self): + pattern = self.pattern if self.field == 'path' and isinstance(pattern, str): pattern = buffer(pattern) + if self.namespace: + #give a flexible attribute clause + c = 'key = ? AND value = ? AND namespace = ?' + return c, [self.field, pattern, self.namespace] return self.field + " = ?", [pattern] def match(self, item): @@ -489,7 +561,12 @@ class SubstringQuery(FieldQuery): def clause(self): search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%') .replace('_','\\_')) + '%' - clause = self.field + " like ? escape '\\'" + if self.namespace: + clause = """key = {0} AND namespace = {1} + AND value like ? escape '\\'""".format( + self.field, self.namespace) + else: + clause = self.field + " like ? escape '\\'" subvals = [search] return clause, subvals @@ -499,12 +576,16 @@ class SubstringQuery(FieldQuery): class RegexpQuery(FieldQuery): """A query that matches a regular expression in a specific item field.""" - def __init__(self, field, pattern): - super(RegexpQuery, self).__init__(field, pattern) + def __init__(self, field, pattern, namespace=None, entity='item'): + super(RegexpQuery, self).__init__(field, pattern, namespace, entity) self.regexp = re.compile(pattern) def clause(self): - clause = self.field + " REGEXP ?" + if self.namespace: + clause = 'key = {0} AND namespace = {1} AND value REGEXP ?'.format( + self.field, self.namespace) + else: + clause = self.field + " REGEXP ?" subvals = [self.pattern] return clause, subvals @@ -553,14 +634,53 @@ class CollectionQuery(Query): """Returns a clause created by joining together the clauses of all subqueries with the string joiner (padded by spaces). """ + entity = None + flexclause_parts = [] + flexsubvals = [] clause_parts = [] subvals = [] for subq in self.subqueries: - subq_clause, subq_subvals = subq.clause() - clause_parts.append('(' + subq_clause + ')') - subvals += subq_subvals + if hasattr(subq, 'namespace') and subq.namespace: + #it's a flex attr query, initiate -ugly hack- mode + entity = subq.entity + clauses = flexclause_parts + subs = flexsubvals + subq_clause, subq_subvals = subq.clause() + if joiner.lower() == 'and': + #flexible attrs need this nested mess for AND queries + subq_clause = ''' + EXISTS (SELECT 1 FROM {0}_attributes AS ia + WHERE {1} AND ia.entity_id = unified_{0}s.id) + '''.format(entity, '('+subq_clause+')') + else: + clauses = clause_parts + subs = subvals + subq_clause, subq_subvals = subq.clause() + clauses.append('(' + subq_clause + ')') + subs += subq_subvals + clause = (' ' + joiner + ' ').join(clause_parts) - return clause, subvals + + if not flexclause_parts: + return clause, subvals + + fclause = (' ' + joiner + ' ').join(flexclause_parts) + if joiner.lower() == 'and': + pass + else: + flexorclause = ''' + id IN ( + SELECT entity_id FROM {0}_attributes + WHERE {1}) + ''' + fclause = flexorclause.format(entity, fclause) + if clause and fclause: + clause = clause+' '+joiner+' '+fclause + elif clause: + clause = clause + elif fclause: + clause = fclause + return clause, subvals+flexsubvals # Regular expression for _parse_query_part, below. _pq_regex = re.compile( @@ -938,6 +1058,7 @@ class BaseAlbum(object): def __init__(self, library, record): super(BaseAlbum, self).__setattr__('_library', library) super(BaseAlbum, self).__setattr__('_record', record) + set_flexattrs(self, record) def __getattr__(self, key): """Get the value for an album attribute.""" @@ -963,6 +1084,16 @@ class BaseAlbum(object): else: super(BaseAlbum, self).__setattr__(key, value) + def __getitem__(self, key): + """Entry point to flexible attributes. + `key` should be a plugin name or an otherwise + valid flexible attr namespace. + """ + if key in self.flexattrs: + return self.flexattrs[key] + else: + raise KeyError('{} is not a valid attribute namespace'.format(key)) + def load(self): """Refresh this album's cached metadata from the library. """ @@ -1122,11 +1253,22 @@ class Library(BaseLibrary): CREATE TABLE IF NOT EXISTS {0}_attributes ( id INTEGER PRIMARY KEY, entity_id INTEGER, + namespace TEXT, key TEXT, value TEXT, - UNIQUE(entity_id, key) ON CONFLICT REPLACE); + UNIQUE(entity_id, key, namespace) ON CONFLICT REPLACE); CREATE INDEX IF NOT EXISTS {0}_id_attribute ON {0}_attributes (entity_id); + CREATE VIEW IF NOT EXISTS unified_{0}s AS + SELECT + GROUP_CONCAT(ia.key) AS flexkeys, + GROUP_CONCAT(ia.value) AS flexvalues, + GROUP_CONCAT(ia.namespace) AS flexns, + {0}s.* + FROM {0}s + LEFT JOIN {0}_attributes AS ia + ON {0}s.id == ia.entity_id + GROUP BY {0}s.id; """.format(entity)) def _connection(self): @@ -1280,7 +1422,7 @@ class Library(BaseLibrary): load_id = item.id with self.transaction() as tx: - rows = tx.query('SELECT * FROM items WHERE id=?', (load_id,)) + rows = tx.query('SELECT * FROM unified_items WHERE id=?', (load_id,)) item._fill_record(rows[0]) item._clear_dirty() @@ -1309,15 +1451,15 @@ class Library(BaseLibrary): subvars.append(store_id) tx.mutate(query, subvars) - # Flexible attributes. - for key in plugins.item_fields(): - if item.dirty.get(key) or store_all: - tx.mutate( - 'INSERT INTO item_attributes (entity_id, key, value)' - ' VALUES (?, ?, ?)', - (store_id, key, getattr(item, key)) - ) - + #flexible attributes + flexins = '''INSERT INTO item_attributes + (entity_id, key, value, namespace) + VALUES (?,?,?,?);''' + for ns,attrs in item.flexattrs.iteritems(): + for key in attrs.dirty: + tx.mutate(flexins, (store_id, key, attrs[key], ns)) + attrs.dirty.clear() + item._clear_dirty() self._memotable = {} @@ -1396,7 +1538,7 @@ class Library(BaseLibrary): # "Add" the artist to the query. query = AndQuery((query, MatchQuery('albumartist', artist))) where, subvals = query.clause() - sql = "SELECT * FROM albums " + \ + sql = "SELECT * FROM unified_albums " + \ "WHERE " + where + \ " ORDER BY %s, album" % \ _orelse("albumartist_sort", "albumartist") @@ -1415,7 +1557,7 @@ class Library(BaseLibrary): super_query = AndQuery(queries) where, subvals = super_query.clause() - sql = "SELECT * FROM items " + \ + sql = "SELECT * FROM unified_items " + \ "WHERE " + where + \ " ORDER BY %s, album, disc, track" % \ _orelse("artist_sort", "artist") @@ -1431,7 +1573,7 @@ class Library(BaseLibrary): """Fetch an Item by its ID. Returns None if no match is found. """ with self.transaction() as tx: - rows = tx.query("SELECT * FROM items WHERE id=?", (id,)) + rows = tx.query("SELECT * FROM unified_items WHERE id=?", (id,)) it = ResultIterator(rows) try: return it.next() @@ -1552,7 +1694,7 @@ class Album(BaseAlbum): """ with self._library.transaction() as tx: rows = tx.query( - 'SELECT * FROM items WHERE album_id=?', + 'SELECT * FROM unified_items WHERE album_id=?', (self.id,) ) return ResultIterator(rows) diff --git a/beets/plugins.py b/beets/plugins.py index 177fce032..c8875ec34 100755 --- a/beets/plugins.py +++ b/beets/plugins.py @@ -48,8 +48,6 @@ class BeetsPlugin(object): self.template_funcs = {} if not self.template_fields: self.template_fields = {} - self.item_fields = [] - self.album_fields = [] def commands(self): """Should return a list of beets.ui.Subcommand objects for @@ -95,6 +93,18 @@ class BeetsPlugin(object): """ return {} + def item_attributes(self): + """Returns a list of registered flexible attribute fields + for `Item` entities. + """ + return [] + + def album_attributes(self): + """Returns a list of registered flexible attribute fields + for `Album` entities. + """ + return [] + listeners = None @classmethod @@ -291,24 +301,23 @@ def import_stages(): return stages def item_fields(): - """Get a list of strings indicating registered flexible Item - attributes. + """Get a dict of string lists indicating registered + flexible Item attributes in the form {'pluginname':[fields...]}. """ - fields = [] + fields = {} for plugin in find_plugins(): - fields += plugin.item_fields + fields[plugin.name] = plugin.item_attributes() return fields def album_fields(): - """Get a list of strings indicating registered flexible Album - attributes. + """Get a dict of string lists indicating registered + flexible Album attributes in the form {'pluginname':[fields...]}. """ - fields = [] + fields = {} for plugin in find_plugins(): - fields += plugin.album_fields + fields[plugin.name] = plugin.album_attributes() return fields - # Event dispatch. def event_handlers():