Merge pull request #208 from steinitzu/flexattr

partial flexattrs

beets/library.py

@@ -229,6 +229,60 @@ def format_for_path(value, key=None, pathmod=None):
 class InvalidFieldError(Exception):
     pass
 
+class FixedDict(dict):
+    """A dict where keys can not be added after creation and keys are marked 
+    as 'dirty' when their values are changed.
+    Accepts a list of `keys` and an optional `values` which should 
+    be a dict compatible object.
+    Any key not in `values` will be implicitly added and given a `None` value.
+    """
+    def __init__(self, keys, values=None):
+        values = values or {}
+        super(FixedDict, self).__init__(values)
+        seti = super(FixedDict, self).__setitem__
+        for key in keys:
+            if not key in self:
+                seti(key, None)
+        self.dirty = {}
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            raise KeyError('{} is not a valid key.'.format(key))
+        elif value != self[key]:
+            super(FixedDict, self).__setitem__(key, value)
+            self.dirty[key] = True
+
+
+def set_flexattrs(obj, values):
+    """Create the initial flexible attribute dict for `obj` (e.g. Item/Album)
+    from given `values` which should be a dict compatible row of values 
+    from the database.
+
+    This sets the `flexattrs` attribute on `obj`.
+    """
+    #make the initial, empty flexible attribute dict
+    flexattrns = plugins.item_fields() 
+    flexattrs = FixedDict(flexattrns.keys())
+    for key in flexattrs.iterkeys():
+        flexattrs[key] = FixedDict(flexattrns[key])
+
+    k = values.keys()
+    if 'flexkeys' in k and 'flexvalues' in k and 'flexns' in k \
+            and values['flexkeys']:
+        flexkeys = values['flexkeys'].split(',')
+        flexvalues = values['flexvalues'].split(',')
+        flexns = values['flexns'].split(',')
+        log.debug('flexkeys: %s - flexvals: %s - flexns: %s',
+                  flexkeys, flexvalues, flexns)
+        for i, key in enumerate(flexkeys):
+            namespace = flexns[i]
+            if namespace not in flexattrs:
+                #not an active plugin, skip
+                continue
+            if not flexattrs.has_key(namespace):
+                flexattrs[namespace] = {}
+            flexattrs[flexns[i]][key] = flexvalues[i]
+    object.__setattr__(obj, 'flexattrs', flexattrs)
 
 # Library items (songs).
 
@@ -252,12 +306,15 @@ class Item(object):
 
     def _fill_record(self, values):
         self.record = {}
+
         for key in ITEM_KEYS:
             try:
                 setattr(self, key, values[key])
             except KeyError:
                 setattr(self, key, None)
 
+        set_flexattrs(self, values)
+
     def _clear_dirty(self):
         self.dirty = {}
         for key in ITEM_KEYS:
@@ -266,7 +323,6 @@ class Item(object):
     def __repr__(self):
         return 'Item(' + repr(self.record) + ')'
 
-
     # Item field accessors.
 
     def __getattr__(self, key):
@@ -293,7 +349,7 @@ class Item(object):
             elif isinstance(value, buffer):
                 value = str(value)
 
-        if key in ITEM_KEYS or key in plugins.item_fields():
+        if key in ITEM_KEYS:
             # If the value changed, mark the field as dirty.
             if (key not in self.record) or (self.record[key] != value):
                 self.record[key] = value
@@ -303,6 +359,15 @@ class Item(object):
         else:
             super(Item, self).__setattr__(key, value)
 
+    def __getitem__(self, key):
+        """Entry point to flexible attributes.
+        `key` should be a plugin name or an otherwise 
+        valid flexible attr namespace.
+        """
+        if key in self.flexattrs:
+            return self.flexattrs[key]
+        else:
+            raise KeyError('{} is not a valid attribute namespace'.format(key))
 
     # Interaction with file metadata.
 
@@ -453,7 +518,7 @@ class Query(object):
         to substitute in for ?s in the query.
         """
         clause, subvals = self.clause()
-        return ('SELECT ' + columns + ' FROM items WHERE ' + clause, subvals)
+        return ('SELECT ' + columns + ' FROM unified_items WHERE ' + clause, subvals)
 
     def count(self, tx):
         """Returns `(num, length)` where `num` is the number of items in
@@ -461,7 +526,7 @@ class Query(object):
         length in seconds.
         """
         clause, subvals = self.clause()
-        statement = 'SELECT COUNT(id), SUM(length) FROM items WHERE ' + clause
+        statement = 'SELECT COUNT(id), SUM(length) FROM unified_items WHERE ' + clause
         result = tx.query(statement, subvals)[0]
         return (result[0], result[1] or 0.0)
 
@@ -469,16 +534,23 @@ class FieldQuery(Query):
     """An abstract query that searches in a specific field for a
     pattern.
     """
-    def __init__(self, field, pattern):
+    def __init__(self, field, pattern, namespace=None, entity='item'):
         self.field = field
         self.pattern = pattern
+        self.namespace = namespace
+        self.entity = entity
 
 class MatchQuery(FieldQuery):
     """A query that looks for exact matches in an item field."""
     def clause(self):
+
         pattern = self.pattern
         if self.field == 'path' and isinstance(pattern, str):
             pattern = buffer(pattern)
+        if self.namespace:
+            #give a flexible attribute clause
+            c = 'key = ? AND value = ? AND namespace = ?'
+            return c, [self.field, pattern, self.namespace]
         return self.field + " = ?", [pattern]
 
     def match(self, item):
@@ -489,7 +561,12 @@ class SubstringQuery(FieldQuery):
     def clause(self):
         search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%')
                             .replace('_','\\_')) + '%'
-        clause = self.field + " like ? escape '\\'"
+        if self.namespace:
+            clause = """key = {0} AND namespace = {1} 
+                     AND value like ? escape '\\'""".format(
+                self.field, self.namespace)
+        else:
+            clause = self.field + " like ? escape '\\'"
         subvals = [search]
         return clause, subvals
 
@@ -499,12 +576,16 @@ class SubstringQuery(FieldQuery):
 
 class RegexpQuery(FieldQuery):
     """A query that matches a regular expression in a specific item field."""
-    def __init__(self, field, pattern):
-        super(RegexpQuery, self).__init__(field, pattern)
+    def __init__(self, field, pattern, namespace=None, entity='item'):
+        super(RegexpQuery, self).__init__(field, pattern, namespace, entity)
         self.regexp = re.compile(pattern)
 
     def clause(self):
-        clause = self.field + " REGEXP ?"
+        if self.namespace:
+            clause = 'key = {0} AND namespace = {1} AND value REGEXP ?'.format(
+                self.field, self.namespace)
+        else:
+            clause = self.field + " REGEXP ?"
         subvals = [self.pattern]
         return clause, subvals
 
@@ -553,14 +634,53 @@ class CollectionQuery(Query):
         """Returns a clause created by joining together the clauses of
         all subqueries with the string joiner (padded by spaces).
         """
+        entity = None
+        flexclause_parts = []
+        flexsubvals = []
         clause_parts = []
         subvals = []
         for subq in self.subqueries:
-            subq_clause, subq_subvals = subq.clause()
-            clause_parts.append('(' + subq_clause + ')')
-            subvals += subq_subvals
+            if hasattr(subq, 'namespace') and subq.namespace:
+                #it's a flex attr query, initiate -ugly hack- mode
+                entity = subq.entity
+                clauses = flexclause_parts
+                subs = flexsubvals
+                subq_clause, subq_subvals = subq.clause()
+                if joiner.lower() == 'and':
+                    #flexible attrs need this nested mess for AND queries
+                    subq_clause = '''
+                    EXISTS (SELECT 1 FROM {0}_attributes AS ia
+                    WHERE {1} AND ia.entity_id = unified_{0}s.id)
+                    '''.format(entity, '('+subq_clause+')')
+            else:
+                clauses = clause_parts
+                subs = subvals
+                subq_clause, subq_subvals = subq.clause()
+            clauses.append('(' + subq_clause + ')')
+            subs += subq_subvals
+
         clause = (' ' + joiner + ' ').join(clause_parts)
-        return clause, subvals
+
+        if not flexclause_parts:
+            return clause, subvals
+        
+        fclause = (' ' + joiner + ' ').join(flexclause_parts)
+        if joiner.lower() == 'and':
+            pass
+        else:
+            flexorclause = '''
+            id IN (
+            SELECT entity_id FROM {0}_attributes 
+            WHERE {1})
+            '''
+            fclause = flexorclause.format(entity, fclause)
+        if clause and fclause:
+            clause = clause+' '+joiner+' '+fclause
+        elif clause:
+            clause = clause
+        elif fclause:
+            clause = fclause
+        return clause, subvals+flexsubvals
 
     # Regular expression for _parse_query_part, below.
     _pq_regex = re.compile(
@@ -938,6 +1058,7 @@ class BaseAlbum(object):
     def __init__(self, library, record):
         super(BaseAlbum, self).__setattr__('_library', library)
         super(BaseAlbum, self).__setattr__('_record', record)
+        set_flexattrs(self, record)
 
     def __getattr__(self, key):
         """Get the value for an album attribute."""
@@ -963,6 +1084,16 @@ class BaseAlbum(object):
         else:
             super(BaseAlbum, self).__setattr__(key, value)
 
+    def __getitem__(self, key):
+        """Entry point to flexible attributes.
+        `key` should be a plugin name or an otherwise 
+        valid flexible attr namespace.
+        """
+        if key in self.flexattrs:
+            return self.flexattrs[key]
+        else:
+            raise KeyError('{} is not a valid attribute namespace'.format(key))
+
     def load(self):
         """Refresh this album's cached metadata from the library.
         """
@@ -1122,11 +1253,22 @@ class Library(BaseLibrary):
                 CREATE TABLE IF NOT EXISTS {0}_attributes (
                     id INTEGER PRIMARY KEY,
                     entity_id INTEGER,
+                    namespace TEXT,
                     key TEXT,
                     value TEXT,
-                    UNIQUE(entity_id, key) ON CONFLICT REPLACE);
+                    UNIQUE(entity_id, key, namespace) ON CONFLICT REPLACE);
                 CREATE INDEX IF NOT EXISTS {0}_id_attribute
                     ON {0}_attributes (entity_id);
+                CREATE VIEW IF NOT EXISTS unified_{0}s AS
+                SELECT 
+                GROUP_CONCAT(ia.key) AS flexkeys, 
+                GROUP_CONCAT(ia.value) AS flexvalues,
+                GROUP_CONCAT(ia.namespace) AS flexns,
+                {0}s.*
+                FROM {0}s  
+                    LEFT JOIN {0}_attributes AS ia
+                    ON {0}s.id == ia.entity_id
+                GROUP BY {0}s.id;
                 """.format(entity))
 
     def _connection(self):
@@ -1280,7 +1422,7 @@ class Library(BaseLibrary):
             load_id = item.id
 
         with self.transaction() as tx:
-            rows = tx.query('SELECT * FROM items WHERE id=?', (load_id,))
+            rows = tx.query('SELECT * FROM unified_items WHERE id=?', (load_id,))
         item._fill_record(rows[0])
         item._clear_dirty()
 
@@ -1309,15 +1451,15 @@ class Library(BaseLibrary):
                 subvars.append(store_id)
                 tx.mutate(query, subvars)
 
-            # Flexible attributes.
-            for key in plugins.item_fields():
-                if item.dirty.get(key) or store_all:
-                    tx.mutate(
-                        'INSERT INTO item_attributes (entity_id, key, value)'
-                        ' VALUES (?, ?, ?)',
-                        (store_id, key, getattr(item, key))
-                    )
-
+            #flexible attributes
+            flexins = '''INSERT INTO item_attributes 
+                      (entity_id, key, value, namespace) 
+                      VALUES (?,?,?,?);'''
+            for ns,attrs in item.flexattrs.iteritems():
+                for key in attrs.dirty:
+                    tx.mutate(flexins, (store_id, key, attrs[key], ns))
+                attrs.dirty.clear()
+        
         item._clear_dirty()
         self._memotable = {}
 
@@ -1396,7 +1538,7 @@ class Library(BaseLibrary):
             # "Add" the artist to the query.
             query = AndQuery((query, MatchQuery('albumartist', artist)))
         where, subvals = query.clause()
-        sql = "SELECT * FROM albums " + \
+        sql = "SELECT * FROM unified_albums " + \
               "WHERE " + where + \
               " ORDER BY %s, album" % \
                 _orelse("albumartist_sort", "albumartist")
@@ -1415,7 +1557,7 @@ class Library(BaseLibrary):
         super_query = AndQuery(queries)
         where, subvals = super_query.clause()
 
-        sql = "SELECT * FROM items " + \
+        sql = "SELECT * FROM unified_items " + \
               "WHERE " + where + \
               " ORDER BY %s, album, disc, track" % \
                 _orelse("artist_sort", "artist")
@@ -1431,7 +1573,7 @@ class Library(BaseLibrary):
         """Fetch an Item by its ID. Returns None if no match is found.
         """
         with self.transaction() as tx:
-            rows = tx.query("SELECT * FROM items WHERE id=?", (id,))
+            rows = tx.query("SELECT * FROM unified_items WHERE id=?", (id,))
         it = ResultIterator(rows)
         try:
             return it.next()
@@ -1552,7 +1694,7 @@ class Album(BaseAlbum):
         """
         with self._library.transaction() as tx:
             rows = tx.query(
-                'SELECT * FROM items WHERE album_id=?',
+                'SELECT * FROM unified_items WHERE album_id=?',
                 (self.id,)
             )
         return ResultIterator(rows)

beets/plugins.py

@@ -48,8 +48,6 @@ class BeetsPlugin(object):
             self.template_funcs = {}
         if not self.template_fields:
             self.template_fields = {}
-        self.item_fields = []
-        self.album_fields = []
 
     def commands(self):
         """Should return a list of beets.ui.Subcommand objects for
@@ -95,6 +93,18 @@ class BeetsPlugin(object):
         """
         return {}
 
+    def item_attributes(self):
+        """Returns a list of registered flexible attribute fields
+        for `Item` entities.
+        """
+        return []
+
+    def album_attributes(self):
+        """Returns a list of registered flexible attribute fields
+        for `Album` entities.
+        """
+        return []
+
     listeners = None
 
     @classmethod
@@ -291,24 +301,23 @@ def import_stages():
     return stages
 
 def item_fields():
-    """Get a list of strings indicating registered flexible Item
-    attributes.
+    """Get a dict of string lists indicating registered 
+    flexible Item attributes in the form {'pluginname':[fields...]}.
     """
-    fields = []
+    fields = {}
     for plugin in find_plugins():
-        fields += plugin.item_fields
+        fields[plugin.name] = plugin.item_attributes()
     return fields
 
 def album_fields():
-    """Get a list of strings indicating registered flexible Album
-    attributes.
+    """Get a dict of string lists indicating registered 
+    flexible Album attributes in the form {'pluginname':[fields...]}.
     """
-    fields = []
+    fields = {}
     for plugin in find_plugins():
-        fields += plugin.album_fields
+        fields[plugin.name] = plugin.album_attributes()
     return fields
 
-
 # Event dispatch.
 
 def event_handlers():