From c793002de7e050750c87da49a24fd8adef753c3b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 08:39:39 +0000
Subject: [PATCH 1/8] Make evaluation of composite columns just-in-time,
instead of just-in-case.
---
src/calibre/library/caches.py | 74 +++++++++++++++++++++++++----------
1 file changed, 53 insertions(+), 21 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index a32c45191f..3a61a8fd5d 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -132,6 +132,48 @@ def _match(query, value, matchkind):
pass
return False
+class CacheRow(object):
+
+ def __init__(self, db, composites, val):
+ self.db = db
+ self.composites = composites
+ self._mydata = val
+ self._must_do = len(composites) > 0
+
+ def __getitem__(self, col):
+ rec = self._mydata
+ if self._must_do and col in self.composites:
+ self._must_do = False
+ mi = self.db.get_metadata(rec[0], index_is_id=True)
+ for c in self.composites:
+ rec[c] = mi.get(self.composites[c])
+ return rec[col]
+
+ def __setitem__ (self, col, val):
+ self._mydata[col] = val
+
+ def append(self, val):
+ self._mydata.append(val)
+
+ def get(self, col, default):
+ try:
+ return self.__getitem__(col)
+ except:
+ return default
+
+ def __len__(self):
+ return len(self._mydata)
+
+ def __iter__(self):
+ for v in self._mydata:
+ yield v
+
+ def __str__(self):
+ return self.__unicode__()
+
+ def __unicode__(self):
+ return unicode(self._mydata)
+
class ResultCache(SearchQueryParser): # {{{
'''
@@ -139,7 +181,12 @@ class ResultCache(SearchQueryParser): # {{{
'''
def __init__(self, FIELD_MAP, field_metadata):
self.FIELD_MAP = FIELD_MAP
- self._map = self._data = self._map_filtered = []
+ self.composites = {}
+ for key in field_metadata:
+ if field_metadata[key]['datatype'] == 'composite':
+ self.composites[field_metadata[key]['rec_index']] = key
+ self._data = []
+ self._map = self._map_filtered = []
self.first_sort = True
self.search_restriction = ''
self.field_metadata = field_metadata
@@ -148,10 +195,6 @@ def __init__(self, FIELD_MAP, field_metadata):
self.build_date_relop_dict()
self.build_numeric_relop_dict()
- self.composites = []
- for key in field_metadata:
- if field_metadata[key]['datatype'] == 'composite':
- self.composites.append((key, field_metadata[key]['rec_index']))
def __getitem__(self, row):
return self._data[self._map_filtered[row]]
@@ -583,13 +626,10 @@ def refresh_ids(self, db, ids):
'''
for id in ids:
try:
- self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
+ self._data[id] = CacheRow(db, self.composites,
+ db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
self._data[id].append(db.book_on_device_string(id))
self._data[id].append(None)
- if len(self.composites) > 0:
- mi = db.get_metadata(id, index_is_id=True)
- for k,c in self.composites:
- self._data[id][c] = mi.get(k, None)
except IndexError:
return None
try:
@@ -603,13 +643,10 @@ def books_added(self, ids, db):
return
self._data.extend(repeat(None, max(ids)-len(self._data)+2))
for id in ids:
- self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
+ self._data[id] = CacheRow(db, self.composites,
+ db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
self._data[id].append(db.book_on_device_string(id))
self._data[id].append(None)
- if len(self.composites) > 0:
- mi = db.get_metadata(id, index_is_id=True)
- for k,c in self.composites:
- self._data[id][c] = mi.get(k)
self._map[0:0] = ids
self._map_filtered[0:0] = ids
@@ -630,16 +667,11 @@ def refresh(self, db, field=None, ascending=True):
temp = db.conn.get('SELECT * FROM meta2')
self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else []
for r in temp:
- self._data[r[0]] = r
+ self._data[r[0]] = CacheRow(db, self.composites, r)
for item in self._data:
if item is not None:
item.append(db.book_on_device_string(item[0]))
item.append(None)
- if len(self.composites) > 0:
- mi = db.get_metadata(item[0], index_is_id=True)
- for k,c in self.composites:
- item[c] = mi.get(k)
-
self._map = [i[0] for i in self._data if i is not None]
if field is not None:
self.sort(field, ascending)
From 2662ab485fb978a27dcdc613b68e45613e8c6ac6 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 09:49:27 +0000
Subject: [PATCH 2/8] Make tags sorted in the meta2 table.
---
src/calibre/gui2/library/models.py | 4 ++--
src/calibre/library/custom_columns.py | 9 +++++----
src/calibre/library/database2.py | 2 +-
src/calibre/library/sqlite.py | 23 ++++++++++++++++++++++-
4 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 49cb1ce182..6a48aef9be 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -526,7 +526,7 @@ def authors(r, idx=-1):
def tags(r, idx=-1):
tags = self.db.data[r][idx]
if tags:
- return QVariant(', '.join(sorted(tags.split(','), key=sort_key)))
+ return QVariant(', '.join(tags.split(',')))
return None
def series_type(r, idx=-1, siix=-1):
@@ -577,7 +577,7 @@ def ondevice_decorator(r, idx=-1):
def text_type(r, mult=False, idx=-1):
text = self.db.data[r][idx]
if text and mult:
- return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
+ return QVariant(', '.join(text.split('|')))
return QVariant(text)
def number_type(r, idx=-1):
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 07ea407460..558f3b8fc9 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -14,6 +14,7 @@
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date
from calibre.utils.config import tweaks
+from calibre.utils.icu import sort_key
class CustomColumns(object):
@@ -181,8 +182,8 @@ def get_custom(self, idx, label=None, num=None, index_is_id=False):
ans = row[self.FIELD_MAP[data['num']]]
if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split('|') if ans else []
- if data['display'].get('sort_alpha', False):
- ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
+ if data['display'].get('sort_alpha', True):
+ ans.sort(key=sort_key)
return ans
def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@@ -534,8 +535,8 @@ def custom_columns_in_meta(self):
if data['normalized']:
query = '%s.value'
if data['is_multiple']:
- query = 'group_concat(%s.value, "|")'
- if not display.get('sort_alpha', False):
+ query = 'cc_sortconcat(%s.value)'
+ if not display.get('sort_alpha', True):
query = 'sort_concat(link.id, %s.value)'
line = '''(SELECT {query} FROM {lt} AS link INNER JOIN
{table} ON(link.value={table}.id) WHERE link.book=books.id)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 611aa1cc89..0b1c6a6cfb 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -242,7 +242,7 @@ def migrate_preference(key, default):
'timestamp',
'(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size',
('rating', 'ratings', 'rating', 'ratings.rating'),
- ('tags', 'tags', 'tag', 'group_concat(name)'),
+ ('tags', 'tags', 'tag', 'tags_sortconcat(name)'),
'(SELECT text FROM comments WHERE book=books.id) comments',
('series', 'series', 'series', 'name'),
('publisher', 'publishers', 'publisher', 'name'),
diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py
index 0458ada27b..0c3ae487ea 100644
--- a/src/calibre/library/sqlite.py
+++ b/src/calibre/library/sqlite.py
@@ -19,7 +19,7 @@
from calibre.utils.date import parse_date, isoformat
from calibre import isbytestring, force_unicode
from calibre.constants import iswindows, DEBUG
-from calibre.utils.icu import strcmp
+from calibre.utils.icu import strcmp, sort_key
global_lock = RLock()
@@ -69,6 +69,25 @@ def finalize(self):
return None
return self.sep.join(self.ans)
+class TagsSortConcatenate(object):
+ '''Sorted string concatenation aggregator for sqlite'''
+ def __init__(self, sep=','):
+ self.sep = sep
+ self.ans = []
+
+ def step(self, value):
+ if value is not None:
+ self.ans.append(value)
+
+ def finalize(self):
+ if not self.ans:
+ return None
+ return self.sep.join(sorted(self.ans, key=sort_key))
+
+class CcSortConcatenate(TagsSortConcatenate):
+ def __init__(self):
+ TagsSortConcatenate.__init__(self, sep='|')
+
class SortedConcatenate(object):
'''String concatenation aggregator for sqlite, sorted by supplied index'''
sep = ','
@@ -155,6 +174,8 @@ def connect(self):
c_ext_loaded = load_c_extensions(self.conn)
self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row)
self.conn.create_aggregate('concat', 1, Concatenate)
+ self.conn.create_aggregate('tags_sortconcat', 1, TagsSortConcatenate)
+ self.conn.create_aggregate('cc_sortconcat', 1, CcSortConcatenate)
if not c_ext_loaded:
self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
From 84c6bcac39d515bda5e3344ad577d0ce214576d8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 12:59:18 +0000
Subject: [PATCH 3/8] Add ability to manipulate int, float, and bool columns in
search replace
---
src/calibre/gui2/dialogs/metadata_bulk.py | 5 ++++-
src/calibre/library/custom_columns.py | 22 +++++++++++++++++++---
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index ef14c95b1d..e1ee4327f3 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -321,7 +321,8 @@ def prepare_search_and_replace(self):
if (f in ['author_sort'] or
(fm[f]['datatype'] in ['text', 'series', 'enumeration']
and fm[f].get('search_terms', None)
- and f not in ['formats', 'ondevice', 'sort'])):
+ and f not in ['formats', 'ondevice', 'sort']) or
+ fm[f]['datatype'] in ['int', 'float', 'bool'] ):
self.all_fields.append(f)
self.writable_fields.append(f)
if f in ['sort'] or fm[f]['datatype'] == 'composite':
@@ -431,6 +432,8 @@ def s_r_get_field(self, mi, field):
val = mi.get('title_sort', None)
else:
val = mi.get(field, None)
+ if isinstance(val, (int, float, bool)):
+ val = str(val)
if val is None:
val = [] if fm['is_multiple'] else ['']
elif not fm['is_multiple']:
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 558f3b8fc9..ccdd55021d 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -134,7 +134,15 @@ def adapt_datetime(x, d):
def adapt_bool(x, d):
if isinstance(x, (str, unicode, bytes)):
- x = bool(int(x))
+ x = x.lower()
+ if x == 'true':
+ x = True
+ elif x == 'false':
+ x = False
+ elif x == 'none':
+ x = None
+ else:
+ x = bool(int(x))
return x
def adapt_enum(x, d):
@@ -143,9 +151,17 @@ def adapt_enum(x, d):
v = None
return v
+ def adapt_number(x, d):
+ if isinstance(x, (str, unicode, bytes)):
+ if x.lower() == 'none':
+ return None
+ if d['datatype'] == 'int':
+ return int(x)
+ return float(x)
+
self.custom_data_adapters = {
- 'float': lambda x,d : x if x is None else float(x),
- 'int': lambda x,d : x if x is None else int(x),
+ 'float': adapt_number,
+ 'int': adapt_number,
'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
'bool': adapt_bool,
'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
From 3f3944e95b6a4fd7d830d98187b78a45298796e8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 14:51:14 +0000
Subject: [PATCH 4/8] Enhancement #8035: Advanced Search,
Titel/Author?series/Tag - Type Ahead Word Lists
---
src/calibre/gui2/dialogs/search.py | 44 +++++++++++++++++++++++++-----
src/calibre/gui2/dialogs/search.ui | 27 ++++++++++++++----
2 files changed, 59 insertions(+), 12 deletions(-)
diff --git a/src/calibre/gui2/dialogs/search.py b/src/calibre/gui2/dialogs/search.py
index 8e8fd09652..62a0f8a9f1 100644
--- a/src/calibre/gui2/dialogs/search.py
+++ b/src/calibre/gui2/dialogs/search.py
@@ -3,7 +3,7 @@
import re, copy
-from PyQt4.QtGui import QDialog, QDialogButtonBox
+from PyQt4.Qt import QDialog, QDialogButtonBox, QCompleter, Qt
from calibre.gui2.dialogs.search_ui import Ui_Dialog
from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
@@ -22,6 +22,28 @@ def __init__(self, parent, db):
key=lambda x: sort_key(x if x[0] != '#' else x[1:]))
self.general_combo.addItems(searchables)
+ all_authors = db.all_authors()
+ all_authors.sort(key=lambda x : sort_key(x[1]))
+ for i in all_authors:
+ id, name = i
+ name = name.strip().replace('|', ',')
+ self.authors_box.addItem(name)
+ self.authors_box.setEditText('')
+ self.authors_box.completer().setCompletionMode(QCompleter.PopupCompletion)
+ self.authors_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+
+ all_series = db.all_series()
+ all_series.sort(key=lambda x : sort_key(x[1]))
+ for i in all_series:
+ id, name = i
+ self.series_box.addItem(name)
+ self.series_box.setEditText('')
+ self.series_box.completer().setCompletionMode(QCompleter.PopupCompletion)
+ self.series_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+
+ all_tags = db.all_tags()
+ self.tags_box.update_tags_cache(all_tags)
+
self.box_last_values = copy.deepcopy(box_values)
if self.box_last_values:
for k,v in self.box_last_values.items():
@@ -121,26 +143,34 @@ def token(self):
return tok
def box_search_string(self):
+ mk = self.matchkind.currentIndex()
+ if mk == CONTAINS_MATCH:
+ self.mc = ''
+ elif mk == EQUALS_MATCH:
+ self.mc = '='
+ else:
+ self.mc = '~'
+
ans = []
self.box_last_values = {}
title = unicode(self.title_box.text()).strip()
self.box_last_values['title_box'] = title
if title:
- ans.append('title:"' + title + '"')
+ ans.append('title:"' + self.mc + title + '"')
author = unicode(self.authors_box.text()).strip()
self.box_last_values['authors_box'] = author
if author:
- ans.append('author:"' + author + '"')
+ ans.append('author:"' + self.mc + author + '"')
series = unicode(self.series_box.text()).strip()
self.box_last_values['series_box'] = series
if series:
- ans.append('series:"' + series + '"')
- self.mc = '='
+ ans.append('series:"' + self.mc + series + '"')
+
tags = unicode(self.tags_box.text())
self.box_last_values['tags_box'] = tags
- tags = self.tokens(tags)
+ tags = [t.strip() for t in tags.split(',') if t.strip()]
if tags:
- tags = ['tags:' + t for t in tags]
+ tags = ['tags:"=' + t + '"' for t in tags]
ans.append('(' + ' or '.join(tags) + ')')
general = unicode(self.general_box.text())
self.box_last_values['general_box'] = general
diff --git a/src/calibre/gui2/dialogs/search.ui b/src/calibre/gui2/dialogs/search.ui
index 7bb4c15363..6848a45506 100644
--- a/src/calibre/gui2/dialogs/search.ui
+++ b/src/calibre/gui2/dialogs/search.ui
@@ -21,7 +21,7 @@
-
- What kind of match to use:
+ &What kind of match to use:
matchkind
@@ -228,7 +228,7 @@
-
-
+
Enter the title.
@@ -265,21 +265,21 @@
-
-
+
Enter an author's name. Only one author can be used.
-
-
+
Enter a series name, without an index. Only one series name can be used.
-
-
+
Enter tags separated by spaces
@@ -348,6 +348,23 @@
+
+
+ EnLineEdit
+ QLineEdit
+
+
+
+ EnComboBox
+ QComboBox
+
+
+
+ TagsLineEdit
+ QLineEdit
+
+
+
all
phrase
From 7e7f0954ce61c319b1223e8fe1baee53dfbb58ff Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 18:51:35 +0000
Subject: [PATCH 5/8] New implementation of CacheRow
---
src/calibre/library/caches.py | 54 ++++++++++++++---------------------
1 file changed, 21 insertions(+), 33 deletions(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 3a61a8fd5d..ada1ee0a77 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -132,47 +132,35 @@ def _match(query, value, matchkind):
pass
return False
-class CacheRow(object):
+class CacheRow(list):
def __init__(self, db, composites, val):
self.db = db
- self.composites = composites
- self._mydata = val
+ self._composites = composites
+ list.__init__(self, val)
self._must_do = len(composites) > 0
def __getitem__(self, col):
- rec = self._mydata
- if self._must_do and col in self.composites:
- self._must_do = False
- mi = self.db.get_metadata(rec[0], index_is_id=True)
- for c in self.composites:
- rec[c] = mi.get(self.composites[c])
- return rec[col]
+ if self._must_do:
+ is_comp = False
+ if isinstance(col, slice):
+ for c in range(col.start, col.stop):
+ if c in self._composites:
+ is_comp = True
+ break
+ elif col in self._composites:
+ is_comp = True
+ if is_comp:
+ id = list.__getitem__(self, 0)
+ self._must_do = False
+ mi = self.db.get_metadata(id, index_is_id=True)
+ for c in self._composites:
+ self[c] = mi.get(self._composites[c])
+ return list.__getitem__(self, col)
- def __setitem__ (self, col, val):
- self._mydata[col] = val
+ def __getslice__(self, i, j):
+ return self.__getitem__(slice(i, j))
- def append(self, val):
- self._mydata.append(val)
-
- def get(self, col, default):
- try:
- return self.__getitem__(col)
- except:
- return default
-
- def __len__(self):
- return len(self._mydata)
-
- def __iter__(self):
- for v in self._mydata:
- yield v
-
- def __str__(self):
- return self.__unicode__()
-
- def __unicode__(self):
- return unicode(self._mydata)
class ResultCache(SearchQueryParser): # {{{
From cbd880e8f736fe3ee20c3fea62a1895a8848194f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 19:17:01 +0000
Subject: [PATCH 6/8] Take out sorting change
---
src/calibre/gui2/library/models.py | 4 ++--
src/calibre/library/custom_columns.py | 31 ++++++---------------------
src/calibre/library/database2.py | 2 +-
src/calibre/library/sqlite.py | 23 +-------------------
4 files changed, 11 insertions(+), 49 deletions(-)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 6a48aef9be..49cb1ce182 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -526,7 +526,7 @@ def authors(r, idx=-1):
def tags(r, idx=-1):
tags = self.db.data[r][idx]
if tags:
- return QVariant(', '.join(tags.split(',')))
+ return QVariant(', '.join(sorted(tags.split(','), key=sort_key)))
return None
def series_type(r, idx=-1, siix=-1):
@@ -577,7 +577,7 @@ def ondevice_decorator(r, idx=-1):
def text_type(r, mult=False, idx=-1):
text = self.db.data[r][idx]
if text and mult:
- return QVariant(', '.join(text.split('|')))
+ return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
return QVariant(text)
def number_type(r, idx=-1):
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index ccdd55021d..07ea407460 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -14,7 +14,6 @@
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date
from calibre.utils.config import tweaks
-from calibre.utils.icu import sort_key
class CustomColumns(object):
@@ -134,15 +133,7 @@ def adapt_datetime(x, d):
def adapt_bool(x, d):
if isinstance(x, (str, unicode, bytes)):
- x = x.lower()
- if x == 'true':
- x = True
- elif x == 'false':
- x = False
- elif x == 'none':
- x = None
- else:
- x = bool(int(x))
+ x = bool(int(x))
return x
def adapt_enum(x, d):
@@ -151,17 +142,9 @@ def adapt_enum(x, d):
v = None
return v
- def adapt_number(x, d):
- if isinstance(x, (str, unicode, bytes)):
- if x.lower() == 'none':
- return None
- if d['datatype'] == 'int':
- return int(x)
- return float(x)
-
self.custom_data_adapters = {
- 'float': adapt_number,
- 'int': adapt_number,
+ 'float': lambda x,d : x if x is None else float(x),
+ 'int': lambda x,d : x if x is None else int(x),
'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
'bool': adapt_bool,
'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
@@ -198,8 +181,8 @@ def get_custom(self, idx, label=None, num=None, index_is_id=False):
ans = row[self.FIELD_MAP[data['num']]]
if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split('|') if ans else []
- if data['display'].get('sort_alpha', True):
- ans.sort(key=sort_key)
+ if data['display'].get('sort_alpha', False):
+ ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
return ans
def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@@ -551,8 +534,8 @@ def custom_columns_in_meta(self):
if data['normalized']:
query = '%s.value'
if data['is_multiple']:
- query = 'cc_sortconcat(%s.value)'
- if not display.get('sort_alpha', True):
+ query = 'group_concat(%s.value, "|")'
+ if not display.get('sort_alpha', False):
query = 'sort_concat(link.id, %s.value)'
line = '''(SELECT {query} FROM {lt} AS link INNER JOIN
{table} ON(link.value={table}.id) WHERE link.book=books.id)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 0b1c6a6cfb..611aa1cc89 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -242,7 +242,7 @@ def migrate_preference(key, default):
'timestamp',
'(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size',
('rating', 'ratings', 'rating', 'ratings.rating'),
- ('tags', 'tags', 'tag', 'tags_sortconcat(name)'),
+ ('tags', 'tags', 'tag', 'group_concat(name)'),
'(SELECT text FROM comments WHERE book=books.id) comments',
('series', 'series', 'series', 'name'),
('publisher', 'publishers', 'publisher', 'name'),
diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py
index 0c3ae487ea..0458ada27b 100644
--- a/src/calibre/library/sqlite.py
+++ b/src/calibre/library/sqlite.py
@@ -19,7 +19,7 @@
from calibre.utils.date import parse_date, isoformat
from calibre import isbytestring, force_unicode
from calibre.constants import iswindows, DEBUG
-from calibre.utils.icu import strcmp, sort_key
+from calibre.utils.icu import strcmp
global_lock = RLock()
@@ -69,25 +69,6 @@ def finalize(self):
return None
return self.sep.join(self.ans)
-class TagsSortConcatenate(object):
- '''Sorted string concatenation aggregator for sqlite'''
- def __init__(self, sep=','):
- self.sep = sep
- self.ans = []
-
- def step(self, value):
- if value is not None:
- self.ans.append(value)
-
- def finalize(self):
- if not self.ans:
- return None
- return self.sep.join(sorted(self.ans, key=sort_key))
-
-class CcSortConcatenate(TagsSortConcatenate):
- def __init__(self):
- TagsSortConcatenate.__init__(self, sep='|')
-
class SortedConcatenate(object):
'''String concatenation aggregator for sqlite, sorted by supplied index'''
sep = ','
@@ -174,8 +155,6 @@ def connect(self):
c_ext_loaded = load_c_extensions(self.conn)
self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row)
self.conn.create_aggregate('concat', 1, Concatenate)
- self.conn.create_aggregate('tags_sortconcat', 1, TagsSortConcatenate)
- self.conn.create_aggregate('cc_sortconcat', 1, CcSortConcatenate)
if not c_ext_loaded:
self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)
From 9ee7dc27e85f1d6e948210818a4b68082bf5792d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 19:21:17 +0000
Subject: [PATCH 7/8] Do slice stepping correctly
---
src/calibre/library/caches.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index ada1ee0a77..2596b494bf 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -144,7 +144,9 @@ def __getitem__(self, col):
if self._must_do:
is_comp = False
if isinstance(col, slice):
- for c in range(col.start, col.stop):
+ start = 0 if col.start is None else col.start
+ step = 1 if col.stop is None else col.stop
+ for c in range(start, col.stop, step):
if c in self._composites:
is_comp = True
break
From 5adaf263e4c29cc3810a198f44777d90b5999ad5 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 19:52:55 +0000
Subject: [PATCH 8/8] Ensure that multiples are sorted in alpha order before
column sort and during template processing
---
src/calibre/ebooks/metadata/book/base.py | 7 ++++---
src/calibre/library/caches.py | 15 ++++++---------
2 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 77df6b00c2..17f2c6705c 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -16,6 +16,7 @@
from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import isoformat, format_date
+from calibre.utils.icu import sort_key
from calibre.utils.formatter import TemplateFormatter
@@ -490,7 +491,7 @@ def format_authors(self):
return authors_to_string(self.authors)
def format_tags(self):
- return u', '.join([unicode(t) for t in self.tags])
+ return u', '.join([unicode(t) for t in sorted(self.tags, key=sort_key)])
def format_rating(self):
return unicode(self.rating)
@@ -530,7 +531,7 @@ def format_field_extended(self, key, series_with_index=True):
orig_res = res
datatype = cmeta['datatype']
if datatype == 'text' and cmeta['is_multiple']:
- res = u', '.join(res)
+ res = u', '.join(sorted(res, key=sort_key))
elif datatype == 'series' and series_with_index:
if self.get_extra(key) is not None:
res = res + \
@@ -560,7 +561,7 @@ def format_field_extended(self, key, series_with_index=True):
elif key == 'series_index':
res = self.format_series_index(res)
elif datatype == 'text' and fmeta['is_multiple']:
- res = u', '.join(res)
+ res = u', '.join(sorted(res, key=sort_key))
elif datatype == 'series' and series_with_index:
res = res + ' [%s]'%self.format_series_index()
elif datatype == 'datetime':
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 2596b494bf..980c9f1fa9 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -691,13 +691,7 @@ def multisort(self, fields=[], subsort=False):
fields = [('timestamp', False)]
keyg = SortKeyGenerator(fields, self.field_metadata, self._data)
- # For efficiency, the key generator returns a plain value if only one
- # field is in the sort field list. Because the normal cmp function will
- # always assume asc, we must deal with asc/desc here.
- if len(fields) == 1:
- self._map.sort(key=keyg, reverse=not fields[0][1])
- else:
- self._map.sort(key=keyg)
+ self._map.sort(key=keyg)
tmap = list(itertools.repeat(False, len(self._data)))
for x in self._map_filtered:
@@ -730,8 +724,6 @@ def __init__(self, fields, field_metadata, data):
def __call__(self, record):
values = tuple(self.itervals(self.data[record]))
- if len(values) == 1:
- return values[0]
return SortKey(self.orders, values)
def itervals(self, record):
@@ -754,6 +746,11 @@ def itervals(self, record):
val = (self.string_sort_key(val), sidx)
elif dt in ('text', 'comments', 'composite', 'enumeration'):
+ if val:
+ sep = fm['is_multiple']
+ if sep:
+ val = sep.join(sorted(val.split(sep),
+ key=self.string_sort_key))
val = self.string_sort_key(val)
elif dt == 'bool':