From 07c912166cef60acff36c3a24bca3bffa1b9bd8e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 17 Sep 2010 14:20:37 +0100 Subject: [PATCH 1/4] Add search & replace to bulk edit --- src/calibre/gui2/dialogs/metadata_bulk.py | 221 +++++++++++++++++++++- src/calibre/gui2/dialogs/metadata_bulk.ui | 112 +++++++++++ src/calibre/library/database2.py | 15 +- 3 files changed, 336 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index de483720fc..50a49be532 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -4,8 +4,10 @@ '''Dialog to edit metadata in bulk''' from threading import Thread +import os, re, shutil -from PyQt4.Qt import QDialog, QGridLayout +from PyQt4.Qt import SIGNAL, QDialog, QGridLayout +from PyQt4 import QtGui from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor @@ -83,7 +85,6 @@ def doit(self): w.commit(self.ids) self.db.bulk_modify_tags(self.ids, add=add, remove=remove, notify=False) - self.db.clean() def run(self): try: @@ -127,12 +128,211 @@ def __init__(self, window, rows, db): self.series.currentIndexChanged[int].connect(self.series_changed) self.series.editTextChanged.connect(self.series_changed) self.tag_editor_button.clicked.connect(self.tag_editor) - if len(db.custom_column_label_map) == 0: - self.central_widget.tabBar().setVisible(False) - else: - self.create_custom_column_editors() + +# Haven't yet figured out how to hide a single tab +# if len(db.custom_column_label_map) == 0: +# self.central_widget.widget(1).setVisible(False) +# else: +# self.create_custom_column_editors() + self.create_custom_column_editors() + + self.prepare_search_and_replace() self.exec_() + def prepare_search_and_replace(self): + self.search_for.initialize('bulk_edit_search_for') + self.replace_with.initialize('bulk_edit_replace_with') + self.test_text.initialize('bulk_edit_test_test') + fields = [''] + fm = self.db.field_metadata + for f in fm: + if (f in ['author_sort'] or ( + fm[f]['datatype'] == 'text' or fm[f]['datatype'] == 'series') + and fm[f].get('search_terms', None) + and f not in ['formats', 'ondevice']): + fields.append(f) + fields.sort() + self.search_field.addItems(fields) + offset = 10 + self.s_r_number_of_books = min(7, len(self.ids)) + for i in range(1,self.s_r_number_of_books+1): + w = QtGui.QLabel(self.tabWidgetPage3) + w.setText(_('Book %d:'%i)) + self.gridLayout1.addWidget(w, i+offset, 0, 1, 1) + w = QtGui.QLineEdit(self.tabWidgetPage3) + w.setReadOnly(True) + name = 'book_%d_text'%i + setattr(self, name, w) + self.book_1_text.setObjectName(name) + self.gridLayout1.addWidget(w, i+offset, 1, 1, 1) + w = QtGui.QLineEdit(self.tabWidgetPage3) + w.setReadOnly(True) + name = 'book_%d_result'%i + setattr(self, name, w) + self.book_1_text.setObjectName(name) + self.gridLayout1.addWidget(w, i+offset, 2, 1, 1) + + self.s_r_heading.setText( + _('Search and replace in text fields using ' + 'regular expressions. The search text is an ' + 'arbitrary python-compatible regular expression. ' + 'The replacement text can contain backreferences ' + 'to parenthesized expressions in the pattern. ' + 'The search is not anchored, and can match and ' + 'replace times on the same string. See ' + ' ' + 'http://docs.python.org/library/re.html ' + 'for more information, and in particular the \'sub\' ' + 'function.
' + 'Note: you can destroy your library ' + 'using this feature. Changes are permanent. There ' + 'is no undo function. You are strongly encouraged ' + 'to backup the metadata.db file in your library ' + 'before proceeding.')) + self.s_r_error = None + self.s_r_obj = None + + self.replace_func.addItems(['', 'upper', 'lower', 'title']) + self.connect(self.search_field, + SIGNAL('currentIndexChanged(const QString &)'), + self.s_r_field_changed) + self.connect(self.replace_func, + SIGNAL('currentIndexChanged(const QString &)'), + self.s_r_paint_results) + self.connect(self.search_for, + SIGNAL('editTextChanged(const QString &)'), + self.s_r_paint_results) + self.connect(self.replace_with, + SIGNAL('editTextChanged(const QString &)'), + self.s_r_paint_results) + self.connect(self.test_text, + SIGNAL('editTextChanged(const QString &)'), + self.s_r_paint_results) + + def s_r_field_changed(self, txt): + txt = unicode(txt) + for i in range(0,self.s_r_number_of_books): + if txt: + fm = self.db.field_metadata[txt] + id = self.ids[i] + val = self.db.get_property(id, index_is_id=True, + loc=fm['rec_index']) + if val is None: + val = '' + if fm['is_multiple']: + val = [t.strip() for t in val.split(fm['is_multiple']) if t.strip()] + if val: + val.sort(cmp=lambda x,y: cmp(x.lower(), y.lower())) + val = val[0] + else: + val = '' + else: + val = '' + w = getattr(self, 'book_%d_text'%(i+1)) + w.setText(val) + self.s_r_paint_results(None) + + def s_r_set_colors(self): + if self.s_r_error is not None: + col = 'rgb(255, 0, 0, 20%)' + self.test_result.setText(self.s_r_error.message) + else: + col = 'rgb(0, 255, 0, 20%)' + self.test_result.setStyleSheet('QLineEdit { color: black; ' + 'background-color: %s; }'%col) + for i in range(0,self.s_r_number_of_books): + getattr(self, 'book_%d_result'%(i+1)).setText('') + + def s_r_func(self, match): + rf = unicode(self.replace_func.currentText()) + rv = unicode(self.replace_with.text()) + val = match.expand(rv) + if rf == 'upper': + return val.upper() + if rf == 'lower': + return val.lower() + if rf == 'title': + return val.title() + return val + + def s_r_paint_results(self, txt): + self.s_r_error = None + self.s_r_set_colors() + try: + self.s_r_obj = re.compile(unicode(self.search_for.text())) + except re.error as e: + self.s_r_obj = None + self.s_r_error = e + self.s_r_set_colors() + return + + try: + self.test_result.setText(self.s_r_obj.sub(self.s_r_func, + unicode(self.test_text.text()))) + except re.error as e: + self.s_r_error = e + self.s_r_set_colors() + return + + for i in range(0,self.s_r_number_of_books): + wt = getattr(self, 'book_%d_text'%(i+1)) + wr = getattr(self, 'book_%d_result'%(i+1)) + try: + wr.setText(self.s_r_obj.sub(self.s_r_func, unicode(wt.text()))) + except re.error as e: + self.s_r_error = e + self.s_r_set_colors() + break + + def do_search_replace(self): + field = unicode(self.search_field.currentText()) + if not field or not self.s_r_obj: + return + if self.s_r_backup_db.isChecked(): + self.db.commit() + src = self.db.dbpath + dest = self.db.dbpath+'.backup' + if os.path.exists(dest): + os.remove(dest) + shutil.copyfile(src, dest) + + fm = self.db.field_metadata[field] + + def apply_pattern(val): + try: + return self.s_r_obj.sub(self.s_r_func, val) + except: + return val + + for id in self.ids: + val = self.db.get_property(id, index_is_id=True, + loc=fm['rec_index']) + if val is None: + continue + if fm['is_multiple']: + res = [] + for val in [t.strip() for t in val.split(fm['is_multiple'])]: + v = apply_pattern(val).strip() + if v: + res.append(v) + val = fm['is_multiple'].join(res) + else: + val = apply_pattern(val) + + if fm['is_custom']: + extra = self.db.get_custom_extra(id, label=fm['label'], index_is_id=True) + self.db.set_custom(id, val, label=fm['label'], extra=extra, + commit=False) + else: + if field == 'comments': + setter = self.db.set_comment + else: + setter = getattr(self.db, 'set_'+field) + if field == 'authors': + val = string_to_authors(val) + setter(id, val, notify=False, commit=False) + self.db.commit() + def create_custom_column_editors(self): w = self.central_widget.widget(1) layout = QGridLayout() @@ -193,6 +393,11 @@ def accept(self): if len(self.ids) < 1: return QDialog.accept(self) + if self.s_r_error is not None: + error_dialog(self, _('Search/replace invalid'), + _('Search pattern is invalid: %s')%self.s_r_error.message, + show=True) + return False self.changed = bool(self.ids) # Cache values from GUI so that Qt widgets are not used in # non GUI thread @@ -234,6 +439,10 @@ def accept(self): return error_dialog(self, _('Failed'), self.worker.error[0], det_msg=self.worker.error[1], show=True) + + self.do_search_replace() + + self.db.clean() return QDialog.accept(self) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 0eeee61c7e..f51b93cafa 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -301,6 +301,113 @@ Future conversion of these books will use the default settings. &Custom metadata + + + &Search and replace + + + + QLayout::SetMinimumSize + + + + + true + + + + + + + Backup 'metadata.db' to 'metadata.db.backup' before applying changes + + + true + + + + + + + Search field: + + + + + + + Search for: + + + + + + + Replace with: + + + + + + + + + + + + + + + + Apply function after replace: + + + + + + + + + + Test text + + + + + + + Test result + + + + + + + Your test: + + + + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + @@ -333,6 +440,11 @@ Future conversion of these books will use the default settings. QLineEdit
widgets.h
+ + HistoryLineEdit + QLineEdit +
widgets.h
+ authors diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f5f0f724ba..c2d727e3c2 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -336,6 +336,7 @@ def get_property(idx, index_is_id=False, loc=-1): loc=self.FIELD_MAP['comments' if prop == 'comment' else prop])) setattr(self, 'title_sort', functools.partial(get_property, loc=self.FIELD_MAP['sort'])) + setattr(self, 'get_property', get_property) def initialize_database(self): metadata_sqlite = open(P('metadata_sqlite.sql'), 'rb').read() @@ -1128,7 +1129,7 @@ def author_sort_from_authors(self, authors): result.append(r) return ' & '.join(result).replace('|', ',') - def set_authors(self, id, authors, notify=True): + def set_authors(self, id, authors, notify=True, commit=True): ''' `authors`: A list of authors. ''' @@ -1156,16 +1157,17 @@ def set_authors(self, id, authors, notify=True): ss = self.author_sort_from_book(id, index_is_id=True) self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (ss, id)) - self.conn.commit() + if commit: + self.conn.commit() self.data.set(id, self.FIELD_MAP['authors'], ','.join([a.replace(',', '|') for a in authors]), row_is_id=True) self.data.set(id, self.FIELD_MAP['author_sort'], ss, row_is_id=True) - self.set_path(id, True) + self.set_path(id, index_is_id=True, commit=commit) if notify: self.notify('metadata', [id]) - def set_title(self, id, title, notify=True): + def set_title(self, id, title, notify=True, commit=True): if not title: return if not isinstance(title, unicode): @@ -1176,8 +1178,9 @@ def set_title(self, id, title, notify=True): self.data.set(id, self.FIELD_MAP['sort'], title_sort(title), row_is_id=True) else: self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True) - self.set_path(id, True) - self.conn.commit() + self.set_path(id, index_is_id=True, commit=commit) + if commit: + self.conn.commit() if notify: self.notify('metadata', [id]) From 4b4ed2cd597aee57cf4559b1ffaeff1c88afb4f1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 17 Sep 2010 14:42:29 +0100 Subject: [PATCH 2/4] Improved sr function setup --- src/calibre/gui2/dialogs/metadata_bulk.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 50a49be532..0132f958b0 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -102,6 +102,13 @@ def run(self): class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): + s_r_functions = { + '' : lambda x: x, + 'lower' : lambda x: x.lower(), + 'upper' : lambda x: x.upper(), + 'title' : lambda x: x.title(), + } + def __init__(self, window, rows, db): QDialog.__init__(self, window) Ui_MetadataBulkDialog.__init__(self) @@ -192,7 +199,7 @@ def prepare_search_and_replace(self): self.s_r_error = None self.s_r_obj = None - self.replace_func.addItems(['', 'upper', 'lower', 'title']) + self.replace_func.addItems(sorted(self.s_r_functions.keys())) self.connect(self.search_field, SIGNAL('currentIndexChanged(const QString &)'), self.s_r_field_changed) @@ -244,16 +251,10 @@ def s_r_set_colors(self): getattr(self, 'book_%d_result'%(i+1)).setText('') def s_r_func(self, match): - rf = unicode(self.replace_func.currentText()) + rf = self.s_r_functions[unicode(self.replace_func.currentText())] rv = unicode(self.replace_with.text()) val = match.expand(rv) - if rf == 'upper': - return val.upper() - if rf == 'lower': - return val.lower() - if rf == 'title': - return val.title() - return val + return rf(val) def s_r_paint_results(self, txt): self.s_r_error = None From 5e2296e59ef63d8b645b1a4b5534bfe0fc7cd672 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 17 Sep 2010 14:59:05 +0100 Subject: [PATCH 3/4] Fix size of fields pulldown --- src/calibre/gui2/dialogs/metadata_bulk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 0132f958b0..6bda4db138 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -160,6 +160,7 @@ def prepare_search_and_replace(self): fields.append(f) fields.sort() self.search_field.addItems(fields) + self.search_field.setMaxVisibleItems(min(len(fields), 20)) offset = 10 self.s_r_number_of_books = min(7, len(self.ids)) for i in range(1,self.s_r_number_of_books+1): From 4d4595039245f49fa2461e60af59087ccc2a7653 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 17 Sep 2010 18:23:23 +0100 Subject: [PATCH 4/4] Fix problem with passing strings instead of lists for standard is_multiple fields --- src/calibre/gui2/dialogs/metadata_bulk.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 6bda4db138..173c69910f 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -317,7 +317,11 @@ def apply_pattern(val): v = apply_pattern(val).strip() if v: res.append(v) - val = fm['is_multiple'].join(res) + val = res + if fm['is_custom']: + # The standard tags and authors values want to be lists. + # All custom columns are to be strings + val = fm['is_multiple'].join(val) else: val = apply_pattern(val) @@ -330,8 +334,6 @@ def apply_pattern(val): setter = self.db.set_comment else: setter = getattr(self.db, 'set_'+field) - if field == 'authors': - val = string_to_authors(val) setter(id, val, notify=False, commit=False) self.db.commit()